diff --git a/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs b/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs new file mode 100644 index 000000000..19c726c0f --- /dev/null +++ b/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs @@ -0,0 +1,161 @@ +using System.Buffers; +using System.Text; + +using BenchmarkDotNet.Attributes; + +using LogExpert.Core.Classes.Log.Streamreaders; +using LogExpert.Core.Entities; + +namespace LogExpert.Benchmarks; + +/// +/// Measures the default (Direct) stream reader the way LogfileReader drives it: lines come +/// from TryReadLine, and every lines the char blocks are +/// detached and returned to the pool, as a LogBuffer does when it is evicted. +/// The valid-UTF-8 workloads are the regression gate for reader changes; the invalid-bytes +/// workload covers files with binary blocks (issue #639). +/// +[MemoryDiagnoser] +public class DirectReaderBenchmarks +{ + private const int LINES_PER_BUFFER = 500; + private const int MAX_LINE_LENGTH = 20_000; + + private byte[] _asciiMedium = null!; + private byte[] _asciiLarge = null!; + private byte[] _unicode = null!; + private byte[] _mixedLineEndings = null!; + private byte[] _longLines = null!; + private byte[] _utf16 = null!; + private byte[] _invalidBytes = null!; + + [GlobalSetup] + public void Setup () + { + _asciiMedium = Encoding.UTF8.GetBytes(GenerateAscii(10_000, 100, "\r\n")); + _asciiLarge = Encoding.UTF8.GetBytes(GenerateAscii(200_000, 200, "\r\n")); + _unicode = Encoding.UTF8.GetBytes(GenerateUnicode(100_000)); + _mixedLineEndings = Encoding.UTF8.GetBytes(GenerateMixedLineEndings(100_000)); + _longLines = Encoding.UTF8.GetBytes(GenerateAscii(500, 40_000, "\n")); + _utf16 = Encoding.Unicode.GetBytes(GenerateAscii(100_000, 100, "\r\n")); + _invalidBytes = GenerateInvalidBytes(100_000); + } + + [Benchmark(Baseline = true)] + public int AsciiMedium () => ReadAll(_asciiMedium, Encoding.UTF8); + + [Benchmark] + public int AsciiLarge () => ReadAll(_asciiLarge, Encoding.UTF8); + + [Benchmark] + public int Unicode () => ReadAll(_unicode, Encoding.UTF8); + + [Benchmark] + public int MixedLineEndings () => ReadAll(_mixedLineEndings, Encoding.UTF8); + + [Benchmark] + public int LongLines () => ReadAll(_longLines, Encoding.UTF8); + + [Benchmark] + public int Utf16 () => ReadAll(_utf16, Encoding.Unicode); + + [Benchmark] + public int InvalidBytes () => ReadAll(_invalidBytes, Encoding.UTF8); + + private static int ReadAll (byte[] data, Encoding encoding) + { + using var stream = new MemoryStream(data, writable: false); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, MAX_LINE_LENGTH); + + var lines = 0; + while (reader.TryReadLine(out _)) + { + if (++lines % LINES_PER_BUFFER == 0) + { + ReturnBlocks(reader.DetachCharBlocks()); + } + } + + ReturnBlocks(reader.DetachCharBlocks()); + return lines; + } + + private static void ReturnBlocks (List blocks) + { + foreach (var block in blocks) + { + ArrayPool.Shared.Return(block); + } + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")] + private static string GenerateAscii (int lineCount, int avgLineLength, string newLine) + { + var sb = new StringBuilder(); + var random = new Random(42); + + for (var i = 0; i < lineCount; i++) + { + var lineLength = avgLineLength + random.Next(-10, 11); + _ = sb.Append("2026-04-23 12:00:00.").Append(i % 1000).Append(" [INFO] ") + .Append('X', Math.Max(0, lineLength - 32)) + .Append(newLine); + } + + return sb.ToString(); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")] + private static string GenerateUnicode (int lineCount) + { + var sb = new StringBuilder(); + var random = new Random(42); + + for (var i = 0; i < lineCount; i++) + { + _ = random.Next(0, 3) switch + { + 0 => sb.Append("Line ").Append(i).Append(": ASCII text only, nothing special here\r\n"), + 1 => sb.Append("Line ").Append(i).Append(": Hello 世界, 日志文件 (Chinese)\r\n"), + _ => sb.Append("Line ").Append(i).Append(": Спасибо большое 🙂 (Russian, emoji)\r\n") + }; + } + + return sb.ToString(); + } + + private static string GenerateMixedLineEndings (int lineCount) + { + var sb = new StringBuilder(); + string[] endings = ["\r\n", "\n", "\r"]; + + for (var i = 0; i < lineCount; i++) + { + _ = sb.Append("2026-04-23 12:00:00 [WARN] mixed ending line ").Append(i).Append(endings[i % endings.Length]); + } + + return sb.ToString(); + } + + // CSV-like text with a random binary block every 1000 lines, the shape of the issue #639 sample. + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")] + private static byte[] GenerateInvalidBytes (int lineCount) + { + var random = new Random(42); + using var stream = new MemoryStream(); + + for (var i = 0; i < lineCount; i++) + { + if (i % 1000 == 500) + { + var binary = new byte[2048]; + random.NextBytes(binary); + stream.Write(binary); + } + + stream.Write(Encoding.UTF8.GetBytes($"06/23/2026 00:00:00,MXS SEP26,3Minute,Info,OnVarUpdate(),\"line {i}\"\r\n")); + } + + return stream.ToArray(); + } +} \ No newline at end of file diff --git a/src/LogExpert.Benchmarks/Program.cs b/src/LogExpert.Benchmarks/Program.cs index b29c91719..10dbe02e0 100644 --- a/src/LogExpert.Benchmarks/Program.cs +++ b/src/LogExpert.Benchmarks/Program.cs @@ -12,7 +12,7 @@ public static void Main (string[] args) Console.WriteLine("No benchmarks specified. Running all benchmarks..."); // Run all benchmarks if no arguments are provided - _ = BenchmarkRunner.Run(); + _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); @@ -25,7 +25,7 @@ public static void Main (string[] args) } Console.WriteLine("Replace with the name of the benchmark you want to run, e.g. "); - Console.WriteLine("StreamReaderBenchmarks: Benchmarks for stream readers"); + Console.WriteLine("DirectReaderBenchmarks: Benchmarks for the default (Direct) reader, the regression gate for reader changes"); Console.WriteLine("ReadThroughputBenchmarks: Benchmarks for read throughput"); Console.WriteLine("BufferIndexBenchmarks: Benchmarks for buffer index"); Console.WriteLine("BufferIndexContentionBenchmarks: Benchmarks for buffer index contention"); @@ -42,11 +42,11 @@ public static void Main (string[] args) /* * Comment / Uncommen the benchmark to run, careful some can run longer * 1.) a dry run - * dotnet run -c Release -- --filter "StreamReaderBenchmarks" --job Dry --noOverwrite + * dotnet run -c Release -- --filter "DirectReaderBenchmarks" --job Dry --noOverwrite * 2.) a short run - * dotnet run -c Release -- --filter "StreamReaderBenchmarks" --job Short --noOverwrite + * dotnet run -c Release -- --filter "DirectReaderBenchmarks" --job Short --noOverwrite * 3.) a full baseline run - * dotnet run -c Release -- --filter "StreamReaderBenchmarks" --noOverwrite + * dotnet run -c Release -- --filter "DirectReaderBenchmarks" --noOverwrite * * The full baseline run generates a MD file * BenchmarkDotNet.Artifacts/results/*-report-github.md diff --git a/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs b/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs index 688e2bd5e..2b2610743 100644 --- a/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs +++ b/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs @@ -4,7 +4,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; namespace LogExpert.Benchmarks; @@ -45,7 +44,6 @@ public int ReadWithNullReporter () bufferCount: 500, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -67,7 +65,6 @@ public int ReadWithPeriodicReporter () bufferCount: 500, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500); // No progressReporter = default PeriodicProgressReporter @@ -76,54 +73,6 @@ public int ReadWithPeriodicReporter () return reader.LineCount; } - /// - /// Post-change: read with block-based allocation (System reader uses CharBlockAllocator). - /// Compare Gen0/Gen1/Gen2 collections vs baseline to validate allocation reduction. - /// This method is identical to ReadWithNullReporter — it exists solely for explicit - /// before/after naming in benchmark reports. - /// - [Benchmark] - public int ReadWithBlockAllocation () - { - using var reader = new LogfileReader( - _tempFile, - new EncodingOptions { Encoding = Encoding.UTF8 }, - multiFile: false, - bufferCount: 500, - linesPerBuffer: 500, - new MultiFileOptions(), - ReaderType.System, - PluginRegistry.PluginRegistry.Instance, - maximumLineLength: 500, - progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); - - reader.ReadFiles(); - return reader.LineCount; - } - - /// - /// Direct-read: reads decoded chars directly into pooled blocks without per-line string allocation. - /// Compare allocations and GC counts vs ReadWithBlockAllocation. - /// - [Benchmark] - public int ReadWithDirectRead () - { - using var reader = new LogfileReader( - _tempFile, - new EncodingOptions { Encoding = Encoding.UTF8 }, - multiFile: false, - bufferCount: 500, - linesPerBuffer: 500, - new MultiFileOptions(), - ReaderType.SystemDirect, - PluginRegistry.PluginRegistry.Instance, - maximumLineLength: 500, - progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); - - reader.ReadFiles(); - return reader.LineCount; - } - [GlobalCleanup] public void Cleanup () { diff --git a/src/LogExpert.Benchmarks/StreamReaderBenchmarks.cs b/src/LogExpert.Benchmarks/StreamReaderBenchmarks.cs deleted file mode 100644 index a82f04329..000000000 --- a/src/LogExpert.Benchmarks/StreamReaderBenchmarks.cs +++ /dev/null @@ -1,152 +0,0 @@ -using System.Text; - -using BenchmarkDotNet.Attributes; - -using LogExpert.Core.Classes.Log.Streamreaders; -using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; - -namespace LogExpert.Benchmarks; - -[MemoryDiagnoser] -[RankColumn] -public class StreamReaderBenchmarks -{ - private byte[] _smallTestData; - private byte[] _mediumTestData; - private byte[] _largeTestData; - private byte[] _unicodeTestData; - - [GlobalSetup] - public void Setup () - { - // Small: 1000 lines, ~50 bytes each = ~50 KB - _smallTestData = GenerateTestData(1000, 50); - - // Medium: 10000 lines, ~100 bytes each = ~1 MB - _mediumTestData = GenerateTestData(10000, 100); - - // Large: 100000 lines, ~200 bytes each = ~20 MB - _largeTestData = GenerateTestData(100000, 200); - - // Unicode: 5000 lines with mixed ASCII and Unicode - _unicodeTestData = GenerateUnicodeTestData(5000); - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Test")] - private static byte[] GenerateTestData (int lineCount, int avgLineLength) - { - var sb = new StringBuilder(); - var random = new Random(42); // Fixed seed for reproducibility - - for (int i = 0; i < lineCount; i++) - { - var lineLength = avgLineLength + random.Next(-10, 11); // Vary line length slightly - var line = $"Line {i:D10} " + new string('X', Math.Max(0, lineLength - 20)); - _ = sb.AppendLine(line); - } - - return Encoding.UTF8.GetBytes(sb.ToString()); - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Test")] - private static byte[] GenerateUnicodeTestData (int lineCount) - { - var sb = new StringBuilder(); - var random = new Random(42); - - for (int i = 0; i < lineCount; i++) - { - var lineType = random.Next(0, 3); - var line = lineType switch - { - 0 => $"Line {i}: ASCII text only", - 1 => $"Line {i}: Hello 世界 (Chinese)", - _ => $"Line {i}: Спасибо большое (Russian)" - }; - _ = sb.AppendLine(line); - } - - return Encoding.UTF8.GetBytes(sb.ToString()); - } - - [Benchmark(Baseline = true)] - [BenchmarkCategory("Legacy", "Small", "ReadAll")] - public void Legacy_ReadAll_Small () - { - using var stream = new MemoryStream(_smallTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Small", "ReadAll")] - public void System_ReadAll_Small () - { - using var stream = new MemoryStream(_smallTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("Legacy", "Medium", "ReadAll")] - public void Legacy_ReadAll_Medium () - { - using var stream = new MemoryStream(_mediumTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Medium", "ReadAll")] - public void System_ReadAll_Medium () - { - using var stream = new MemoryStream(_mediumTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("Legacy", "Large", "ReadAll")] - public void Legacy_ReadAll_Large () - { - using var stream = new MemoryStream(_largeTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Large", "ReadAll")] - public void System_ReadAll_Large () - { - using var stream = new MemoryStream(_largeTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Unicode", "ReadAll")] - public void System_ReadAll_Unicode () - { - using var stream = new MemoryStream(_unicodeTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("Legacy", "Unicode", "ReadAll")] - public void Legacy_ReadAll_Unicode () - { - using var stream = new MemoryStream(_unicodeTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - private static void ReadAllLines (ILogStreamReader reader) - { - while (reader.ReadLine() != null) - { - // Consume the line - } - } -} \ No newline at end of file diff --git a/src/LogExpert.Core/Classes/Log/LogfileReader.cs b/src/LogExpert.Core/Classes/Log/LogfileReader.cs index 142f3ad6d..d0a23ef7e 100644 --- a/src/LogExpert.Core/Classes/Log/LogfileReader.cs +++ b/src/LogExpert.Core/Classes/Log/LogfileReader.cs @@ -8,7 +8,6 @@ using LogExpert.Core.Classes.Log.Streamreaders; using LogExpert.Core.Classes.xml; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.EventArguments; using LogExpert.Core.Interfaces; @@ -29,7 +28,6 @@ public partial class LogfileReader : ILogfileReader, IMultiFileNavigation, ILogf private readonly MultiFileOptions _multiFileOptions; private readonly IPluginRegistry _pluginRegistry; private readonly CancellationTokenSource _cts = new(); - private readonly ReaderType _readerType; private readonly int _maximumLineLength; private readonly LogBufferPool _bufferPool; @@ -67,7 +65,6 @@ public LogfileReader ( int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, - ReaderType readerType, IPluginRegistry pluginRegistry, int maximumLineLength, ILoadProgressReporter? progressReporter = null) @@ -78,7 +75,6 @@ public LogfileReader ( bufferCount, linesPerBuffer, multiFileOptions, - readerType, pluginRegistry, maximumLineLength, progressReporter) @@ -92,7 +88,6 @@ public LogfileReader ( int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, - ReaderType readerType, IPluginRegistry pluginRegistry, int maximumLineLength, ILoadProgressReporter? progressReporter = null, @@ -104,7 +99,6 @@ public LogfileReader ( bufferCount, linesPerBuffer, multiFileOptions, - readerType, pluginRegistry, maximumLineLength, progressReporter, @@ -121,7 +115,6 @@ private LogfileReader ( int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, - ReaderType readerType, IPluginRegistry pluginRegistry, int maximumLineLength, ILoadProgressReporter? progressReporter = null, @@ -140,7 +133,6 @@ private LogfileReader ( } _maximumLineLength = maximumLineLength; - _readerType = readerType; EncodingOptions = encodingOptions; _max_buffers = bufferCount; _maxLinesPerBuffer = linesPerBuffer; @@ -1709,14 +1701,7 @@ private ILogStreamReader GetLogStreamReader (Stream stream, EncodingOptions enco /// private ILogStreamReader CreateLogStreamReader (Stream stream, EncodingOptions encodingOptions) { - return _readerType switch - { - ReaderType.Legacy => new PositionAwareStreamReaderLegacy(stream, encodingOptions, _maximumLineLength), - ReaderType.System => new PositionAwareStreamReaderSystem(stream, encodingOptions, _maximumLineLength), - ReaderType.SystemDirect => new PositionAwareStreamReaderDirect(stream, encodingOptions, _maximumLineLength), - //Default will be SystemDirect, because it is the best performing reader and should be used if not explicitly overridden by user. - _ => new PositionAwareStreamReaderDirect(stream, encodingOptions, _maximumLineLength), - }; + return new PositionAwareStreamReaderDirect(stream, encodingOptions, _maximumLineLength); } /// diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs index ac234e5c9..7c60d3f17 100644 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs +++ b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs @@ -148,10 +148,14 @@ protected virtual void ResetReader () _reader.DiscardBufferedData(); } - protected StreamReader GetStreamReader () + /// + /// The raw byte stream, positioned where points (past the preamble), for + /// readers that decode bytes themselves; reads through the base StreamReader. + /// + protected Stream GetByteStream () { ObjectDisposedException.ThrowIf(IsDisposed, GetType()); - return _reader; + return _stream; } protected void MovePosition (int offset) diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs index a02989d3d..bee4c10b1 100644 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs +++ b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Runtime.InteropServices; using System.Text; using LogExpert.Core.Entities; @@ -7,30 +8,44 @@ namespace LogExpert.Core.Classes.Log.Streamreaders; /// -/// Reads decoded characters directly into pooled char[] blocks via StreamReader.Read(), -/// scans for newline boundaries, and returns ReadOnlyMemory<char> slices without -/// allocating per-line strings. Eliminates the StreamReader.ReadLine() string allocation. +/// Finds line boundaries in the raw bytes, decodes each line into pooled char[] blocks and returns +/// ReadOnlyMemory<char> slices without allocating per-line strings. /// +/// +/// advances by the raw bytes of each line, so it +/// stays exact for any line ending and for bytes that are invalid in the encoding (ADR 0010). +/// reads through the base ; do not interleave it with +/// without setting the position in between. +/// public class PositionAwareStreamReaderDirect : PositionAwareStreamReaderBase, ILogStreamReaderMemory { #region Constants - private const int BLOCK_SIZE = 32_768; // 64 KB (32K chars × 2 bytes), under LOH threshold - private const char CHAR_LF = '\n'; - private const char CHAR_CR = '\r'; + private const int BLOCK_SIZE = 32_768; // 64 KB (32K chars × 2 bytes), under LOH threshold + private const int BYTE_BUFFER_SIZE = 65_536; + private const int CODE_PAGE_UTF16_LE = 1200; - private static readonly SearchValues _lineTerminators = SearchValues.Create("\r\n"); + private static readonly SearchValues _surrogates = SearchValues.Create(Enumerable.Range(0xD800, 0x800).Select(code => (char)code).ToArray()); #endregion #region Fields private char[] _readBlock; - private int _readBlockLength; // valid chars in _readBlock - private int _scanOffset; // current scan position in _readBlock + private int _usedChars; // chars of _readBlock already handed out as lines + private byte[] _byteBuffer; + private int _byteStart; // first byte of the line in progress + private int _byteEnd; + private int _scannedBytes; // bytes after _byteStart already searched without a terminator private bool _eof; - private bool _initialized; // first block filled from the current stream position - private int _terminatorCharByteSize; // bytes for a single '\r' or '\n' in the active encoding + private bool _initialized; + + private int _unitSize; // bytes per code unit: 1, 2 or 4 + private uint _crUnit; + private uint _lfUnit; + private bool _asciiCompatible; + private bool _isUtf16LittleEndian; + private readonly List _completedBlocks = []; public override bool IsDisposed { get; protected set; } @@ -43,9 +58,6 @@ public PositionAwareStreamReaderDirect (Stream stream, EncodingOptions encodingO : base(stream, encodingOptions, maximumLineLength) { _readBlock = ArrayPool.Shared.Rent(BLOCK_SIZE); - _readBlockLength = 0; - _scanOffset = 0; - _eof = false; } #endregion @@ -58,86 +70,58 @@ public override string ReadLine () } /// - /// Reads the next line by scanning the current block for the next line terminator - /// (\n, \r\n, or a bare \r). If the block is exhausted, tail-copies - /// the partial line to a new block and refills. Returns a zero-copy - /// ReadOnlyMemory<char> slice into the pooled block. The byte position advances by the - /// content bytes plus the bytes of the actual terminator, so it stays exact on - /// files with mixed line endings. + /// Reads the next line, ended by \n, \r\n or a bare \r, and returns a zero-copy + /// slice of its decoded text, capped at the maximum line length. The byte position advances by the + /// line's raw bytes plus the bytes of its actual terminator. /// public bool TryReadLine (out ReadOnlyMemory lineMemory) { - var reader = GetStreamReader(); - - EnsureInitialized(reader); + EnsureInitialized(); while (true) { - // If we have data to scan, look for the next \r or \n - if (_scanOffset < _readBlockLength) + var available = _byteEnd - _byteStart; + var hitIndex = FindTerminator(_byteBuffer.AsSpan(_byteStart + _scannedBytes, available - _scannedBytes)); + + if (hitIndex >= 0) { - var searchSpan = _readBlock.AsSpan(_scanOffset, _readBlockLength - _scanOffset); - var hitIndex = searchSpan.IndexOfAny(_lineTerminators); + hitIndex += _scannedBytes; - if (hitIndex >= 0) + int terminatorBytes; + if (UnitAt(hitIndex) == _lfUnit) { - // Number of char cells the terminator occupies (1 for \n or bare \r, 2 for \r\n). - int terminatorChars; - - if (searchSpan[hitIndex] == CHAR_LF) - { - terminatorChars = 1; - } - else if (hitIndex + 1 < searchSpan.Length) - { - // \r with a known following char: \r\n if it's \n, otherwise a bare \r. - terminatorChars = searchSpan[hitIndex + 1] == CHAR_LF ? 2 : 1; - } - else if (_eof) - { - // \r is the very last char in the file: a bare \r. - terminatorChars = 1; - } - else - { - // \r is the last char of the block but more data follows. Refill so the - // next char becomes available (the tail-copy carries the \r forward), - // then re-scan to classify it as \r\n or a bare \r. - RefillBlock(reader); - continue; - } - - var lineLength = hitIndex; - - // Enforce MaximumLineLength on the returned slice, but count the full - // content for the byte position. - var cappedLength = Math.Min(lineLength, MaximumLineLength); - lineMemory = _readBlock.AsMemory(_scanOffset, cappedLength); - - var contentSpan = _readBlock.AsSpan(_scanOffset, lineLength); - MovePosition(Encoding.GetByteCount(contentSpan) + (terminatorChars * _terminatorCharByteSize)); - - // Advance scan past the content and its terminator. - _scanOffset += hitIndex + terminatorChars; - - return true; + terminatorBytes = _unitSize; } + else if (hitIndex + (2 * _unitSize) <= available) + { + // \r with a known following unit: \r\n if it's \n, otherwise a bare \r. + terminatorBytes = UnitAt(hitIndex + _unitSize) == _lfUnit ? 2 * _unitSize : _unitSize; + } + else if (_eof) + { + // \r is the very last unit in the file: a bare \r. + terminatorBytes = _unitSize; + } + else + { + // \r is the last unit read so far: refill, then re-scan from the \r to classify it. + _scannedBytes = hitIndex; + Refill(); + continue; + } + + lineMemory = DecodeLine(hitIndex); + ConsumeBytes(hitIndex + terminatorBytes); + return true; } - // No terminator found (or no data at all). Need to refill. if (_eof) { // Emit remaining content as final line (no trailing newline) - if (_scanOffset < _readBlockLength) + if (available > 0) { - var remaining = _readBlockLength - _scanOffset; - var cappedLength = Math.Min(remaining, MaximumLineLength); - lineMemory = _readBlock.AsMemory(_scanOffset, cappedLength); - - var fullSpan = _readBlock.AsSpan(_scanOffset, remaining); - MovePosition(Encoding.GetByteCount(fullSpan)); - - _scanOffset = _readBlockLength; + lineMemory = DecodeLine(available); + ConsumeBytes(available); return true; } @@ -145,8 +129,9 @@ public bool TryReadLine (out ReadOnlyMemory lineMemory) return false; } - // Tail-copy: move unconsumed chars to a new block and refill - RefillBlock(reader); + // Only whole units are searched; a partial one is completed by the refill. + _scannedBytes = available - (available % _unitSize); + Refill(); } } @@ -156,129 +141,246 @@ public void ReturnMemory (ReadOnlyMemory memory) } /// - /// Detaches completed blocks (fully scanned) for transfer to the LogBuffer. - /// The current _readBlock (partially scanned) stays with the reader. + /// Detaches the blocks that back the lines read so far, for transfer to the LogBuffer. + /// The reader continues in a fresh block. /// public List DetachCharBlocks () { - // Nothing to detach: no completed blocks and no lines were scanned from the current block. - if (_completedBlocks.Count == 0 && _scanOffset == 0) + if (_usedChars > 0) + { + _completedBlocks.Add(_readBlock); + _readBlock = ArrayPool.Shared.Rent(BLOCK_SIZE); + _usedChars = 0; + } + + if (_completedBlocks.Count == 0) { return []; } - // The current _readBlock contains memory backing lines already added to the LogBuffer. - // It must be transferred to the buffer along with any completed blocks. - _completedBlocks.Add(_readBlock); + var blocks = _completedBlocks.ToList(); + _completedBlocks.Clear(); + return blocks; + } + + #endregion - // Rent a fresh block and carry over any unscanned data (partial line in progress) - var tailLength = _readBlockLength - _scanOffset; + #region Private Methods - // The tail may exceed BLOCK_SIZE after reading a long line (buffer was grown). - var newBlockSize = BLOCK_SIZE; - while (tailLength > newBlockSize) + private void EnsureInitialized () + { + if (_initialized) { - newBlockSize *= 2; + return; } - var newBlock = ArrayPool.Shared.Rent(newBlockSize); + _initialized = true; + _byteBuffer ??= ArrayPool.Shared.Rent(BYTE_BUFFER_SIZE); - if (tailLength > 0) + if (_unitSize == 0) { - _readBlock.AsSpan(_scanOffset, tailLength).CopyTo(newBlock.AsSpan(0, tailLength)); + ResolveTerminatorUnits(); } + } - _readBlock = newBlock; - _readBlockLength = tailLength; - _scanOffset = 0; + // In every encoding LogExpert offers, \r and \n are single code units (1 byte, or 2/4 for + // UTF-16/UTF-32) that never occur inside a multi-byte character, so lines can be split on bytes. + private void ResolveTerminatorUnits () + { + var cr = Encoding.GetBytes("\r"); + var lf = Encoding.GetBytes("\n"); - var blocks = _completedBlocks.ToList(); - _completedBlocks.Clear(); - return blocks; + if (cr.Length != lf.Length || lf.Length is not (1 or 2 or 4)) + { + throw new NotSupportedException($"Encoding '{Encoding.WebName}' does not encode line terminators as single code units."); + } + + _unitSize = lf.Length; + _crUnit = ReadUnit(cr, _unitSize); + _lfUnit = ReadUnit(lf, _unitSize); + _asciiCompatible = _unitSize == 1 && DecodesAsciiUnchanged(Encoding); + _isUtf16LittleEndian = Encoding.CodePage == CODE_PAGE_UTF16_LE; } - #endregion + // False for encodings like UTF-7 or EBCDIC, whose 0x00–0x7F bytes are not plain ASCII. + private static bool DecodesAsciiUnchanged (Encoding encoding) + { + Span ascii = stackalloc byte[128]; + for (var i = 0; i < ascii.Length; i++) + { + ascii[i] = (byte)i; + } - #region Private Methods + Span decoded = stackalloc char[encoding.GetMaxCharCount(ascii.Length)]; + var charCount = encoding.GetChars(ascii, decoded); + if (charCount != ascii.Length) + { + return false; + } + + for (var i = 0; i < ascii.Length; i++) + { + if (decoded[i] != i) + { + return false; + } + } + + return true; + } - private void RefillBlock (StreamReader reader) + private static uint ReadUnit (ReadOnlySpan bytes, int unitSize) { - var tailLength = _readBlockLength - _scanOffset; + return unitSize switch + { + 1 => bytes[0], + 2 => MemoryMarshal.Read(bytes), + _ => MemoryMarshal.Read(bytes) + }; + } - // Determine new block size: if the tail already fills a standard block, - // grow the buffer so there's room to read more data. This handles lines - // longer than BLOCK_SIZE (e.g. huge XML payloads). - var newBlockSize = BLOCK_SIZE; - while (tailLength >= newBlockSize) + private uint UnitAt (int offset) + { + return _unitSize == 1 + ? _byteBuffer[_byteStart + offset] + : ReadUnit(_byteBuffer.AsSpan(_byteStart + offset, _unitSize), _unitSize); + } + + private int FindTerminator (ReadOnlySpan bytes) + { + switch (_unitSize) { - newBlockSize *= 2; + case 1: + { + return bytes.IndexOfAny((byte)_crUnit, (byte)_lfUnit); + } + case 2: + { + var index = MemoryMarshal.Cast(bytes).IndexOfAny((ushort)_crUnit, (ushort)_lfUnit); + return index < 0 ? -1 : index * 2; + } + default: + { + var index = MemoryMarshal.Cast(bytes).IndexOfAny(_crUnit, _lfUnit); + return index < 0 ? -1 : index * 4; + } } + } - // Rent a new block (may be larger than BLOCK_SIZE for very long lines) - var newBlock = ArrayPool.Shared.Rent(newBlockSize); + // A line is decoded on its own: a sequence cut short by the terminator becomes U+FFFD in this line. + private ReadOnlyMemory DecodeLine (int byteCount) + { + var bytes = _byteBuffer.AsSpan(_byteStart, byteCount); - // Copy the tail (partial line) to the start of the new block - if (tailLength > 0) + int charCount; + if (_asciiCompatible) { - _readBlock.AsSpan(_scanOffset, tailLength).CopyTo(newBlock.AsSpan(0, tailLength)); + // Pure-ASCII lines skip the per-call cost of the encoding's decoder. + EnsureCharCapacity(byteCount); + if (Ascii.ToUtf16(bytes, _readBlock.AsSpan(_usedChars), out charCount) != OperationStatus.Done) + { + charCount = DecodeWithEncoding(bytes); + } + } + else if (_isUtf16LittleEndian && (byteCount & 1) == 0 && !MemoryMarshal.Cast(bytes).ContainsAny(_surrogates)) + { + // UTF-16 LE without surrogates decodes to its own code units. + var source = MemoryMarshal.Cast(bytes); + EnsureCharCapacity(source.Length); + source.CopyTo(_readBlock.AsSpan(_usedChars)); + charCount = source.Length; + } + else + { + charCount = DecodeWithEncoding(bytes); } - // The old block is fully scanned — add to completed list - _completedBlocks.Add(_readBlock); + var cappedLength = Math.Min(charCount, MaximumLineLength); - _readBlock = newBlock; - _scanOffset = 0; + var lineMemory = _readBlock.AsMemory(_usedChars, cappedLength); + _usedChars += cappedLength; + return lineMemory; + } - // Fill the rest of the block from the stream - var available = newBlock.Length - tailLength; - var charsRead = reader.Read(newBlock, tailLength, available); + private int DecodeWithEncoding (ReadOnlySpan bytes) + { + EnsureCharCapacity(Encoding.GetMaxCharCount(bytes.Length)); + return Encoding.GetChars(bytes, _readBlock.AsSpan(_usedChars)); + } - _readBlockLength = tailLength + charsRead; + private void ConsumeBytes (int byteCount) + { + MovePosition(byteCount); + _byteStart += byteCount; + _scannedBytes = 0; + } - if (charsRead == 0) + // A block that already holds lines is kept for DetachCharBlocks; lines longer than BLOCK_SIZE get a block of their own. + private void EnsureCharCapacity (int charCount) + { + if (_readBlock.Length - _usedChars >= charCount) { - _eof = true; + return; + } + + if (_usedChars > 0) + { + _completedBlocks.Add(_readBlock); } + else + { + ArrayPool.Shared.Return(_readBlock); + } + + var blockSize = BLOCK_SIZE; + while (blockSize < charCount) + { + blockSize *= 2; + } + + _readBlock = ArrayPool.Shared.Rent(blockSize); + _usedChars = 0; } - /// - /// Lazily fills the first block from the current stream position. Called on the first read - /// after construction and after any seek - /// (which resets _initialized via ). - /// - private void EnsureInitialized (StreamReader reader) + // When the buffer is full, the line in progress moves to the front, or the buffer doubles if that line fills it. + private void Refill () { - if (_initialized) + if (_byteEnd == _byteBuffer.Length) { - return; - } + var lineLength = _byteEnd - _byteStart; - _initialized = true; + if (_byteStart > 0) + { + _byteBuffer.AsSpan(_byteStart, lineLength).CopyTo(_byteBuffer); + } + else + { + var grown = ArrayPool.Shared.Rent(_byteBuffer.Length * 2); + _byteBuffer.AsSpan(0, lineLength).CopyTo(grown); + ArrayPool.Shared.Return(_byteBuffer); + _byteBuffer = grown; + } - // A single '\r' and a single '\n' encode to the same number of bytes in every encoding - // LogExpert uses (ASCII control chars), so one value covers \n, \r and (×2) \r\n. - Span singleTerminator = [CHAR_LF]; - _terminatorCharByteSize = Encoding.GetByteCount(singleTerminator); + _byteStart = 0; + _byteEnd = lineLength; + } - var charsRead = reader.Read(_readBlock, 0, BLOCK_SIZE); - _readBlockLength = charsRead; - _scanOffset = 0; + var bytesRead = GetByteStream().Read(_byteBuffer, _byteEnd, _byteBuffer.Length - _byteEnd); + _byteEnd += bytesRead; - if (charsRead == 0) + if (bytesRead == 0) { _eof = true; } } - /// - /// Resets scan state so the next read re-fills from the (just seeked) stream position. - /// Only touches value-type fields, which is safe under the base constructor's virtual call - /// to this method (the pooled _readBlock is left untouched). - /// + // Only value-type fields: the base constructor calls this before the fields are initialized. + // Lines already handed out stay valid until DetachCharBlocks. protected override void ResetReader () { - _scanOffset = 0; - _readBlockLength = 0; + _byteStart = 0; + _byteEnd = 0; + _scannedBytes = 0; _eof = false; _initialized = false; @@ -289,13 +391,19 @@ protected override void Dispose (bool disposing) { if (disposing) { - // Return the current (partially scanned) block + // Return the current (partially filled) block if (_readBlock != null) { ArrayPool.Shared.Return(_readBlock); _readBlock = null!; } + if (_byteBuffer != null) + { + ArrayPool.Shared.Return(_byteBuffer); + _byteBuffer = null!; + } + // Return any completed blocks not yet detached foreach (var block in _completedBlocks) { diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderLegacy.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderLegacy.cs deleted file mode 100644 index 2a2880be9..000000000 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderLegacy.cs +++ /dev/null @@ -1,149 +0,0 @@ -using LogExpert.Core.Classes.Log.Buffers; -using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; - -namespace LogExpert.Core.Classes.Log.Streamreaders; - -public class PositionAwareStreamReaderLegacy (Stream stream, EncodingOptions encodingOptions, int maximumLineLength) : PositionAwareStreamReaderBase(stream, encodingOptions, maximumLineLength), ILogStreamReaderMemory -{ - #region Fields - - private readonly char[] _charBuffer = new char[maximumLineLength]; - - private int _charBufferPos; - private bool _crDetect; - - public override bool IsDisposed { get; protected set; } - - #endregion - - #region Properties - - private CharBlockAllocator BlockAllocator - { - get => field ??= new CharBlockAllocator(); - set; - } - - #endregion - - #region Public methods - - public bool TryReadLine (out ReadOnlyMemory lineMemory) - { - var line = ReadLine(); - - if (line is null) - { - lineMemory = default; - return false; - } - - var target = BlockAllocator.Rent(line.Length); - line.AsSpan().CopyTo(target.Span); - lineMemory = target; - return true; - } - - public void ReturnMemory (ReadOnlyMemory memory) - { - // Bulk return via DetachCharBlocks() when the LogBuffer is evicted, or Dispose(). - } - - public List DetachCharBlocks () => BlockAllocator.DetachBlocks(); - - protected override void Dispose (bool disposing) - { - if (disposing) - { - BlockAllocator?.Dispose(); - BlockAllocator = null; - } - - base.Dispose(disposing); - } - - public override string ReadLine () - { - int readInt; - - while (-1 != (readInt = ReadChar())) - { - var readChar = (char)readInt; - - switch (readChar) - { - case '\n': - { - _crDetect = false; - return GetLineAndResetCharBufferPos(); - } - case '\r': - { - if (_crDetect) - { - return GetLineAndResetCharBufferPos(); - } - - _crDetect = true; - break; - } - default: - { - if (_crDetect) - { - _crDetect = false; - var line = GetLineAndResetCharBufferPos(); - AppendToCharBuffer(readChar); - return line; - } - - AppendToCharBuffer(readChar); - break; - } - } - } - - var result = GetLineAndResetCharBufferPos(); - if (readInt == -1 && result.Length == 0 && !_crDetect) - { - return null; // EOF - } - - _crDetect = false; - return result; - } - - protected override void ResetReader () - { - ResetCharBufferPos(); - - base.ResetReader(); - } - - #endregion - - #region Private Methods - - private string GetLineAndResetCharBufferPos () - { - string result = new(_charBuffer, 0, _charBufferPos); - ResetCharBufferPos(); - return result; - } - - private void AppendToCharBuffer (char readChar) - { - if (_charBufferPos < MaximumLineLength) - { - _charBuffer[_charBufferPos++] = readChar; - } - } - - private void ResetCharBufferPos () - { - _charBufferPos = 0; - } - - #endregion -} \ No newline at end of file diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderSystem.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderSystem.cs deleted file mode 100644 index 55090406e..000000000 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderSystem.cs +++ /dev/null @@ -1,174 +0,0 @@ -using System.Text; - -using LogExpert.Core.Classes.Log.Buffers; -using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; - -namespace LogExpert.Core.Classes.Log.Streamreaders; - -/// -/// This class is responsible for reading line from the log file. It also decodes characters with the appropriate charset encoding. -/// PositionAwareStreamReaderSystem tries a BOM detection to determine correct file offsets when directly seeking into the file (on re-loading flushed buffers). -/// UTF-8 handling is a bit slower, because after reading a character the byte length of the character must be determined. -/// Lines are read char-by-char. StreamReader.ReadLine() is not used because StreamReader cannot tell a file position. -/// -public class PositionAwareStreamReaderSystem : PositionAwareStreamReaderBase, ILogStreamReaderMemory -{ - #region Fields - - private const int CHAR_CR = 0x0D; - private const int CHAR_LF = 0x0A; - - private int _newLineSequenceLength; - - public override bool IsDisposed { get; protected set; } - - #endregion - - #region cTor - - public PositionAwareStreamReaderSystem (Stream stream, EncodingOptions encodingOptions, int maximumLineLength) : base(stream, encodingOptions, maximumLineLength) - { - } - - #endregion - - #region Properties - - /// - /// The block allocator used by this reader instance. Blocks are handed to the owning buffer via - /// ; ownership of the allocator does not cross the reader seam. - /// - private CharBlockAllocator BlockAllocator - { - get => field ??= new CharBlockAllocator(); - set; - } - - #endregion - - #region Public methods - - public override string ReadLine () - { - var reader = GetStreamReader(); - - if (_newLineSequenceLength == 0) - { - _newLineSequenceLength = GuessNewLineSequenceLength(reader); - } - - var line = reader.ReadLine(); - - if (line != null) - { - MovePosition(Encoding.GetByteCount(line) + _newLineSequenceLength); - - if (line.Length > MaximumLineLength) - { - line = line[..MaximumLineLength]; - } - } - - return line; - } - - /// - /// Attempts to read the next line from the stream without allocating a new string. - /// The returned Memory<char> is valid until the next call to TryReadLine or ReturnMemory. - /// - public bool TryReadLine (out ReadOnlyMemory lineMemory) - { - var reader = GetStreamReader(); - - if (_newLineSequenceLength == 0) - { - _newLineSequenceLength = GuessNewLineSequenceLength(reader); - } - - var line = reader.ReadLine(); - - if (line is null) - { - lineMemory = default; - return false; - } - - MovePosition(Encoding.GetByteCount(line) + _newLineSequenceLength); - - var length = Math.Min(line.Length, MaximumLineLength); - - // Allocate from block and copy - var allocator = BlockAllocator; - var target = allocator.Rent(length); - line.AsSpan(0, length).CopyTo(target.Span); - lineMemory = target; - return true; - } - - /// - /// Returns the memory buffer. For the block-based reader, individual returns are not tracked — blocks are returned - /// in bulk via the BlockAllocator when the LogBuffer is evicted or the reader is disposed. - /// - public void ReturnMemory (ReadOnlyMemory memory) - { - // Bulk return via DetachCharBlocks() or Dispose(). - // Individual per-line return is not needed with block-based allocation. - } - - public List DetachCharBlocks () => BlockAllocator.DetachBlocks(); - - #endregion - - #region Private Methods - - private int GuessNewLineSequenceLength (StreamReader reader) - { - var currentPos = Position; - - try - { - var line = reader.ReadLine(); - - if (line != null) - { - Position += Encoding.GetByteCount(line); - - var firstChar = reader.Read(); - if (firstChar == CHAR_CR) // check \r - { - var secondChar = reader.Read(); - if (secondChar == CHAR_LF) // check \n - { - // Use stackalloc or SpanOwner instead of string - Span newline = ['\r', '\n']; - return Encoding.GetByteCount(newline); - //return Encoding.GetByteCount("\r\n"); - } - } - - Span singleChar = [(char)firstChar]; - return Encoding.GetByteCount(singleChar); - } - - return 0; - } - finally - { - Position = currentPos; - } - } - - protected override void Dispose (bool disposing) - { - if (disposing) - { - BlockAllocator?.Dispose(); - BlockAllocator = null; - } - - base.Dispose(disposing); - } - - #endregion -} \ No newline at end of file diff --git a/src/LogExpert.Core/Config/Preferences.cs b/src/LogExpert.Core/Config/Preferences.cs index c3125158c..1bcef9106 100644 --- a/src/LogExpert.Core/Config/Preferences.cs +++ b/src/LogExpert.Core/Config/Preferences.cs @@ -77,8 +77,7 @@ public List HilightGroupList /// /// /// This property controls line truncation at the I/O reader level before lines are processed by columnizers. - /// It is implemented in and - /// . Related property: + /// It is implemented in . Related property: /// controls display-level truncation in UI columns, which must not exceed this /// value. Default is 20000 characters. /// @@ -134,13 +133,6 @@ public bool? DarkMode } } - [Obsolete("This setting is no longer used and will be removed in version 1.50. The 'UseLegacyReader' now works with ReaderType.Legacy")] - [System.Text.Json.Serialization.JsonIgnore] - [Newtonsoft.Json.JsonIgnore] - public bool UseLegacyReader { get; set; } - - public ReaderType ReaderType { get; set; } = ReaderType.SystemDirect; - public List ToolEntries { get; set; } = []; public DragOrientations TimestampControlDragOrientation { get; set; } = DragOrientations.Horizontal; diff --git a/src/LogExpert.Core/Enums/ReaderType.cs b/src/LogExpert.Core/Enums/ReaderType.cs deleted file mode 100644 index facd26c45..000000000 --- a/src/LogExpert.Core/Enums/ReaderType.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; - -namespace LogExpert.Core.Enums; - -/// -/// Defines the available stream reader implementations. -/// -[JsonConverter(typeof(StringEnumConverter))] -public enum ReaderType -{ - /// - /// Direct-read implementation: reads decoded chars directly into pooled blocks via - /// StreamReader.Read(char[], offset, count), eliminating per-line string allocation. - /// - SystemDirect = 0, - - /// - /// Legacy reader implementation (original). - /// - Legacy = 1, - - /// - /// System.IO.StreamReader based implementation. - /// - System = 2 -} diff --git a/src/LogExpert.Core/Interfaces/ILogStreamReader.cs b/src/LogExpert.Core/Interfaces/ILogStreamReader.cs index 8c8837986..7303eee2d 100644 --- a/src/LogExpert.Core/Interfaces/ILogStreamReader.cs +++ b/src/LogExpert.Core/Interfaces/ILogStreamReader.cs @@ -15,9 +15,7 @@ namespace LogExpert.Core.Interfaces; /// /// Implementations include: /// -/// PositionAwareStreamReaderDirect - Default. Reads decoded chars into pooled blocks and returns zero-copy slices; fastest, assumes \n/\r\n line endings -/// PositionAwareStreamReaderSystem - Uses .NET's StreamReader.ReadLine(); also splits bare \r (classic-Mac) line endings -/// PositionAwareStreamReaderLegacy - Character-by-character reading; exact byte position on mixed/pathological line endings, slowest +/// PositionAwareStreamReaderDirect - Finds lines in the raw bytes, decodes them into pooled blocks and returns zero-copy slices; exact byte position for any content and line ending (ADR 0010) /// XmlLogReader - Decorator for reading structured XML log blocks (e.g., Log4j XML format) /// /// diff --git a/src/LogExpert.Resources/Resources.Designer.cs b/src/LogExpert.Resources/Resources.Designer.cs index e877eb2ea..99ba15df0 100644 --- a/src/LogExpert.Resources/Resources.Designer.cs +++ b/src/LogExpert.Resources/Resources.Designer.cs @@ -5960,15 +5960,6 @@ public static string SettingsDialog_UI_CheckBox_ToolTip_toolTipPortableMode { } } - /// - /// 查找类似 File Reader algorithm 的本地化字符串。 - /// - public static string SettingsDialog_UI_CheckBox_ToolTip_toolTipReaderTyp { - get { - return ResourceManager.GetString("SettingsDialog_UI_CheckBox_ToolTip_toolTipReaderTyp", resourceCulture); - } - } - /// /// 查找类似 HeaderName 的本地化字符串。 /// diff --git a/src/LogExpert.Resources/Resources.de.resx b/src/LogExpert.Resources/Resources.de.resx index 594df5d81..09297e16c 100644 --- a/src/LogExpert.Resources/Resources.de.resx +++ b/src/LogExpert.Resources/Resources.de.resx @@ -900,9 +900,6 @@ Damit die anderen Fenster beim Auswählen einer Zeile (Mausklick oder Pfeiltaste Wähle ein Verzeichnis für die LogExpert Sessiondateien - - Dateiauslese algorithmus - Bei Aktivierung des Modus, wird die gespeicherte Datei aus dem Verzeichnis der Executable geladen diff --git a/src/LogExpert.Resources/Resources.resx b/src/LogExpert.Resources/Resources.resx index 078363a3c..f1c165757 100644 --- a/src/LogExpert.Resources/Resources.resx +++ b/src/LogExpert.Resources/Resources.resx @@ -1030,9 +1030,6 @@ Checked tools will appear in the icon bar. All other tools are available in the If this mode is activated, the save file will be loaded from the Executable Location - - File Reader algorithm - Settings diff --git a/src/LogExpert.Resources/Resources.zh-CN.resx b/src/LogExpert.Resources/Resources.zh-CN.resx index 9ed0b1dd2..7b93aec6b 100644 --- a/src/LogExpert.Resources/Resources.zh-CN.resx +++ b/src/LogExpert.Resources/Resources.zh-CN.resx @@ -919,9 +919,6 @@ 如果启用此模式,保存文件将从可执行文件所在位置加载 - - 文件读取算法 - 设置 diff --git a/src/LogExpert.Tests/Buffers/BufferShiftTest.cs b/src/LogExpert.Tests/Buffers/BufferShiftTest.cs index 8233da597..bf34f19c6 100644 --- a/src/LogExpert.Tests/Buffers/BufferShiftTest.cs +++ b/src/LogExpert.Tests/Buffers/BufferShiftTest.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.PluginRegistry.FileSystem; using NUnit.Framework; @@ -25,13 +24,11 @@ public void Boot () } [Test] - [TestCase(ReaderType.System)] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Test")] - //[TestCase(ReaderType.Legacy)] Legacy Reader does not Support this //TO Test real life scenario, use the LogRotator tool, in the src/Tools/LogRotator directory, //to create files and perform rollovers while watching the files in LogExpert with MultiFile enabled //(pattern: *$J(.)) - public void TestShiftBuffers1 (ReaderType readerType) + public void TestShiftBuffers1 () { var linesPerFile = 10; MultiFileOptions options = new() @@ -48,7 +45,7 @@ public void TestShiftBuffers1 (ReaderType readerType) }; _ = PluginRegistry.PluginRegistry.Create(TestDirectory.FullName, 500); - LogfileReader reader = new(files.Last.Value, encodingOptions, true, 40, 50, options, readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(files.Last.Value, encodingOptions, true, 40, 50, options, PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); var lil = reader.GetLogFileInfoList(); diff --git a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs index 0e03e196b..106a74ae5 100644 --- a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs @@ -6,7 +6,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using Moq; @@ -282,14 +281,14 @@ public void SplitLine_BadCsvData_ReturnsSingleColumnInsteadOfCrash () Assert.That(result.ColumnValues[0].FullValue.ToString(), Does.Contain("6774DC1dB00BD11")); } - [TestCase(@".\TestData\organizations-10000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" }, ReaderType.System)] - [TestCase(@".\TestData\organizations-1000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" }, ReaderType.System)] - [TestCase(@".\TestData\people-10000.csv", new[] { "Index", "User Id", "First Name", "Last Name", "Sex", "Email", "Phone", "Date of birth", "Job Title" }, ReaderType.System)] - public void Instantiat_CSVFile_BuildCorrectColumnizer (string filename, string[] expectedHeaders, ReaderType readerType) + [TestCase(@".\TestData\organizations-10000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" })] + [TestCase(@".\TestData\organizations-1000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" })] + [TestCase(@".\TestData\people-10000.csv", new[] { "Index", "User Id", "First Name", "Last Name", "Sex", "Email", "Phone", "Date of birth", "Job Title" })] + public void Instantiat_CSVFile_BuildCorrectColumnizer (string filename, string[] expectedHeaders) { CsvColumnizer.CsvColumnizer csvColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, filename); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); var line = reader.GetLogLineMemory(0); IColumnizedLogLineMemory logline = new ColumnizedLogLine(); @@ -309,7 +308,7 @@ public void LogfileReader_SemicolonCsv_ReadsAllLines () { // semicolon.csv: CRLF between lines, no trailing newline on last line var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\semicolon.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); Assert.That(reader.LineCount, Is.EqualTo(2), $"semicolon.csv should have 2 lines, got {reader.LineCount}"); @@ -320,7 +319,7 @@ public void LogfileReader_TabCsv_ReadsAllLines () { // tab.csv: header + 1 data line, CRLF between lines var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\tab.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); Assert.That(reader.LineCount, Is.EqualTo(2), $"tab.csv should have 2 lines, got {reader.LineCount}"); @@ -331,7 +330,7 @@ public void LogfileReader_CsvTest01_ReadsAllLines () { // CsvTest_01.csv: CR line endings, no trailing newline on last line var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\CsvTest_01.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); Assert.That(reader.LineCount, Is.EqualTo(31), $"CsvTest_01.csv should have 31 lines (header + 30 data), got {reader.LineCount}"); @@ -343,7 +342,7 @@ public void LogfileReader_SemicolonCsv_WithPreProcess_DataLinesVisible () // With CsvColumnizer as PreProcessColumnizer, header should be dropped, data line should remain CsvColumnizer.CsvColumnizer csvColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\semicolon.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -360,7 +359,7 @@ public void LogfileReader_TabCsv_WithPreProcess_DataLinesVisible () { CsvColumnizer.CsvColumnizer csvColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\tab.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -382,7 +381,7 @@ public void LogfileReader_SemicolonCsv_SimulateGuiReload_DataLinesVisible () CsvColumnizer.CsvColumnizer csvColumnizer = new(); // The GUI Reload creates a brand new LogfileReader with PreProcess already set - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -416,7 +415,7 @@ public void LogfileReader_SemicolonCsv_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -450,7 +449,7 @@ public void LogfileReader_CsvTest01_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -483,7 +482,7 @@ public void LogfileReader_TabCsv_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -505,7 +504,7 @@ public void LogfileReader_CommaCsv_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -550,7 +549,7 @@ public void LogfileReader_SemicolonCsv_SplitLine_ReturnsCorrectValues () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -595,7 +594,7 @@ public void LogfileReader_CommaCsv_ReloadWithPreProcess_DataLineNotEmpty () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); // NO PreProcessColumnizer set initially reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -609,7 +608,7 @@ public void LogfileReader_CommaCsv_ReloadWithPreProcess_DataLineNotEmpty () reader.StopMonitoring(); using ManualResetEventSlim reloadDone = new(false); - LogfileReader reader2 = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader2 = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader2.PreProcessColumnizer = csvColumnizer; reader2.LoadingFinished += (_, _) => reloadDone.Set(); reader2.StartMonitoring(); @@ -658,7 +657,7 @@ public void LogfileReader_CommaCsv_SingleFile_WithPreProcess_DataLineNotEmpty () using ManualResetEventSlim loadingDone = new(false); // multiFile=FALSE, like the real GUI - LogfileReader reader = new(path, new EncodingOptions(), false, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), false, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -695,7 +694,7 @@ public void LogfileReader_CsvNoTrailingNewline_HeaderDropped_DataLineVisible () CsvColumnizer.CsvColumnizer csvColumnizer = new(); - LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -727,7 +726,7 @@ public void LogfileReader_CsvWithTrailingNewline_HeaderDropped_DataLineVisible ( CsvColumnizer.CsvColumnizer csvColumnizer = new(); - LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); diff --git a/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs b/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs index 3d69741e2..975806e0c 100644 --- a/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs @@ -5,7 +5,6 @@ using LogExpert.Core.Classes.Columnizer; using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using Moq; @@ -105,14 +104,14 @@ public void FindColumnizer_ReturnCorrectColumnizer (string expectedColumnizerNam Assert.That(result.GetName(), Is.EqualTo(expectedColumnizerName)); } - [TestCase(@".\TestData\JsonColumnizerTest_01.txt", typeof(JsonCompactColumnizer.JsonCompactColumnizer), ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", typeof(SquareBracketColumnizer), ReaderType.System)] - public void FindReplacementForAutoColumnizer_ValidTextFile_ReturnCorrectColumnizer (string fileName, Type columnizerType, ReaderType readerType) + [TestCase(@".\TestData\JsonColumnizerTest_01.txt", typeof(JsonCompactColumnizer.JsonCompactColumnizer))] + [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", typeof(SquareBracketColumnizer))] + public void FindReplacementForAutoColumnizer_ValidTextFile_ReturnCorrectColumnizer (string fileName, Type columnizerType) { var pluginRegistry = PluginRegistry.PluginRegistry.Instance; var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, pluginRegistry, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), pluginRegistry, 500); reader.ReadFiles(); Mock autoColumnizer = new(); diff --git a/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs index fc344da57..d6b3dacc7 100644 --- a/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -44,12 +43,12 @@ private static void ResetPluginRegistrySingleton () instanceField?.SetValue(null, null); } - [TestCase(@".\TestData\JsonColumnizerTest_01.txt", "time @m level", ReaderType.System)] - public void GetColumnNames_HappyFile_ColumnNameMatches (string fileName, string expectedHeaders, ReaderType readerType) + [TestCase(@".\TestData\JsonColumnizerTest_01.txt", "time @m level")] + public void GetColumnNames_HappyFile_ColumnNameMatches (string fileName, string expectedHeaders) { var jsonColumnizer = new JsonColumnizer.JsonColumnizer(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); var line = reader.GetLogLineMemory(0); diff --git a/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs index e16118ab5..21f912c5d 100644 --- a/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs @@ -4,7 +4,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -47,14 +46,14 @@ private static void ResetPluginRegistrySingleton () } // As long as the json file contains one of the pre-defined key, it's perfectly supported. - [TestCase(@".\TestData\JsonCompactColumnizerTest_01.json", Priority.PerfectlySupport, ReaderType.System)] - [TestCase(@".\TestData\JsonCompactColumnizerTest_02.json", Priority.PerfectlySupport, ReaderType.System)] - [TestCase(@".\TestData\JsonCompactColumnizerTest_03.json", Priority.WellSupport, ReaderType.System)] - public void GetPriority_HappyFile_PriorityMatches (string fileName, Priority priority, ReaderType readerType) + [TestCase(@".\TestData\JsonCompactColumnizerTest_01.json", Priority.PerfectlySupport)] + [TestCase(@".\TestData\JsonCompactColumnizerTest_02.json", Priority.PerfectlySupport)] + [TestCase(@".\TestData\JsonCompactColumnizerTest_03.json", Priority.WellSupport)] + public void GetPriority_HappyFile_PriorityMatches (string fileName, Priority priority) { var jsonCompactColumnizer = new JsonCompactColumnizer.JsonCompactColumnizer(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); logFileReader.ReadFiles(); List loglines = [ diff --git a/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs index d1112a93a..2285af0e3 100644 --- a/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs @@ -5,7 +5,6 @@ using LogExpert.Core.Classes.Columnizer; using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -47,16 +46,16 @@ private static void ResetPluginRegistrySingleton () instanceField?.SetValue(null, null); } - [TestCase(@".\TestData\SquareBracketColumnizerTest_01.txt", 5, ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", 5, ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_03.txt", 6, ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_05.txt", 3, ReaderType.System)] - public void GetPriority_HappyFile_ColumnCountMatches (string fileName, int count, ReaderType readerType) + [TestCase(@".\TestData\SquareBracketColumnizerTest_01.txt", 5)] + [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", 5)] + [TestCase(@".\TestData\SquareBracketColumnizerTest_03.txt", 6)] + [TestCase(@".\TestData\SquareBracketColumnizerTest_05.txt", 3)] + public void GetPriority_HappyFile_ColumnCountMatches (string fileName, int count) { SquareBracketColumnizer squareBracketColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); logFileReader.ReadFiles(); List loglines = [ @@ -85,7 +84,7 @@ public void SplitLine_NestedBrackets_KeepsInnerBracketsInOneColumn () SquareBracketColumnizer squareBracketColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\SquareBracketColumnizerTest_06.txt"); - LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); logFileReader.ReadFiles(); List loglines = diff --git a/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs b/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs index a6ffa31e3..589afab37 100644 --- a/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs +++ b/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs @@ -437,6 +437,36 @@ public void LoadOrCreateNew_ValidFile_LoadsSuccessfully () Assert.That(loadResult.Settings.FilterList[0].SearchText, Is.EqualTo("VALID_FILTER_TEST")); } + // ADR 0010 removed the reader choice; settings saved by earlier versions still carry "ReaderType". + [Test] + [Category("DeserializationRecovery")] + [TestCase("SystemDirect")] + [TestCase("System")] + [TestCase("Legacy")] + public void LoadOrCreateNew_RemovedReaderTypeSetting_LoadsTheRestOfTheSettings (string storedReaderType) + { + // Arrange + Settings settings = CreatePopulatedSettings(); + settings.FilterList.Clear(); + settings.FilterList.Add(new FilterParams { SearchText = "KEPT_FILTER" }); + settings.Preferences.MaxLineLength = 12_345; + var json = Newtonsoft.Json.Linq.JObject.Parse(JsonConvert.SerializeObject(settings)); + json["Preferences"]!["ReaderType"] = storedReaderType; + File.WriteAllText(_testSettingsFile.FullName, json.ToString()); + + // Act + LoadResult loadResult = InvokePrivateInstanceMethod("LoadOrCreateNew", _testSettingsFile); + + // Assert + Assert.Multiple(() => + { + Assert.That(loadResult.CriticalFailure, Is.False); + Assert.That(loadResult.LoadedFromBackup, Is.False); + Assert.That(loadResult.Settings.FilterList.Select(filter => filter.SearchText), Is.EqualTo(new[] { "KEPT_FILTER" })); + Assert.That(loadResult.Settings.Preferences.MaxLineLength, Is.EqualTo(12_345)); + }); + } + [Test] [Category("DeserializationRecovery")] [Description("LoadOrCreateNew should handle missing file by creating new settings")] diff --git a/src/LogExpert.Tests/ReaderTest.cs b/src/LogExpert.Tests/ReaderTest.cs index f03feda55..b6dc04d53 100644 --- a/src/LogExpert.Tests/ReaderTest.cs +++ b/src/LogExpert.Tests/ReaderTest.cs @@ -1,8 +1,5 @@ -using System.Text; - using LogExpert.Core.Classes.Log.Streamreaders; using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; using NUnit.Framework; @@ -21,7 +18,7 @@ public void Boot () { } - #region GuessNewLineSequenceLength / TryReadLine tests + #region TryReadLine line endings [Test] public void TryReadLine_CrLf_NoTrailingNewline_ReadsAllLines () @@ -29,7 +26,7 @@ public void TryReadLine_CrLf_NoTrailingNewline_ReadsAllLines () // File with CRLF between lines but NO trailing newline on last line var content = "line1\r\nline2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -44,7 +41,7 @@ public void TryReadLine_CrLf_WithTrailingNewline_ReadsAllLines () // File with CRLF between lines AND trailing newline var content = "line1\r\nline2\r\n"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -59,7 +56,7 @@ public void TryReadLine_CrOnly_NoTrailingNewline_ReadsAllLines () // File with CR-only line endings (like old Mac), no trailing newline var content = "line1\rline2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -74,7 +71,7 @@ public void TryReadLine_LfOnly_NoTrailingNewline_ReadsAllLines () // File with LF-only line endings (Unix), no trailing newline var content = "line1\nline2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -89,7 +86,7 @@ public void TryReadLine_CrLf_ThreeLines_NoTrailingNewline_ReadsAllLines () // 3 lines with CRLF, no trailing newline var content = "header\r\ndata1\r\ndata2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -105,7 +102,7 @@ public void TryReadLine_CrLf_Position_TracksCorrectly () // Verify position tracking for CRLF file var content = "AB\r\nCD\r\n"u8.ToArray(); // 4 + 4 = 8 bytes using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.ToString(), Is.EqualTo("AB")); @@ -124,7 +121,7 @@ public void TryReadLine_CrLf_NoTrailingNewline_Position_TracksCorrectly () // Verify position tracking for CRLF file without trailing newline var content = "AB\r\nCD"u8.ToArray(); // 4 + 2 = 6 bytes using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.ToString(), Is.EqualTo("AB")); @@ -132,10 +129,9 @@ public void TryReadLine_CrLf_NoTrailingNewline_Position_TracksCorrectly () Assert.That(reader.TryReadLine(out var line2), Is.True); Assert.That(line2.ToString(), Is.EqualTo("CD")); - // Last line has no newline, but current implementation adds _newLineSequenceLength anyway. - // BUG: position is 8 (adds 2 for nonexistent CRLF), should be 6. - // This causes incorrect seeking on buffer re-read. - Assert.That(reader.Position, Is.EqualTo(8), "Known issue: overcounts by newline length on last line without trailing newline"); + // The last line has no newline, so only its content bytes count (the removed System reader added + // a guessed newline length here, which broke seeking on buffer re-read). + Assert.That(reader.Position, Is.EqualTo(6), "After 'CD' at EOF, position should be 6"); } [Test] @@ -144,7 +140,7 @@ public void TryReadLine_SingleLine_NoNewline_ReadsLine () // Single line file with no newline at all var content = "onlyline"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -152,7 +148,7 @@ public void TryReadLine_SingleLine_NoNewline_ReadsLine () Assert.That(lines[0], Is.EqualTo("onlyline")); } - private static List ReadAllLines (PositionAwareStreamReaderSystem reader) + private static List ReadAllLines (PositionAwareStreamReaderDirect reader) { List lines = []; while (reader.TryReadLine(out var lineMemory)) @@ -164,50 +160,4 @@ private static List ReadAllLines (PositionAwareStreamReaderSystem reader } #endregion - - //TODO reimplement - private void CompareReaderImplementationsInternal (string fileName, Encoding enc, int maxPosition) - { - var path = Environment.CurrentDirectory + "\\data\\"; - EncodingOptions encOpts = new() - { - Encoding = enc - }; - - using Stream s1 = new FileStream(path + fileName, FileMode.Open, FileAccess.Read); - using Stream s2 = new FileStream(path + fileName, FileMode.Open, FileAccess.Read); - using ILogStreamReader r1 = new PositionAwareStreamReaderLegacy(s1, encOpts, 500); - using ILogStreamReader r2 = new PositionAwareStreamReaderSystem(s2, encOpts, 500); - for (var lineNum = 0; ; lineNum++) - { - var line1 = r1.ReadLine(); - var line2 = r2.ReadLine(); - if (line1 == null && line2 == null) - { - break; - } - - Assert.That(line1, Is.EqualTo(line2), "File " + fileName); - - if (r1.Position != maxPosition) - { - Assert.That(r2.Position, Is.EqualTo(r1.Position), "Line " + lineNum + ", File: " + fileName); - } - else - { - //Its desired that the position of the new implementation is 2 bytes ahead to fix the problem of empty lines every time a new line is appended. - Assert.That(r2.Position - 2, Is.EqualTo(r1.Position), "Line " + lineNum + ", File: " + fileName); - } - } - } - - //TODO find out why it does not work with appveyor, but works fine with normal environment! - //[Test] - //[TestCase("50 MB.txt", "Windows-1252", 50000000)] - //[TestCase("50 MB UTF16.txt", "Unicode", 49999998)] - //[TestCase("50 MB UTF8.txt", "UTF-8", 50000000)] - //public void CompareReaderImplementations(string fileName, string encoding, int maxPosition) - //{ - // CompareReaderImplementationsInternal(fileName, Encoding.GetEncoding(encoding), maxPosition); - //} } \ No newline at end of file diff --git a/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs new file mode 100644 index 000000000..a86633e21 --- /dev/null +++ b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs @@ -0,0 +1,236 @@ +using System.Text; + +using LogExpert.Core.Classes.Log.Streamreaders; +using LogExpert.Core.Entities; +using LogExpert.Core.Helpers; + +using NUnit.Framework; + +namespace LogExpert.Tests.StreamReaderTests; + +// Issue #639: Position must equal the raw bytes consumed, whatever they are. Re-encoding decoded text +// counted each U+FFFD as 3 UTF-8 bytes, so a file with a binary block looked larger than it was. +[TestFixture] +public class DirectReaderExactPositionTests +{ + #region Invalid byte sequences + + private static IEnumerable InvalidUtf8Cases () + { + yield return Case("StrayContinuationBytes", "abc", [0x80, 0x81], "def\nnext\n"); + yield return Case("TruncatedSequenceBeforeLf", "x", [0xE6, 0x97], "\ny\n"); + yield return Case("TruncatedSequenceBeforeCrLf", "x", [0xE6], "\r\ny"); + yield return Case("NulAndHighBytes", "a", [0x00, 0x00, 0xFF, 0xFE], "\nb\n"); + yield return Case("OverlongAndEncodedSurrogate", "", [0xC0, 0xAF, 0xED, 0xA0, 0x80], "\nok\n"); + yield return Case("TruncatedSequenceAtEof", "tail", [0xE6, 0x97], ""); + yield return Case("GenuineReplacementCharacter", "valid ", [0xEF, 0xBF, 0xBD], " stays 3 bytes\n"); + yield return Case("BareCrAfterInvalidByte", "", [0xFF], "\rz"); + yield return Case("InvalidLineOnly", "", [0xF5, 0xF6, 0xF7], "\n"); + yield return Case("FourByteSequenceCutShort", "emoji ", [0xF0, 0x9F, 0x99], "\nafter\n"); + + static TestCaseData Case (string name, string before, byte[] invalid, string after) + { + var bytes = Encoding.UTF8.GetBytes(before).Concat(invalid).Concat(Encoding.UTF8.GetBytes(after)).ToArray(); + return new TestCaseData(bytes).SetName($"Position_InvalidUtf8_{name}_MatchesRawBytes"); + } + } + + [TestCaseSource(nameof(InvalidUtf8Cases))] + public void Position_InvalidUtf8_MatchesRawBytes (byte[] data) + { + RawByteLineOracle.AssertMatches(data, Encoding.UTF8); + } + + [Test] + public void Position_InvalidUtf8_ContentShowsReplacementCharacters () + { + byte[] data = [.. "abc"u8, 0x80, .. "def\n"u8]; + + var lines = RawByteLineOracle.ReadAll(data, Encoding.UTF8); + + Assert.That(lines.Select(line => line.Content), Is.EqualTo(new[] { "abc�def" })); + } + + [Test] + public void Position_InvalidUtf8LineLongerThanMaximum_TruncatesContentButCountsAllBytes () + { + byte[] data = [.. Encoding.UTF8.GetBytes(new string('A', 50)), 0xFF, 0xFE, .. Encoding.UTF8.GetBytes(new string('B', 50)), .. "\nnext\n"u8]; + + using var stream = new MemoryStream(data); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 20); + + Assert.That(reader.TryReadLine(out var first), Is.True); + Assert.Multiple(() => + { + Assert.That(first.ToString(), Is.EqualTo(new string('A', 20))); + Assert.That(reader.Position, Is.EqualTo(103)); + }); + + Assert.That(reader.TryReadLine(out var second), Is.True); + Assert.Multiple(() => + { + Assert.That(second.ToString(), Is.EqualTo("next")); + Assert.That(reader.Position, Is.EqualTo(data.Length)); + }); + } + + [Test] + public void Position_Utf16WithLoneSurrogateAndOddTrailingByte_MatchesRawBytes () + { + byte[] data = [.. Encoding.Unicode.GetBytes("ab"), 0x00, 0xD8, .. Encoding.Unicode.GetBytes("\r\ncd\n"), 0x41]; + + RawByteLineOracle.AssertMatches(data, Encoding.Unicode); + } + + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] + public void Position_Utf16WithSurrogatePairs_MatchesRawBytes () + { + var data = Encoding.Unicode.GetBytes("plain\r\nemoji 🙂 and 𝄞\npair at end 🙂"); + + RawByteLineOracle.AssertMatches(data, Encoding.Unicode); + } + + [Test] + public void Position_Utf16BigEndian_MatchesRawBytes () + { + var data = Encoding.BigEndianUnicode.GetBytes("first\r\nsecond\nthird\rfourth"); + + RawByteLineOracle.AssertMatches(data, Encoding.BigEndianUnicode); + } + + [Test] + public void Position_Gb2312WithInvalidLeadBytes_MatchesRawBytes () + { + var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); + byte[] data = [.. gb2312.GetBytes("错误 "), 0x81, .. "\n"u8, 0xFF, .. gb2312.GetBytes("警告\n")]; + + RawByteLineOracle.AssertMatches(data, gb2312); + } + + #endregion + + #region Buffer boundaries + + // Boundaries cover the old 32 K char block and common byte buffer sizes. + [Test] + public void Position_SequenceStraddlesBufferBoundary_MatchesRawBytes ( + [Values(16_383, 16_384, 32_767, 32_768, 65_535, 65_536, 131_071, 131_072)] int boundary, + [Values("ValidThreeByte", "InvalidTruncated", "CrLf", "Emoji")] string kind, + [Values(true, false)] bool shortLines) + { + byte[] sequence = kind switch + { + "ValidThreeByte" => [0xE4, 0xB8, 0x96], + "InvalidTruncated" => [0xE4, 0xB8], + "CrLf" => [0x0D, 0x0A], + _ => [0xF0, 0x9F, 0x99, 0x82] + }; + + for (var startOffset = 1; startOffset < sequence.Length + 1; startOffset++) + { + var prefix = Filler(boundary - startOffset, shortLines); + byte[] data = [.. prefix, .. sequence, .. "after\nlast"u8]; + + RawByteLineOracle.AssertMatches(data, Encoding.UTF8); + } + + static byte[] Filler (int length, bool shortLines) + { + var filler = new byte[length]; + for (var i = 0; i < length; i++) + { + filler[i] = shortLines && i % 100 == 99 ? (byte)'\n' : (byte)'A'; + } + + return filler; + } + } + + #endregion + + #region Seeking + + [Test] + public void Position_RecordedLineStarts_SeekBackToTheSameLines () + { + var data = TextWithBinaryBlock(lineCount: 3_000, binaryLength: 33_000, seed: 639); + + var expected = new List<(long Start, string Content)>(); + using (var stream = new MemoryStream(data)) + using (var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, RawByteLineOracle.MAX_LINE_LENGTH)) + { + var start = reader.Position; + while (reader.TryReadLine(out var line)) + { + expected.Add((start, line.ToString())); + start = reader.Position; + } + } + + using var seekStream = new MemoryStream(data); + using var seekReader = new PositionAwareStreamReaderDirect(seekStream, new EncodingOptions { Encoding = Encoding.UTF8 }, RawByteLineOracle.MAX_LINE_LENGTH); + + // Walk backwards so every seek moves the stream and discards what the reader had buffered. + for (var index = expected.Count - 1; index >= 0; index -= 37) + { + seekReader.Position = expected[index].Start; + + Assert.That(seekReader.TryReadLine(out var line), Is.True, $"Line {index}: nothing read after seek"); + Assert.That(line.ToString(), Is.EqualTo(expected[index].Content), $"Line {index}: content after seek"); + } + } + + #endregion + + #region Generated input + + [Test] + public void Position_TextWithBinaryBlock_MatchesRawBytes () + { + RawByteLineOracle.AssertMatches(TextWithBinaryBlock(lineCount: 3_000, binaryLength: 33_000, seed: 42), Encoding.UTF8); + } + + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] + public void Position_RandomBytes_MatchesRawBytes ([Range(0, 49)] int seed) + { + var random = new Random(seed); + var data = new byte[random.Next(1, 200_000)]; + for (var i = 0; i < data.Length; i++) + { + data[i] = random.Next(10) switch + { + 0 => (byte)'\n', + 1 => (byte)'\r', + 2 or 3 => (byte)random.Next(0x80, 0x100), + _ => (byte)random.Next(0x20, 0x7F) + }; + } + + RawByteLineOracle.AssertMatches(data, Encoding.UTF8); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] + private static byte[] TextWithBinaryBlock (int lineCount, int binaryLength, int seed) + { + var random = new Random(seed); + using var stream = new MemoryStream(); + + for (var i = 0; i < lineCount; i++) + { + if (i == lineCount / 2) + { + var binary = new byte[binaryLength]; + random.NextBytes(binary); + stream.Write(binary); + } + + stream.Write(Encoding.UTF8.GetBytes($"06/23/2026 00:00:00,MXS SEP26,3Minute,Info,OnVarUpdate(),\"Größe {i} 日志\"\r\n")); + } + + return stream.ToArray(); + } + + #endregion +} \ No newline at end of file diff --git a/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs b/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs index 37ee01e3d..84b343b4c 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs @@ -3,7 +3,6 @@ using LogExpert.Core.Classes.Log.Streamreaders; using LogExpert.Core.Entities; using LogExpert.Core.Helpers; -using LogExpert.Core.Interfaces; using NUnit.Framework; @@ -19,10 +18,10 @@ public class LogStreamReaderTest [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n", 3)] [TestCase("Line 1\rLine 2\rLine 3", 3)] [TestCase("Line 1\rLine 2\rLine 3\r", 3)] - public void ReadLinesWithSystemNewLine (string text, int expectedLines) + public void ReadLine_AllLineEndings_ReadsAllLines (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (true) { @@ -44,10 +43,10 @@ public void ReadLinesWithSystemNewLine (string text, int expectedLines) [TestCase("\n\n\n", 3)] [TestCase("\r\n\r\n\r\n", 3)] [TestCase("\r\r\r", 3)] - public void CountLinesWithSystemNewLine (string text, int expectedLines) + public void ReadLine_OnlyTerminators_CountsEmptyLines (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (reader.ReadLine() != null) { @@ -64,54 +63,10 @@ public void CountLinesWithSystemNewLine (string text, int expectedLines) [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n", 3)] [TestCase("Line 1\rLine 2\rLine 3", 3)] [TestCase("Line 1\rLine 2\rLine 3\r", 3)] - public void ReadLinesWithLegacyNewLine (string text, int expectedLines) + public void TryReadLine_ReturnsCorrectContent (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 500); - var lineCount = 0; - while (true) - { - var line = reader.ReadLine(); - if (line == null) - { - break; - } - - lineCount += 1; - - Assert.That(line.StartsWith($"Line {lineCount}", StringComparison.OrdinalIgnoreCase), $"Invalid line: {line}"); - } - - Assert.That(expectedLines, Is.EqualTo(lineCount), $"Unexpected lines:\n{text}"); - } - [Test] - [TestCase("\n\n\n", 3)] - [TestCase("\r\n\r\n\r\n", 3)] - [TestCase("\r\r\r", 3)] - public void CountLinesWithLegacyNewLine (string text, int expectedLines) - { - using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 500); - var lineCount = 0; - while (reader.ReadLine() != null) - { - lineCount += 1; - } - - Assert.That(expectedLines, Is.EqualTo(lineCount), $"Unexpected lines:\n{text}"); - } - - [Test] - [TestCase("Line 1\nLine 2\nLine 3", 3)] - [TestCase("Line 1\nLine 2\nLine 3\n", 3)] - [TestCase("Line 1\r\nLine 2\r\nLine 3", 3)] - [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n", 3)] - [TestCase("Line 1\rLine 2\rLine 3", 3)] - [TestCase("Line 1\rLine 2\rLine 3\r", 3)] - public void TryReadLine_ReturnsCorrectContent_SystemReader (string text, int expectedLines) - { - using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (reader.TryReadLine(out var lineMemory)) { @@ -127,10 +82,10 @@ public void TryReadLine_ReturnsCorrectContent_SystemReader (string text, int exp [TestCase("\n\n\n", 3)] [TestCase("\r\n\r\n\r\n", 3)] [TestCase("\r\r\r", 3)] - public void TryReadLine_CountsEmptyLines_SystemReader (string text, int expectedLines) + public void TryReadLine_CountsEmptyLines (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (reader.TryReadLine(out _)) { @@ -141,11 +96,11 @@ public void TryReadLine_CountsEmptyLines_SystemReader (string text, int expected } [Test] - public void TryReadLine_TracksPositionCorrectly_SystemReader () + public void TryReadLine_TracksPositionCorrectly () { var text = "Line 1\nLine 2\nLine 3\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.Span.ToString(), Is.EqualTo("Line 1")); @@ -167,7 +122,7 @@ public void TryReadLine_ReturnsBlockBackedMemory_NotStringBacked () { var text = "Hello World\nSecond Line\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var lineMemory), Is.True); @@ -183,7 +138,7 @@ public void TryReadLine_DetachCharBlocks_ReturnsFilledBlocks_AfterReading () { var text = "Line 1\nLine 2\nLine 3\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); while (reader.TryReadLine(out _)) { @@ -199,7 +154,7 @@ public void TryReadLine_DetachCharBlocks_TransfersOwnership () { var text = "Line 1\nLine 2\nLine 3\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); while (reader.TryReadLine(out _)) { @@ -222,8 +177,8 @@ public void TryReadLine_MatchesReadLine_ContentAndPosition () using var stream1 = new MemoryStream(Encoding.UTF8.GetBytes(text)); using var stream2 = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var readLineReader = new PositionAwareStreamReaderSystem(stream1, new EncodingOptions(), 500); - using var tryReadLineReader = new PositionAwareStreamReaderSystem(stream2, new EncodingOptions(), 500); + using var readLineReader = new PositionAwareStreamReaderDirect(stream1, new EncodingOptions(), 500); + using var tryReadLineReader = new PositionAwareStreamReaderDirect(stream2, new EncodingOptions(), 500); while (true) { @@ -248,7 +203,7 @@ public void TryReadLine_RespectsMaximumLineLength () var longLine = new string('X', 1000); var text = $"{longLine}\nShort\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var lineMemory), Is.True); Assert.That(lineMemory.Length, Is.EqualTo(500), "Line should be truncated to MaximumLineLength"); @@ -263,7 +218,7 @@ public void TryReadLine_UTF8_MultiByteCharacters () // Japanese characters: 3 bytes each in UTF-8 var text = "日本語テスト\nLine 2\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.Span.ToString(), Is.EqualTo("日本語テスト")); @@ -273,45 +228,22 @@ public void TryReadLine_UTF8_MultiByteCharacters () Assert.That(line2.Span.ToString(), Is.EqualTo("Line 2")); } - [Test] - [TestCase("Line 1\nLine 2\nLine 3", 3)] - [TestCase("Line 1\r\nLine 2\r\nLine 3", 3)] - [TestCase("Line 1\rLine 2\rLine 3", 3)] - public void TryReadLine_LegacyReader_ReadsAllLines (string text, int expectedLines) - { - using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 500); - - var memoryReader = reader as ILogStreamReaderMemory; - Assert.That(memoryReader, Is.Not.Null, "Legacy reader must implement ILogStreamReaderMemory"); - - var lineCount = 0; - while (memoryReader!.TryReadLine(out var lineMemory)) - { - lineCount++; - Assert.That(lineMemory.Span.ToString(), Does.StartWith($"Line {lineCount}")); - } - - Assert.That(lineCount, Is.EqualTo(expectedLines)); - } - /// - /// The legacy reader advances its position per character read, by a per-encoding step - /// (GetPosIncPrecomputed). GB2312 (issue #688) is the first offered encoding that is neither - /// single-byte nor Unicode — one byte per ASCII character, two per Chinese one — so a step of "1 - /// byte unless UTF-8 or UTF-16" drifted the position on the first Chinese character and every - /// subsequent line started at the wrong offset. + /// GB2312 (issue #688) is the first offered encoding that is neither single-byte nor Unicode — one + /// byte per ASCII character, two per Chinese one — so a position that assumed a fixed width per + /// character drifted on the first Chinese character and every subsequent line started at the wrong + /// offset. /// [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] - public void TryReadLine_LegacyReader_Gb2312_TracksTheBytePositionPerLine () + public void TryReadLine_Gb2312_TracksTheBytePositionPerLine () { var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); string[] lines = ["错误: 连接失败", "INFO 启动完成", "plain ascii", "警告"]; var text = string.Join("\n", lines) + "\n"; using var stream = new MemoryStream(gb2312.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions { Encoding = gb2312 }, 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = gb2312 }, 500); var expectedPosition = 0; diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs index 6bd1ae97a..1040c97d5 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -45,7 +44,6 @@ public void ReadFiles_AllLinesCorrect_WithBlockAllocation (int lineCount) bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -76,7 +74,6 @@ public void ReadFiles_AllLinesCorrect_WithDirectRead (int lineCount) bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.SystemDirect, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -103,7 +100,6 @@ public void ReadFiles_EmptyLines_PreservedCorrectly () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -133,7 +129,6 @@ public void ReadFiles_UTF8MultiByte_ContentPreserved () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -162,7 +157,6 @@ public void ReadFiles_LongLine_TruncatedToMaxLength () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -185,7 +179,6 @@ public void ReadFiles_LineContentBackedByPooledMemory () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -213,7 +206,6 @@ public void ReadFiles_MultipleBufferRotations_AllLinesCorrect () bufferCount: 100, linesPerBuffer: 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs index 9d791bf4d..87dc2087e 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs @@ -3,7 +3,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Classes.Log.ProgressReporters; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.Helpers; using NUnit.Framework; @@ -156,7 +155,6 @@ private LogfileReader CreateReader (EncodingOptions encodingOptions) bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.SystemDirect, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: NullProgressReporter.Instance); diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs index 00e36f731..39d62052d 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs @@ -3,7 +3,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Classes.Log.ProgressReporters; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -95,7 +94,6 @@ public void MultiFileCtor_AlwaysMultiFile () bufferCount: 40, linesPerBuffer: 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: NullProgressReporter.Instance); @@ -118,7 +116,6 @@ private LogfileReader CreateSingleFileReader (bool multiFile, MultiFileOptions? bufferCount: 40, linesPerBuffer: 50, options ?? new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: NullProgressReporter.Instance); diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs index 3953b04a2..ca0a58de9 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.Interfaces; using Moq; @@ -57,7 +56,6 @@ public void SingleFileReader_OnTruncation_ReportsNewFileNotRollover () bufferCount: 40, linesPerBuffer: 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: progressReporterMock.Object); @@ -73,4 +71,65 @@ public void SingleFileReader_OnTruncation_ReportsNewFileNotRollover () Assert.That(newFileReported.Wait(TimeSpan.FromSeconds(5)), Is.True, "Truncation did not report a new file"); Assert.That(rolloverReported, Is.False); } + + // Issue #639: an overcounted position made every append look like a new file, so the window reloaded. + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] + public void SingleFileReader_GrowingFileWithBinaryBlock_TakesGrowthPathNotNewFile () + { + // No BOM: FileSize counts from after the preamble, so without one it equals the length on disk. + var binary = new byte[33_000]; + new Random(639).NextBytes(binary); + using (var stream = new FileStream(_logFile, FileMode.Create, FileAccess.Write)) + { + stream.Write(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Range(1, 100).Select(index => $"Line {index}\n")))); + stream.Write(binary); + stream.Write(Encoding.UTF8.GetBytes("Line after binary\n")); + } + + using var loadingFinished = new ManualResetEventSlim(false); + using var grown = new ManualResetEventSlim(false); + var newFileReported = false; + var progressReporterMock = new Mock(); + _ = progressReporterMock + .Setup(reporter => reporter.ReportLoadingFinished()) + .Callback(() => loadingFinished.Set()); + _ = progressReporterMock + .Setup(reporter => reporter.ReportNewFile(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback(() => newFileReported = true); + + using var reader = new LogfileReader( + _logFile, + new EncodingOptions { Encoding = Encoding.UTF8 }, + multiFile: false, + bufferCount: 40, + linesPerBuffer: 50, + new MultiFileOptions(), + PluginRegistry.PluginRegistry.Instance, + maximumLineLength: 500, + progressReporter: progressReporterMock.Object); + + reader.StartMonitoring(); + Assert.That(loadingFinished.Wait(TimeSpan.FromSeconds(5)), Is.True, "Initial load did not finish"); + Assert.That(reader.FileSize, Is.EqualTo(new FileInfo(_logFile).Length), "Size read after the initial load"); + + var linesBefore = reader.LineCount; + reader.FileSizeChanged += (_, args) => + { + if (args.LineCount > linesBefore) + { + grown.Set(); + } + }; + + File.AppendAllText(_logFile, "Appended 1\nAppended 2\n", Encoding.UTF8); + + Assert.That(grown.Wait(TimeSpan.FromSeconds(5)), Is.True, "Growth was not reported"); + Assert.Multiple(() => + { + Assert.That(newFileReported, Is.False, "Growth was mistaken for a new file"); + Assert.That(reader.LineCount, Is.EqualTo(linesBefore + 2)); + Assert.That(reader.FileSize, Is.EqualTo(new FileInfo(_logFile).Length), "Size read after the growth"); + }); + } } diff --git a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs index 198433340..c0e6d6b7e 100644 --- a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs @@ -159,117 +159,140 @@ public void TryReadLine_LineCrossesBlockBoundary_Correct () #endregion - #region Position parity with System reader + #region Position matches the raw bytes [Test] [TestCase("Line 1\nLine 2\nLine 3\n")] [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n")] [TestCase("Short\nMedium line here\nA much longer line with more content\n")] - public void Position_MatchesSystemReader_UTF8 (string text) + public void Position_MatchesRawBytes_UTF8 (string text) { - ComparePositions(text, Encoding.UTF8); + RawByteLineOracle.AssertMatches(Encoding.UTF8.GetBytes(text), Encoding.UTF8); } [Test] [TestCase("Line 1\nLine 2\nLine 3\n")] - public void Position_MatchesSystemReader_ASCII (string text) + public void Position_MatchesRawBytes_ASCII (string text) { - ComparePositions(text, Encoding.ASCII); + RawByteLineOracle.AssertMatches(Encoding.ASCII.GetBytes(text), Encoding.ASCII); } [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] - public void Position_MatchesSystemReader_UTF8_MultibyteChars () + public void Position_MatchesRawBytes_UTF8_MultibyteChars () { var text = "Héllo wörld\nCafé résumé\nこんにちは\n"; - ComparePositions(text, Encoding.UTF8); + RawByteLineOracle.AssertMatches(Encoding.UTF8.GetBytes(text), Encoding.UTF8); } /// /// GB2312 (issue #688) is the first offered encoding that is neither single-byte nor Unicode: two - /// bytes per Chinese character, one per ASCII character, mixed within a line. The direct reader - /// advances its byte position by Encoding.GetByteCount over the decoded chars, so a wrong - /// assumption there would drift the position from the second line on — and every buffer boundary, + /// bytes per Chinese character, one per ASCII character, mixed within a line. A wrong assumption + /// about the width would drift the position from the second line on — and every buffer boundary, /// rollover check and .lxp scroll position is computed from it. /// [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] - public void Position_MatchesSystemReader_Gb2312_MixedAsciiAndChinese () + public void Position_MatchesRawBytes_Gb2312_MixedAsciiAndChinese () { + var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); var text = "2026-07-30 错误: 连接失败\nINFO 启动完成\nplain ascii line\n警告\n"; - ComparePositions(text, EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312)); + RawByteLineOracle.AssertMatches(gb2312.GetBytes(text), gb2312); } - private static void ComparePositions (string text, Encoding encoding) - { - var bytes = encoding.GetBytes(text); + #endregion - using var s1 = new MemoryStream(bytes); - using var s2 = new MemoryStream(bytes); - using var systemReader = new PositionAwareStreamReaderSystem(s1, new EncodingOptions { Encoding = encoding }, 500); - using var directReader = new PositionAwareStreamReaderDirect(s2, new EncodingOptions { Encoding = encoding }, 500); + #region Content of a large file - var lineNum = 0; - while (true) + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1305:Specify IFormatProvider", Justification = "Unit Tests")] + public void TryReadLine_LargeFile_ReturnsEveryLine () + { + var expected = new List(); + var sb = new StringBuilder(); + var rng = new Random(42); + for (var i = 0; i < 10_000; i++) { - var systemHasLine = systemReader.TryReadLine(out var systemLine); - var directHasLine = directReader.TryReadLine(out var directLine); - - Assert.That(directHasLine, Is.EqualTo(systemHasLine), $"Line {lineNum}: EOF mismatch"); - - if (!systemHasLine) - { - break; - } + var line = $"2026-04-23 12:00:00.{i % 1000} [INFO] Thread-{rng.Next(1, 32)} SomeClass - Log message number {i}"; + expected.Add(line); + _ = sb.AppendLine(line); + } - Assert.That(directLine.Span.ToString(), Is.EqualTo(systemLine.Span.ToString()), - $"Line {lineNum}: content mismatch"); - Assert.That(directReader.Position, Is.EqualTo(systemReader.Position), - $"Line {lineNum}: position mismatch (System={systemReader.Position}, Direct={directReader.Position})"); + using var stream = CreateStream(sb.ToString()); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); - lineNum++; + var actual = new List(); + while (reader.TryReadLine(out var line)) + { + actual.Add(line.Span.ToString()); } + + Assert.That(actual, Is.EqualTo(expected)); } #endregion - #region Content parity with System reader + #region Allocations + // Only detaching blocks may allocate (the list handed to the buffer). Measured on the reading thread, + // so the result is deterministic; any per-line allocation (at least 24 bytes) breaks 0.25 bytes per line. [Test] - [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] - [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1305:Specify IFormatProvider", Justification = "Unit Tests")] - public void ContentParity_LargeFile () + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] + public void TryReadLine_ValidText_DoesNotAllocatePerLine ([Values(65001, 1200)] int codePage, [Values(false, true)] bool nonAscii) { + const int lineCount = 200_000; + const int linesPerBuffer = 500; + var encoding = Encoding.GetEncoding(codePage); + var message = nonAscii ? " [INFO] Größe 日志 🙂 - Log message number " : " [INFO] Thread-7 SomeClass - Log message number "; + var sb = new StringBuilder(); - var rng = new Random(42); - for (var i = 0; i < 10_000; i++) + for (var i = 0; i < lineCount; i++) { - _ = sb.Append("2026-04-23 12:00:00."); - _ = sb.Append(i % 1000); - _ = sb.Append(" [INFO] Thread-"); - _ = sb.Append(rng.Next(1, 32)); - _ = sb.Append(" SomeClass - Log message number "); - _ = sb.AppendLine(i.ToString()); + _ = sb.Append("2026-04-23 12:00:00.").Append(i % 1000).Append(message).Append(i).Append("\r\n"); } - var text = sb.ToString(); - var bytes = Encoding.UTF8.GetBytes(text); + var data = encoding.GetBytes(sb.ToString()); - using var s1 = new MemoryStream(bytes); - using var s2 = new MemoryStream(bytes); - using var systemReader = new PositionAwareStreamReaderSystem(s1, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); - using var directReader = new PositionAwareStreamReaderDirect(s2, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); + // Warm up the JIT and the array pool, then measure one full read. + _ = ReadAllAndReturnBlocks(data, encoding, linesPerBuffer); + _ = ReadAllAndReturnBlocks(data, encoding, linesPerBuffer); - var lineNum = 0; - while (systemReader.TryReadLine(out var systemLine)) + var before = GC.GetAllocatedBytesForCurrentThread(); + var lines = ReadAllAndReturnBlocks(data, encoding, linesPerBuffer); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Multiple(() => { - Assert.That(directReader.TryReadLine(out var directLine), Is.True, $"Direct reader ended early at line {lineNum}"); - Assert.That(directLine.Span.ToString(), Is.EqualTo(systemLine.Span.ToString()), $"Line {lineNum}: content mismatch"); - lineNum++; + Assert.That(lines, Is.EqualTo(lineCount)); + Assert.That(allocated, Is.LessThan(lineCount / 4), $"Allocated {allocated} bytes for {lines} lines"); + }); + } + + private static int ReadAllAndReturnBlocks (byte[] data, Encoding encoding, int linesPerBuffer) + { + using var stream = new MemoryStream(data, writable: false); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, 20_000); + + var lines = 0; + while (reader.TryReadLine(out _)) + { + if (++lines % linesPerBuffer == 0) + { + ReturnBlocks(reader.DetachCharBlocks()); + } } - Assert.That(directReader.TryReadLine(out _), Is.False, "Direct reader has extra lines"); - Assert.That(lineNum, Is.EqualTo(10_000)); + ReturnBlocks(reader.DetachCharBlocks()); + return lines; + + static void ReturnBlocks (List blocks) + { + foreach (var block in blocks) + { + System.Buffers.ArrayPool.Shared.Return(block); + } + } } #endregion @@ -446,51 +469,14 @@ public void TryReadLine_LineMultipleTimesBlockSize_ReadsCorrectly () } [Test] - public void TryReadLine_LineLongerThanBlockSize_PositionParityWithSystemReader () + public void TryReadLine_LineLongerThanBlockSize_PositionMatchesRawBytes () { RunWithTimeout(() => { - // Verify byte positions match between Direct and System readers for long lines const int lineLength = 50_000; var text = $"short\n{new string('Q', lineLength)}\nend\n"; - var bytes = Encoding.UTF8.GetBytes(text); - - using var s1 = new MemoryStream(bytes); - using var s2 = new MemoryStream(bytes); - // Note: System reader has a pre-existing bug in CharBlockAllocator.ReturnAll() for - // oversized lines (allocates with new[] but tries to return to ArrayPool). Suppress - // disposal to avoid that unrelated failure. - var systemReader = new PositionAwareStreamReaderSystem(s1, new EncodingOptions { Encoding = Encoding.UTF8 }, lineLength + 100); - using var directReader = new PositionAwareStreamReaderDirect(s2, new EncodingOptions { Encoding = Encoding.UTF8 }, lineLength + 100); - - try - { - var lineNum = 0; - while (true) - { - var systemHasLine = systemReader.TryReadLine(out var systemLine); - var directHasLine = directReader.TryReadLine(out var directLine); - - Assert.That(directHasLine, Is.EqualTo(systemHasLine), $"Line {lineNum}: EOF mismatch"); - if (!systemHasLine) - { - break; - } - Assert.That(directLine.Span.ToString(), Is.EqualTo(systemLine.Span.ToString()), $"Line {lineNum}: content mismatch"); - Assert.That(directReader.Position, Is.EqualTo(systemReader.Position), $"Line {lineNum}: position mismatch (System={systemReader.Position}, Direct={directReader.Position})"); - - lineNum++; - } - - Assert.That(lineNum, Is.EqualTo(3)); - } - finally - { - // Don't call Dispose on systemReader — CharBlockAllocator has a pre-existing - // bug returning oversized (new[]) arrays to ArrayPool. - // The GC will reclaim the memory. - } + RawByteLineOracle.AssertMatches(Encoding.UTF8.GetBytes(text), Encoding.UTF8); }); } diff --git a/src/LogExpert.Tests/StreamReaderTests/RawByteLineOracle.cs b/src/LogExpert.Tests/StreamReaderTests/RawByteLineOracle.cs new file mode 100644 index 000000000..df063fcfd --- /dev/null +++ b/src/LogExpert.Tests/StreamReaderTests/RawByteLineOracle.cs @@ -0,0 +1,87 @@ +using System.Text; + +using LogExpert.Core.Classes.Log.Streamreaders; +using LogExpert.Core.Entities; + +using NUnit.Framework; + +namespace LogExpert.Tests.StreamReaderTests; + +// The raw bytes as oracle for the Direct reader (ADR 0010): a line ends at the next \n, \r\n or bare +// \r code unit of the encoding, its text is that line's bytes decoded on their own, and the position +// after it is the byte offset after its terminator. +internal static class RawByteLineOracle +{ + public const int MAX_LINE_LENGTH = 500_000; + + public static void AssertMatches (byte[] data, Encoding encoding) + { + var expected = ExpectedLines(data, encoding); + var actual = ReadAll(data, encoding); + + Assert.That(actual.Count, Is.EqualTo(expected.Count), "Line count"); + for (var i = 0; i < expected.Count; i++) + { + if (actual[i] != expected[i]) + { + Assert.Fail($"Line {i}: expected end {expected[i].End} \"{Shorten(expected[i].Content)}\", got end {actual[i].End} \"{Shorten(actual[i].Content)}\""); + } + } + } + + public static List<(string Content, long End)> ReadAll (byte[] data, Encoding encoding) + { + using var stream = new MemoryStream(data); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, MAX_LINE_LENGTH); + + var lines = new List<(string, long)>(); + while (reader.TryReadLine(out var line)) + { + lines.Add((line.ToString(), reader.Position)); + } + + return lines; + } + + // A trailing line without terminator is a line; an empty remainder after the last terminator is not. + private static List<(string Content, long End)> ExpectedLines (byte[] data, Encoding encoding) + { + var cr = encoding.GetBytes("\r"); + var lf = encoding.GetBytes("\n"); + var unit = lf.Length; + + var lines = new List<(string, long)>(); + var lineStart = 0; + var position = 0; + + while (position + unit <= data.Length) + { + var span = data.AsSpan(position, unit); + if (span.SequenceEqual(lf) || span.SequenceEqual(cr)) + { + var terminatorLength = unit; + if (span.SequenceEqual(cr) && position + (2 * unit) <= data.Length && data.AsSpan(position + unit, unit).SequenceEqual(lf)) + { + terminatorLength = 2 * unit; + } + + lines.Add((encoding.GetString(data, lineStart, position - lineStart), position + terminatorLength)); + position += terminatorLength; + lineStart = position; + } + else + { + position += unit; + } + } + + if (lineStart < data.Length) + { + lines.Add((encoding.GetString(data, lineStart, data.Length - lineStart), data.Length)); + } + + return lines; + } + + private static string Shorten (string text) => text.Length <= 60 ? text : $"{text[..60]}… ({text.Length} chars)"; +} \ No newline at end of file diff --git a/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs b/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs index a954e6f70..21602d547 100644 --- a/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs +++ b/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs @@ -6,7 +6,6 @@ using LogExpert.Core.Classes.Log.ProgressReporters; using LogExpert.Core.Config; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.Interfaces; using LogExpert.UI.Controls.LogWindow; using LogExpert.UI.Interface; @@ -527,7 +526,7 @@ public void LoadDroppedFiles_CombinedChoice_ReadsAccessibleSelectedFiles (int un // Exercise the same reader endpoint used by LogWindow.LoadFilesAsMulti. using var reader = new LogfileReader(e.MultiFileNames, new EncodingOptions { Encoding = Encoding.UTF8 }, 40, 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); + PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); reader.ReadFiles(); lineCount = reader.GetLogLineMemories(0, 10).Length; }; @@ -587,7 +586,7 @@ public void LoadDroppedFiles_CombinedSessionAndLog_DoesNotOpenUnselectedRotation { using var reader = new LogfileReader(e.MultiFileNames!, new EncodingOptions { Encoding = Encoding.UTF8 }, 40, 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); + PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); reader.ReadFiles(); observed = reader.GetLogLineMemories(0, 10).Select(line => line.FullLine.ToString()).ToArray(); }; @@ -620,7 +619,7 @@ public void LoadDroppedFiles_CombinedSelectionAfterTruncation_KeepsOnlySelectedP { using var reader = new LogfileReader(e.MultiFileNames!, new EncodingOptions { Encoding = Encoding.UTF8 }, 40, 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); + PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); reader.ReadFiles(); File.WriteAllText(second, "short\n"); _ = reader.ShiftBuffers(); diff --git a/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs b/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs index 56174ee81..2ee38b6d6 100644 --- a/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs +++ b/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs @@ -6071,7 +6071,7 @@ public void LoadFile (string fileName, EncodingOptions encodingOptions) try { - _logFileReader = new(fileName, EncodingOptions, IsMultiFile, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, Preferences.ReaderType, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength); + _logFileReader = new(fileName, EncodingOptions, IsMultiFile, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength); } catch (LogFileException lfe) { @@ -6146,7 +6146,7 @@ private void StartMultiFileReader (string[] fileNames, EncodingOptions encodingO _columnCache = new ColumnCache(); _filterColumnCache = new ColumnCache(); - _logFileReader = new(fileNames, EncodingOptions, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, Preferences.ReaderType, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength, useExplicitFileList: true); + _logFileReader = new(fileNames, EncodingOptions, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength, useExplicitFileList: true); RegisterLogFileReaderEvents(); _logFileReader.StartMonitoring(); diff --git a/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs b/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs index a404a21a9..6d871e779 100644 --- a/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs +++ b/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs @@ -163,7 +163,6 @@ private void InitializeComponent () checkBoxSaveSessions = new CheckBox(); tabPageMemory = new TabPage(); groupBoxCPUAndStuff = new GroupBox(); - comboBoxReaderType = new ComboBox(); checkBoxMultiThread = new CheckBox(); labelFilePollingInterval = new Label(); upDownPollingInterval = new NumericUpDown(); @@ -1917,7 +1916,6 @@ private void InitializeComponent () // // groupBoxCPUAndStuff // - groupBoxCPUAndStuff.Controls.Add(comboBoxReaderType); groupBoxCPUAndStuff.Controls.Add(checkBoxMultiThread); groupBoxCPUAndStuff.Controls.Add(labelFilePollingInterval); groupBoxCPUAndStuff.Controls.Add(upDownPollingInterval); @@ -1930,14 +1928,6 @@ private void InitializeComponent () groupBoxCPUAndStuff.TabStop = false; groupBoxCPUAndStuff.Text = "CPU and stuff"; // - // comboBoxReaderType - // - comboBoxReaderType.FormattingEnabled = true; - comboBoxReaderType.Location = new Point(14, 76); - comboBoxReaderType.Name = "comboBoxReaderType"; - comboBoxReaderType.Size = new Size(262, 23); - comboBoxReaderType.TabIndex = 10; - // // checkBoxMultiThread // checkBoxMultiThread.AutoSize = true; @@ -2366,7 +2356,6 @@ private void InitializeComponent () private System.Windows.Forms.Label labelMaxDisplayLength; private Label labelLanguage; private ComboBox comboBoxLanguage; - private ComboBox comboBoxReaderType; private DataGridViewCheckBoxColumn columnControlCharEnabled; private DataGridViewTextBoxColumn columnControlCharHex; private DataGridViewTextBoxColumn columnControlCharAbbr; diff --git a/src/LogExpert.UI/Dialogs/SettingsDialog.cs b/src/LogExpert.UI/Dialogs/SettingsDialog.cs index c04df562c..87b61f40a 100644 --- a/src/LogExpert.UI/Dialogs/SettingsDialog.cs +++ b/src/LogExpert.UI/Dialogs/SettingsDialog.cs @@ -286,7 +286,6 @@ private void FillDialog () FillMultifileSettings(); FillEncodingList(); FillLanguageList(); - FillReaderTypeList(); FillControlCharsTab(); FillMarkerBarTab(); @@ -314,19 +313,6 @@ private void FillDialog () checkBoxShowErrorMessageOnlyOneInstance.Checked = Preferences.ShowErrorMessageAllowOnlyOneInstances; } - private void FillReaderTypeList () - { - foreach (var readerType in Enum.GetValues()) - { - if (!comboBoxReaderType.Items.Contains(readerType)) - { - _ = comboBoxReaderType.Items.Add(readerType); - } - } - - comboBoxReaderType.SelectedItem = Preferences.ReaderType; - } - private void FillColorModeList () { foreach (var colorMode in Enum.GetValues().Where(cm => !comboBoxColorMode.Items.Contains(cm))) @@ -827,7 +813,6 @@ private void OnBtnOkClick (object sender, EventArgs e) Preferences.DefaultEncoding = comboBoxEncoding.SelectedItem is Encoding selectedEncoding ? selectedEncoding.HeaderName : FallbackEncoding.HeaderName; Preferences.DefaultLanguage = comboBoxLanguage.SelectedItem != null ? (comboBoxLanguage.SelectedItem as string) : CultureInfo.GetCultureInfo("en-US").Name; Preferences.ShowColumnFinder = checkBoxColumnFinder.Checked; - Preferences.ReaderType = comboBoxReaderType.SelectedItem != null ? (ReaderType)comboBoxReaderType.SelectedItem : ReaderType.SystemDirect; Preferences.MaximumFilterEntries = (int)upDownMaximumFilterEntries.Value; Preferences.MaximumFilterEntriesDisplayed = (int)upDownMaximumFilterEntriesDisplayed.Value; @@ -1400,7 +1385,6 @@ private Dictionary GetToolTipMap () { comboBoxEncoding, Resources.SettingsDialog_UI_ComboBox_ToolTip_toolTipEncoding }, { checkBoxPortableMode, Resources.SettingsDialog_UI_CheckBox_ToolTip_toolTipPortableMode }, { radioButtonSessionApplicationStartupDir, Resources.SettingsDialog_UI_RadioButton_ToolTip_toolTipSessionApplicationStartupDir }, - { comboBoxReaderType, Resources.SettingsDialog_UI_CheckBox_ToolTip_toolTipReaderTyp }, { upDownMultifileDays, Resources.SettingsDialog_UI_NumericUpDown_ToolTip_upDownMultifileDays }, { upDownMultifileMaxFileAgeDays, Resources.SettingsDialog_UI_NumericUpDown_ToolTip_upDownMultifileMaxFileAgeDays } };