From 8dadc3744117507b0a3c37f48576bd1030969f59 Mon Sep 17 00:00:00 2001 From: BRUNER Patrick Date: Fri, 20 Nov 2026 14:42:56 +0100 Subject: [PATCH 1/3] Count raw bytes for the Direct reader position (#639) A log with a binary block in the middle jumped to the top on every append. The Direct reader advanced its byte Position by re-encoding the decoded text; each invalid byte decodes to U+FFFD, which re-encodes as three UTF-8 bytes, so Position ran ahead of the file (by 100,055 bytes on the reporter's sample). LogfileReader takes FileSize from it, saw newSize < FileSize on the next poll, treated the file as recreated and reloaded it. The reader now finds line terminators in the raw bytes (as code units of the encoding: one byte, or two/four for UTF-16/UTF-32) and decodes each line into the pooled char block itself, so Position advances by the bytes a line really occupies. Pure-ASCII lines are widened with Ascii.ToUtf16 and native-order UTF-16 lines without surrogates are copied; other lines go through the encoding. Tests use the raw bytes as oracle: invalid UTF-8 cases, sequences straddling buffer boundaries, seeking back to recorded line starts, UTF-16 and GB2312 edge cases, seeded random input, and a LogfileReader tail test that grows a file with a binary block. A deterministic test guards zero allocations per line. DirectReaderBenchmarks measures the reader as LogfileReader drives it; against the previous reader it is faster on every workload (interleaved A/B). --- .../DirectReaderBenchmarks.cs | 164 +++++++ src/LogExpert.Benchmarks/Program.cs | 2 + .../PositionAwareStreamReaderBase.cs | 10 + .../PositionAwareStreamReaderDirect.cs | 436 ++++++++++++------ .../DirectReaderExactPositionTests.cs | 327 +++++++++++++ .../LogfileReaderSingleFileMonitoringTests.cs | 66 +++ .../PositionAwareStreamReaderDirectTests.cs | 65 +++ 7 files changed, 921 insertions(+), 149 deletions(-) create mode 100644 src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs create mode 100644 src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs diff --git a/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs b/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs new file mode 100644 index 00000000..f577cd68 --- /dev/null +++ b/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs @@ -0,0 +1,164 @@ +using System.Buffers; +using System.Text; + +using BenchmarkDotNet.Attributes; + +using LogExpert.Core.Classes.Log.Streamreaders; +using LogExpert.Core.Entities; + +namespace LogExpert.Benchmarks; + +/// +/// Measures the default (Direct) stream reader the way LogfileReader drives it: lines come +/// from TryReadLine, and every lines the char blocks are +/// detached and returned to the pool, as a LogBuffer does when it is evicted. +/// The valid-UTF-8 workloads are the regression gate for reader changes; the invalid-bytes +/// workload covers files with binary blocks (issue #639). +/// +[MemoryDiagnoser] +public class DirectReaderBenchmarks +{ + private const int LINES_PER_BUFFER = 500; + private const int MAX_LINE_LENGTH = 20_000; + + private byte[] _asciiMedium = null!; + private byte[] _asciiLarge = null!; + private byte[] _unicode = null!; + private byte[] _mixedLineEndings = null!; + private byte[] _longLines = null!; + private byte[] _utf16 = null!; + private byte[] _invalidBytes = null!; + + [GlobalSetup] + public void Setup () + { + _asciiMedium = Encoding.UTF8.GetBytes(GenerateAscii(10_000, 100, "\r\n")); + _asciiLarge = Encoding.UTF8.GetBytes(GenerateAscii(200_000, 200, "\r\n")); + _unicode = Encoding.UTF8.GetBytes(GenerateUnicode(100_000)); + _mixedLineEndings = Encoding.UTF8.GetBytes(GenerateMixedLineEndings(100_000)); + _longLines = Encoding.UTF8.GetBytes(GenerateAscii(500, 40_000, "\n")); + _utf16 = Encoding.Unicode.GetBytes(GenerateAscii(100_000, 100, "\r\n")); + _invalidBytes = GenerateInvalidBytes(100_000); + } + + [Benchmark(Baseline = true)] + public int AsciiMedium () => ReadAll(_asciiMedium, Encoding.UTF8); + + [Benchmark] + public int AsciiLarge () => ReadAll(_asciiLarge, Encoding.UTF8); + + [Benchmark] + public int Unicode () => ReadAll(_unicode, Encoding.UTF8); + + [Benchmark] + public int MixedLineEndings () => ReadAll(_mixedLineEndings, Encoding.UTF8); + + [Benchmark] + public int LongLines () => ReadAll(_longLines, Encoding.UTF8); + + [Benchmark] + public int Utf16 () => ReadAll(_utf16, Encoding.Unicode); + + [Benchmark] + public int InvalidBytes () => ReadAll(_invalidBytes, Encoding.UTF8); + + private static int ReadAll (byte[] data, Encoding encoding) + { + using var stream = new MemoryStream(data, writable: false); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, MAX_LINE_LENGTH); + + var lines = 0; + while (reader.TryReadLine(out _)) + { + if (++lines % LINES_PER_BUFFER == 0) + { + ReturnBlocks(reader.DetachCharBlocks()); + } + } + + ReturnBlocks(reader.DetachCharBlocks()); + return lines; + } + + private static void ReturnBlocks (List blocks) + { + foreach (var block in blocks) + { + ArrayPool.Shared.Return(block); + } + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")] + private static string GenerateAscii (int lineCount, int avgLineLength, string newLine) + { + var sb = new StringBuilder(); + var random = new Random(42); + + for (var i = 0; i < lineCount; i++) + { + var lineLength = avgLineLength + random.Next(-10, 11); + _ = sb.Append("2026-04-23 12:00:00.").Append(i % 1000).Append(" [INFO] ") + .Append('X', Math.Max(0, lineLength - 32)) + .Append(newLine); + } + + return sb.ToString(); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")] + private static string GenerateUnicode (int lineCount) + { + var sb = new StringBuilder(); + var random = new Random(42); + + for (var i = 0; i < lineCount; i++) + { + _ = random.Next(0, 3) switch + { + 0 => sb.Append("Line ").Append(i).Append(": ASCII text only, nothing special here\r\n"), + 1 => sb.Append("Line ").Append(i).Append(": Hello 世界, 日志文件 (Chinese)\r\n"), + _ => sb.Append("Line ").Append(i).Append(": Спасибо большое 🙂 (Russian, emoji)\r\n") + }; + } + + return sb.ToString(); + } + + private static string GenerateMixedLineEndings (int lineCount) + { + var sb = new StringBuilder(); + string[] endings = ["\r\n", "\n", "\r"]; + + for (var i = 0; i < lineCount; i++) + { + _ = sb.Append("2026-04-23 12:00:00 [WARN] mixed ending line ").Append(i).Append(endings[i % endings.Length]); + } + + return sb.ToString(); + } + + /// + /// CSV-like text with a binary block every 1000 lines: NUL runs, stray continuation bytes and + /// truncated multi-byte sequences, the shape of the sample file from issue #639. + /// + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")] + private static byte[] GenerateInvalidBytes (int lineCount) + { + var random = new Random(42); + using var stream = new MemoryStream(); + + for (var i = 0; i < lineCount; i++) + { + if (i % 1000 == 500) + { + var binary = new byte[2048]; + random.NextBytes(binary); + stream.Write(binary); + } + + stream.Write(Encoding.UTF8.GetBytes($"06/23/2026 00:00:00,MXS SEP26,3Minute,Info,OnVarUpdate(),\"line {i}\"\r\n")); + } + + return stream.ToArray(); + } +} \ No newline at end of file diff --git a/src/LogExpert.Benchmarks/Program.cs b/src/LogExpert.Benchmarks/Program.cs index b29c9171..62f85f82 100644 --- a/src/LogExpert.Benchmarks/Program.cs +++ b/src/LogExpert.Benchmarks/Program.cs @@ -13,6 +13,7 @@ public static void Main (string[] args) // Run all benchmarks if no arguments are provided _ = BenchmarkRunner.Run(); + _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); @@ -26,6 +27,7 @@ public static void Main (string[] args) Console.WriteLine("Replace with the name of the benchmark you want to run, e.g. "); Console.WriteLine("StreamReaderBenchmarks: Benchmarks for stream readers"); + Console.WriteLine("DirectReaderBenchmarks: Benchmarks for the default (Direct) reader, the regression gate for reader changes"); Console.WriteLine("ReadThroughputBenchmarks: Benchmarks for read throughput"); Console.WriteLine("BufferIndexBenchmarks: Benchmarks for buffer index"); Console.WriteLine("BufferIndexContentionBenchmarks: Benchmarks for buffer index contention"); diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs index ac234e5c..9121590c 100644 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs +++ b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs @@ -154,6 +154,16 @@ protected StreamReader GetStreamReader () return _reader; } + /// + /// The raw byte stream, positioned where points (past the preamble), for + /// readers that decode bytes themselves instead of going through . + /// + protected Stream GetByteStream () + { + ObjectDisposedException.ThrowIf(IsDisposed, GetType()); + return _stream; + } + protected void MovePosition (int offset) { _position += offset; diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs index a02989d3..c90db3f8 100644 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs +++ b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Runtime.InteropServices; using System.Text; using LogExpert.Core.Entities; @@ -7,30 +8,49 @@ namespace LogExpert.Core.Classes.Log.Streamreaders; /// -/// Reads decoded characters directly into pooled char[] blocks via StreamReader.Read(), -/// scans for newline boundaries, and returns ReadOnlyMemory<char> slices without -/// allocating per-line strings. Eliminates the StreamReader.ReadLine() string allocation. +/// Finds line boundaries in the raw bytes, decodes each line into pooled char[] blocks and returns +/// ReadOnlyMemory<char> slices without allocating per-line strings. /// +/// +/// The byte advances by the bytes a line really +/// occupies, so it stays exact for mixed line endings and for byte sequences that are invalid in the +/// encoding (issue #639): re-encoding decoded text would count each U+FFFD as the bytes of a replacement +/// character instead of the bytes it replaced. Line boundaries are searched as code units of the +/// encoding (one byte, or two or four for UTF-16 and UTF-32); in every encoding LogExpert offers, a +/// \r or \n unit never occurs inside a multi-byte character. +/// still reads through the base ; it and +/// must not be interleaved without setting the position in between. +/// public class PositionAwareStreamReaderDirect : PositionAwareStreamReaderBase, ILogStreamReaderMemory { #region Constants - private const int BLOCK_SIZE = 32_768; // 64 KB (32K chars × 2 bytes), under LOH threshold - private const char CHAR_LF = '\n'; - private const char CHAR_CR = '\r'; + private const int BLOCK_SIZE = 32_768; // 64 KB (32K chars × 2 bytes), under LOH threshold + private const int BYTE_BUFFER_SIZE = 65_536; // raw bytes read from the stream per refill + private const int CODE_PAGE_UTF16_LE = 1200; + private const int CODE_PAGE_UTF16_BE = 1201; - private static readonly SearchValues _lineTerminators = SearchValues.Create("\r\n"); + private static readonly SearchValues _surrogates = SearchValues.Create(Enumerable.Range(0xD800, 0x800).Select(code => (char)code).ToArray()); #endregion #region Fields private char[] _readBlock; - private int _readBlockLength; // valid chars in _readBlock - private int _scanOffset; // current scan position in _readBlock + private int _charFill; // chars of _readBlock already handed out as lines + private byte[] _byteBuffer; // rented on the first read + private int _byteStart; // first byte of the line in progress + private int _byteEnd; // end of the valid bytes in _byteBuffer + private int _scannedBytes; // bytes after _byteStart already searched without a terminator private bool _eof; - private bool _initialized; // first block filled from the current stream position - private int _terminatorCharByteSize; // bytes for a single '\r' or '\n' in the active encoding + private bool _initialized; // terminator units resolved for the encoding + + private int _unitSize; // bytes per code unit: 1, 2 or 4 + private uint _crUnit; + private uint _lfUnit; + private bool _asciiCompatible; // bytes 0x00–0x7F decode to the same chars + private bool _nativeUtf16; // UTF-16 in the machine's byte order + private readonly List _completedBlocks = []; public override bool IsDisposed { get; protected set; } @@ -43,9 +63,6 @@ public PositionAwareStreamReaderDirect (Stream stream, EncodingOptions encodingO : base(stream, encodingOptions, maximumLineLength) { _readBlock = ArrayPool.Shared.Rent(BLOCK_SIZE); - _readBlockLength = 0; - _scanOffset = 0; - _eof = false; } #endregion @@ -58,86 +75,59 @@ public override string ReadLine () } /// - /// Reads the next line by scanning the current block for the next line terminator - /// (\n, \r\n, or a bare \r). If the block is exhausted, tail-copies - /// the partial line to a new block and refills. Returns a zero-copy - /// ReadOnlyMemory<char> slice into the pooled block. The byte position advances by the - /// content bytes plus the bytes of the actual terminator, so it stays exact on - /// files with mixed line endings. + /// Reads the next line, ended by \n, \r\n or a bare \r, and returns a zero-copy + /// slice of its decoded text, capped at the maximum line length. The byte position advances by the + /// line's raw bytes plus the bytes of its actual terminator. /// public bool TryReadLine (out ReadOnlyMemory lineMemory) { - var reader = GetStreamReader(); - - EnsureInitialized(reader); + EnsureInitialized(); while (true) { - // If we have data to scan, look for the next \r or \n - if (_scanOffset < _readBlockLength) + var available = _byteEnd - _byteStart; + var hitIndex = FindTerminator(_byteBuffer.AsSpan(_byteStart + _scannedBytes, available - _scannedBytes)); + + if (hitIndex >= 0) { - var searchSpan = _readBlock.AsSpan(_scanOffset, _readBlockLength - _scanOffset); - var hitIndex = searchSpan.IndexOfAny(_lineTerminators); + hitIndex += _scannedBytes; - if (hitIndex >= 0) + int terminatorBytes; + if (UnitAt(hitIndex) == _lfUnit) { - // Number of char cells the terminator occupies (1 for \n or bare \r, 2 for \r\n). - int terminatorChars; - - if (searchSpan[hitIndex] == CHAR_LF) - { - terminatorChars = 1; - } - else if (hitIndex + 1 < searchSpan.Length) - { - // \r with a known following char: \r\n if it's \n, otherwise a bare \r. - terminatorChars = searchSpan[hitIndex + 1] == CHAR_LF ? 2 : 1; - } - else if (_eof) - { - // \r is the very last char in the file: a bare \r. - terminatorChars = 1; - } - else - { - // \r is the last char of the block but more data follows. Refill so the - // next char becomes available (the tail-copy carries the \r forward), - // then re-scan to classify it as \r\n or a bare \r. - RefillBlock(reader); - continue; - } - - var lineLength = hitIndex; - - // Enforce MaximumLineLength on the returned slice, but count the full - // content for the byte position. - var cappedLength = Math.Min(lineLength, MaximumLineLength); - lineMemory = _readBlock.AsMemory(_scanOffset, cappedLength); - - var contentSpan = _readBlock.AsSpan(_scanOffset, lineLength); - MovePosition(Encoding.GetByteCount(contentSpan) + (terminatorChars * _terminatorCharByteSize)); - - // Advance scan past the content and its terminator. - _scanOffset += hitIndex + terminatorChars; - - return true; + terminatorBytes = _unitSize; } + else if (hitIndex + (2 * _unitSize) <= available) + { + // \r with a known following unit: \r\n if it's \n, otherwise a bare \r. + terminatorBytes = UnitAt(hitIndex + _unitSize) == _lfUnit ? 2 * _unitSize : _unitSize; + } + else if (_eof) + { + // \r is the very last unit in the file: a bare \r. + terminatorBytes = _unitSize; + } + else + { + // \r is the last unit read so far but more data follows. Refill so the next unit + // becomes available, then re-scan from the \r to classify it. + _scannedBytes = hitIndex; + Refill(); + continue; + } + + lineMemory = DecodeLine(hitIndex); + ConsumeBytes(hitIndex + terminatorBytes); + return true; } - // No terminator found (or no data at all). Need to refill. if (_eof) { // Emit remaining content as final line (no trailing newline) - if (_scanOffset < _readBlockLength) + if (available > 0) { - var remaining = _readBlockLength - _scanOffset; - var cappedLength = Math.Min(remaining, MaximumLineLength); - lineMemory = _readBlock.AsMemory(_scanOffset, cappedLength); - - var fullSpan = _readBlock.AsSpan(_scanOffset, remaining); - MovePosition(Encoding.GetByteCount(fullSpan)); - - _scanOffset = _readBlockLength; + lineMemory = DecodeLine(available); + ConsumeBytes(available); return true; } @@ -145,8 +135,9 @@ public bool TryReadLine (out ReadOnlyMemory lineMemory) return false; } - // Tail-copy: move unconsumed chars to a new block and refill - RefillBlock(reader); + // Everything read so far is searched; only whole units count, a partial one is completed by the refill. + _scannedBytes = available - (available % _unitSize); + Refill(); } } @@ -156,129 +147,270 @@ public void ReturnMemory (ReadOnlyMemory memory) } /// - /// Detaches completed blocks (fully scanned) for transfer to the LogBuffer. - /// The current _readBlock (partially scanned) stays with the reader. + /// Detaches the blocks that back the lines read so far, for transfer to the LogBuffer. + /// The reader continues in a fresh block. /// public List DetachCharBlocks () { - // Nothing to detach: no completed blocks and no lines were scanned from the current block. - if (_completedBlocks.Count == 0 && _scanOffset == 0) + if (_charFill > 0) + { + _completedBlocks.Add(_readBlock); + _readBlock = ArrayPool.Shared.Rent(BLOCK_SIZE); + _charFill = 0; + } + + if (_completedBlocks.Count == 0) { return []; } - // The current _readBlock contains memory backing lines already added to the LogBuffer. - // It must be transferred to the buffer along with any completed blocks. - _completedBlocks.Add(_readBlock); + var blocks = _completedBlocks.ToList(); + _completedBlocks.Clear(); + return blocks; + } + + #endregion - // Rent a fresh block and carry over any unscanned data (partial line in progress) - var tailLength = _readBlockLength - _scanOffset; + #region Private Methods - // The tail may exceed BLOCK_SIZE after reading a long line (buffer was grown). - var newBlockSize = BLOCK_SIZE; - while (tailLength > newBlockSize) + private void EnsureInitialized () + { + if (_initialized) { - newBlockSize *= 2; + return; } - var newBlock = ArrayPool.Shared.Rent(newBlockSize); + _initialized = true; + _byteBuffer ??= ArrayPool.Shared.Rent(BYTE_BUFFER_SIZE); - if (tailLength > 0) + if (_unitSize == 0) { - _readBlock.AsSpan(_scanOffset, tailLength).CopyTo(newBlock.AsSpan(0, tailLength)); + ResolveTerminatorUnits(); } + } - _readBlock = newBlock; - _readBlockLength = tailLength; - _scanOffset = 0; + /// + /// Reads how \r and \n are encoded: one byte in single-byte and multi-byte code pages + /// and UTF-8, a 2-byte or 4-byte unit in UTF-16 and UTF-32 of either byte order. + /// + private void ResolveTerminatorUnits () + { + var cr = Encoding.GetBytes("\r"); + var lf = Encoding.GetBytes("\n"); - var blocks = _completedBlocks.ToList(); - _completedBlocks.Clear(); - return blocks; + if (cr.Length != lf.Length || lf.Length is not (1 or 2 or 4)) + { + throw new NotSupportedException($"Encoding '{Encoding.WebName}' does not encode line terminators as single code units."); + } + + _unitSize = lf.Length; + _crUnit = ReadUnit(cr, _unitSize); + _lfUnit = ReadUnit(lf, _unitSize); + _asciiCompatible = _unitSize == 1 && DecodesAsciiUnchanged(Encoding); + _nativeUtf16 = Encoding.CodePage == (BitConverter.IsLittleEndian ? CODE_PAGE_UTF16_LE : CODE_PAGE_UTF16_BE); } - #endregion + /// + /// Whether every byte 0x00–0x7F decodes to the same char: true for UTF-8, ASCII, Latin-1, the + /// Windows code pages and GB2312; false for UTF-7 or EBCDIC, which take the encoding's decoder. + /// + private static bool DecodesAsciiUnchanged (Encoding encoding) + { + Span ascii = stackalloc byte[128]; + for (var i = 0; i < ascii.Length; i++) + { + ascii[i] = (byte)i; + } - #region Private Methods + Span decoded = stackalloc char[encoding.GetMaxCharCount(ascii.Length)]; + var charCount = encoding.GetChars(ascii, decoded); + if (charCount != ascii.Length) + { + return false; + } + + for (var i = 0; i < ascii.Length; i++) + { + if (decoded[i] != i) + { + return false; + } + } + + return true; + } - private void RefillBlock (StreamReader reader) + private static uint ReadUnit (ReadOnlySpan bytes, int unitSize) { - var tailLength = _readBlockLength - _scanOffset; + return unitSize switch + { + 1 => bytes[0], + 2 => MemoryMarshal.Read(bytes), + _ => MemoryMarshal.Read(bytes) + }; + } - // Determine new block size: if the tail already fills a standard block, - // grow the buffer so there's room to read more data. This handles lines - // longer than BLOCK_SIZE (e.g. huge XML payloads). - var newBlockSize = BLOCK_SIZE; - while (tailLength >= newBlockSize) + private uint UnitAt (int offset) + { + return _unitSize == 1 + ? _byteBuffer[_byteStart + offset] + : ReadUnit(_byteBuffer.AsSpan(_byteStart + offset, _unitSize), _unitSize); + } + + /// + /// Byte offset of the first \r or \n unit in , which starts on a + /// unit boundary, or -1. A trailing partial unit is not searched. + /// + private int FindTerminator (ReadOnlySpan bytes) + { + switch (_unitSize) { - newBlockSize *= 2; + case 1: + { + return bytes.IndexOfAny((byte)_crUnit, (byte)_lfUnit); + } + case 2: + { + var index = MemoryMarshal.Cast(bytes).IndexOfAny((ushort)_crUnit, (ushort)_lfUnit); + return index < 0 ? -1 : index * 2; + } + default: + { + var index = MemoryMarshal.Cast(bytes).IndexOfAny(_crUnit, _lfUnit); + return index < 0 ? -1 : index * 4; + } } + } - // Rent a new block (may be larger than BLOCK_SIZE for very long lines) - var newBlock = ArrayPool.Shared.Rent(newBlockSize); + /// + /// Decodes the first bytes of the line in progress into the current + /// block and returns the text, capped at the maximum line length. A line is decoded on its own: + /// a sequence cut short by the terminator becomes U+FFFD in this line, as a continuous decode does. + /// + private ReadOnlyMemory DecodeLine (int byteCount) + { + var bytes = _byteBuffer.AsSpan(_byteStart, byteCount); - // Copy the tail (partial line) to the start of the new block - if (tailLength > 0) + int charCount; + if (_asciiCompatible) + { + // Most log lines are pure ASCII: widen them directly, which skips the per-call cost of the + // encoding's decoder. A line with any other byte is decoded by the encoding instead. + EnsureCharCapacity(byteCount); + if (Ascii.ToUtf16(bytes, _readBlock.AsSpan(_charFill), out charCount) != OperationStatus.Done) + { + charCount = DecodeWithEncoding(bytes); + } + } + else if (_nativeUtf16 && (byteCount & 1) == 0 && !MemoryMarshal.Cast(bytes).ContainsAny(_surrogates)) + { + // UTF-16 in the machine's byte order without surrogates decodes to its own code units. + var source = MemoryMarshal.Cast(bytes); + EnsureCharCapacity(source.Length); + source.CopyTo(_readBlock.AsSpan(_charFill)); + charCount = source.Length; + } + else { - _readBlock.AsSpan(_scanOffset, tailLength).CopyTo(newBlock.AsSpan(0, tailLength)); + charCount = DecodeWithEncoding(bytes); } - // The old block is fully scanned — add to completed list - _completedBlocks.Add(_readBlock); + var cappedLength = Math.Min(charCount, MaximumLineLength); + + var lineMemory = _readBlock.AsMemory(_charFill, cappedLength); + _charFill += cappedLength; + return lineMemory; + } + + private int DecodeWithEncoding (ReadOnlySpan bytes) + { + EnsureCharCapacity(Encoding.GetMaxCharCount(bytes.Length)); + return Encoding.GetChars(bytes, _readBlock.AsSpan(_charFill)); + } - _readBlock = newBlock; - _scanOffset = 0; + private void ConsumeBytes (int byteCount) + { + MovePosition(byteCount); + _byteStart += byteCount; + _scannedBytes = 0; + } - // Fill the rest of the block from the stream - var available = newBlock.Length - tailLength; - var charsRead = reader.Read(newBlock, tailLength, available); + /// + /// Makes room for chars after the lines already in the current block. + /// A block that holds lines is kept for ; an unused one goes back to the pool. + /// Lines longer than BLOCK_SIZE (e.g. huge XML payloads) get a block of their own. + /// + private void EnsureCharCapacity (int charCount) + { + if (_readBlock.Length - _charFill >= charCount) + { + return; + } - _readBlockLength = tailLength + charsRead; + if (_charFill > 0) + { + _completedBlocks.Add(_readBlock); + } + else + { + ArrayPool.Shared.Return(_readBlock); + } - if (charsRead == 0) + var blockSize = BLOCK_SIZE; + while (blockSize < charCount) { - _eof = true; + blockSize *= 2; } + + _readBlock = ArrayPool.Shared.Rent(blockSize); + _charFill = 0; } /// - /// Lazily fills the first block from the current stream position. Called on the first read - /// after construction and after any seek - /// (which resets _initialized via ). + /// Reads more bytes after the line in progress. When the buffer is full, the line in progress is + /// moved to the front, or the buffer doubles if that line already fills it. /// - private void EnsureInitialized (StreamReader reader) + private void Refill () { - if (_initialized) + if (_byteEnd == _byteBuffer.Length) { - return; - } + var lineLength = _byteEnd - _byteStart; - _initialized = true; + if (_byteStart > 0) + { + _byteBuffer.AsSpan(_byteStart, lineLength).CopyTo(_byteBuffer); + } + else + { + var grown = ArrayPool.Shared.Rent(_byteBuffer.Length * 2); + _byteBuffer.AsSpan(0, lineLength).CopyTo(grown); + ArrayPool.Shared.Return(_byteBuffer); + _byteBuffer = grown; + } - // A single '\r' and a single '\n' encode to the same number of bytes in every encoding - // LogExpert uses (ASCII control chars), so one value covers \n, \r and (×2) \r\n. - Span singleTerminator = [CHAR_LF]; - _terminatorCharByteSize = Encoding.GetByteCount(singleTerminator); + _byteStart = 0; + _byteEnd = lineLength; + } - var charsRead = reader.Read(_readBlock, 0, BLOCK_SIZE); - _readBlockLength = charsRead; - _scanOffset = 0; + var bytesRead = GetByteStream().Read(_byteBuffer, _byteEnd, _byteBuffer.Length - _byteEnd); + _byteEnd += bytesRead; - if (charsRead == 0) + if (bytesRead == 0) { _eof = true; } } /// - /// Resets scan state so the next read re-fills from the (just seeked) stream position. - /// Only touches value-type fields, which is safe under the base constructor's virtual call - /// to this method (the pooled _readBlock is left untouched). + /// Discards the bytes read ahead so the next read starts at the (just seeked) stream position. + /// Only touches value-type fields, which is safe under the base constructor's virtual call to this + /// method. Lines already handed out stay valid: their chars are kept until . /// protected override void ResetReader () { - _scanOffset = 0; - _readBlockLength = 0; + _byteStart = 0; + _byteEnd = 0; + _scannedBytes = 0; _eof = false; _initialized = false; @@ -289,13 +421,19 @@ protected override void Dispose (bool disposing) { if (disposing) { - // Return the current (partially scanned) block + // Return the current (partially filled) block if (_readBlock != null) { ArrayPool.Shared.Return(_readBlock); _readBlock = null!; } + if (_byteBuffer != null) + { + ArrayPool.Shared.Return(_byteBuffer); + _byteBuffer = null!; + } + // Return any completed blocks not yet detached foreach (var block in _completedBlocks) { diff --git a/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs new file mode 100644 index 00000000..cf319fb0 --- /dev/null +++ b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs @@ -0,0 +1,327 @@ +using System.Text; + +using LogExpert.Core.Classes.Log.Streamreaders; +using LogExpert.Core.Entities; +using LogExpert.Core.Helpers; + +using NUnit.Framework; + +namespace LogExpert.Tests.StreamReaderTests; + +/// +/// Issue #639: the byte must equal the raw bytes +/// consumed, whatever the bytes are. Re-encoding decoded text overcounts invalid sequences (each U+FFFD +/// re-encodes as 3 UTF-8 bytes), so a file with a binary block looked larger than it was and every +/// growth was mistaken for a recreated file. The oracle here is the raw byte stream itself: a line ends +/// at the next \n, \r\n or bare \r unit, and its text is that line's bytes decoded on +/// their own. +/// +[TestFixture] +public class DirectReaderExactPositionTests +{ + private const int MAX_LINE_LENGTH = 500_000; + + #region Invalid byte sequences + + private static IEnumerable InvalidUtf8Cases () + { + yield return Case("StrayContinuationBytes", "abc", [0x80, 0x81], "def\nnext\n"); + yield return Case("TruncatedSequenceBeforeLf", "x", [0xE6, 0x97], "\ny\n"); + yield return Case("TruncatedSequenceBeforeCrLf", "x", [0xE6], "\r\ny"); + yield return Case("NulAndHighBytes", "a", [0x00, 0x00, 0xFF, 0xFE], "\nb\n"); + yield return Case("OverlongAndEncodedSurrogate", "", [0xC0, 0xAF, 0xED, 0xA0, 0x80], "\nok\n"); + yield return Case("TruncatedSequenceAtEof", "tail", [0xE6, 0x97], ""); + yield return Case("GenuineReplacementCharacter", "valid ", [0xEF, 0xBF, 0xBD], " stays 3 bytes\n"); + yield return Case("BareCrAfterInvalidByte", "", [0xFF], "\rz"); + yield return Case("InvalidLineOnly", "", [0xF5, 0xF6, 0xF7], "\n"); + yield return Case("FourByteSequenceCutShort", "emoji ", [0xF0, 0x9F, 0x99], "\nafter\n"); + + static TestCaseData Case (string name, string before, byte[] invalid, string after) + { + var bytes = Encoding.UTF8.GetBytes(before).Concat(invalid).Concat(Encoding.UTF8.GetBytes(after)).ToArray(); + return new TestCaseData(bytes).SetName($"Position_InvalidUtf8_{name}_MatchesRawBytes"); + } + } + + [TestCaseSource(nameof(InvalidUtf8Cases))] + public void Position_InvalidUtf8_MatchesRawBytes (byte[] data) + { + AssertMatchesOracle(data, Encoding.UTF8); + } + + [Test] + public void Position_InvalidUtf8_ContentShowsReplacementCharacters () + { + byte[] data = [.. "abc"u8, 0x80, .. "def\n"u8]; + + var lines = ReadAll(data, Encoding.UTF8); + + Assert.That(lines.Select(line => line.Content), Is.EqualTo(new[] { "abc�def" })); + } + + [Test] + public void Position_InvalidUtf8LineLongerThanMaximum_TruncatesContentButCountsAllBytes () + { + byte[] data = [.. Encoding.UTF8.GetBytes(new string('A', 50)), 0xFF, 0xFE, .. Encoding.UTF8.GetBytes(new string('B', 50)), .. "\nnext\n"u8]; + + using var stream = new MemoryStream(data); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 20); + + Assert.That(reader.TryReadLine(out var first), Is.True); + Assert.Multiple(() => + { + Assert.That(first.ToString(), Is.EqualTo(new string('A', 20))); + Assert.That(reader.Position, Is.EqualTo(103)); + }); + + Assert.That(reader.TryReadLine(out var second), Is.True); + Assert.Multiple(() => + { + Assert.That(second.ToString(), Is.EqualTo("next")); + Assert.That(reader.Position, Is.EqualTo(data.Length)); + }); + } + + [Test] + public void Position_Utf16WithLoneSurrogateAndOddTrailingByte_MatchesRawBytes () + { + byte[] data = [.. Encoding.Unicode.GetBytes("ab"), 0x00, 0xD8, .. Encoding.Unicode.GetBytes("\r\ncd\n"), 0x41]; + + AssertMatchesOracle(data, Encoding.Unicode); + } + + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] + public void Position_Utf16WithSurrogatePairs_MatchesRawBytes () + { + var data = Encoding.Unicode.GetBytes("plain\r\nemoji 🙂 and 𝄞\npair at end 🙂"); + + AssertMatchesOracle(data, Encoding.Unicode); + } + + [Test] + public void Position_Utf16BigEndian_MatchesRawBytes () + { + var data = Encoding.BigEndianUnicode.GetBytes("first\r\nsecond\nthird\rfourth"); + + AssertMatchesOracle(data, Encoding.BigEndianUnicode); + } + + [Test] + public void Position_Gb2312WithInvalidLeadBytes_MatchesRawBytes () + { + var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); + byte[] data = [.. gb2312.GetBytes("错误 "), 0x81, .. "\n"u8, 0xFF, .. gb2312.GetBytes("警告\n")]; + + AssertMatchesOracle(data, gb2312); + } + + #endregion + + #region Buffer boundaries + + /// + /// A multi-byte character, an invalid sequence or a \r\n pair that straddles the reader's + /// internal buffer must not shift the position. The boundaries cover the old 32 K char block and + /// common byte buffer sizes, on both short-line and single-long-line layouts. + /// + [Test] + public void Position_SequenceStraddlesBufferBoundary_MatchesRawBytes ( + [Values(16_383, 16_384, 32_767, 32_768, 65_535, 65_536, 131_071, 131_072)] int boundary, + [Values("ValidThreeByte", "InvalidTruncated", "CrLf", "Emoji")] string kind, + [Values(true, false)] bool shortLines) + { + byte[] sequence = kind switch + { + "ValidThreeByte" => [0xE4, 0xB8, 0x96], + "InvalidTruncated" => [0xE4, 0xB8], + "CrLf" => [0x0D, 0x0A], + _ => [0xF0, 0x9F, 0x99, 0x82] + }; + + for (var startOffset = 1; startOffset < sequence.Length + 1; startOffset++) + { + var prefix = Filler(boundary - startOffset, shortLines); + byte[] data = [.. prefix, .. sequence, .. "after\nlast"u8]; + + AssertMatchesOracle(data, Encoding.UTF8); + } + + static byte[] Filler (int length, bool shortLines) + { + var filler = new byte[length]; + for (var i = 0; i < length; i++) + { + filler[i] = shortLines && i % 100 == 99 ? (byte)'\n' : (byte)'A'; + } + + return filler; + } + } + + #endregion + + #region Seeking + + [Test] + public void Position_RecordedLineStarts_SeekBackToTheSameLines () + { + var data = TextWithBinaryBlock(lineCount: 3_000, binaryLength: 33_000, seed: 639); + + var expected = new List<(long Start, string Content)>(); + using (var stream = new MemoryStream(data)) + using (var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, MAX_LINE_LENGTH)) + { + var start = reader.Position; + while (reader.TryReadLine(out var line)) + { + expected.Add((start, line.ToString())); + start = reader.Position; + } + } + + using var seekStream = new MemoryStream(data); + using var seekReader = new PositionAwareStreamReaderDirect(seekStream, new EncodingOptions { Encoding = Encoding.UTF8 }, MAX_LINE_LENGTH); + + // Walk backwards so every seek moves the stream and discards what the reader had buffered. + for (var index = expected.Count - 1; index >= 0; index -= 37) + { + seekReader.Position = expected[index].Start; + + Assert.That(seekReader.TryReadLine(out var line), Is.True, $"Line {index}: nothing read after seek"); + Assert.That(line.ToString(), Is.EqualTo(expected[index].Content), $"Line {index}: content after seek"); + } + } + + #endregion + + #region Generated input + + [Test] + public void Position_TextWithBinaryBlock_MatchesRawBytes () + { + AssertMatchesOracle(TextWithBinaryBlock(lineCount: 3_000, binaryLength: 33_000, seed: 42), Encoding.UTF8); + } + + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] + public void Position_RandomBytes_MatchesRawBytes ([Range(0, 49)] int seed) + { + var random = new Random(seed); + var data = new byte[random.Next(1, 200_000)]; + for (var i = 0; i < data.Length; i++) + { + data[i] = random.Next(10) switch + { + 0 => (byte)'\n', + 1 => (byte)'\r', + 2 or 3 => (byte)random.Next(0x80, 0x100), + _ => (byte)random.Next(0x20, 0x7F) + }; + } + + AssertMatchesOracle(data, Encoding.UTF8); + } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] + private static byte[] TextWithBinaryBlock (int lineCount, int binaryLength, int seed) + { + var random = new Random(seed); + using var stream = new MemoryStream(); + + for (var i = 0; i < lineCount; i++) + { + if (i == lineCount / 2) + { + var binary = new byte[binaryLength]; + random.NextBytes(binary); + stream.Write(binary); + } + + stream.Write(Encoding.UTF8.GetBytes($"06/23/2026 00:00:00,MXS SEP26,3Minute,Info,OnVarUpdate(),\"Größe {i} 日志\"\r\n")); + } + + return stream.ToArray(); + } + + #endregion + + #region Oracle + + private static void AssertMatchesOracle (byte[] data, Encoding encoding) + { + var expected = ExpectedLines(data, encoding); + var actual = ReadAll(data, encoding); + + Assert.That(actual.Count, Is.EqualTo(expected.Count), "Line count"); + for (var i = 0; i < expected.Count; i++) + { + if (actual[i] != expected[i]) + { + Assert.Fail($"Line {i}: expected end {expected[i].End} \"{Shorten(expected[i].Content)}\", got end {actual[i].End} \"{Shorten(actual[i].Content)}\""); + } + } + } + + private static List<(string Content, long End)> ReadAll (byte[] data, Encoding encoding) + { + using var stream = new MemoryStream(data); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, MAX_LINE_LENGTH); + + var lines = new List<(string, long)>(); + while (reader.TryReadLine(out var line)) + { + lines.Add((line.ToString(), reader.Position)); + } + + return lines; + } + + /// + /// Splits the raw bytes at \n, \r\n and bare \r code units of the encoding and + /// decodes each line's bytes on its own. A trailing line without terminator is a line; an empty + /// remainder after the last terminator is not. + /// + private static List<(string Content, long End)> ExpectedLines (byte[] data, Encoding encoding) + { + var cr = encoding.GetBytes("\r"); + var lf = encoding.GetBytes("\n"); + var unit = lf.Length; + + var lines = new List<(string, long)>(); + var lineStart = 0; + var position = 0; + + while (position + unit <= data.Length) + { + var span = data.AsSpan(position, unit); + if (span.SequenceEqual(lf) || span.SequenceEqual(cr)) + { + var terminatorLength = unit; + if (span.SequenceEqual(cr) && position + (2 * unit) <= data.Length && data.AsSpan(position + unit, unit).SequenceEqual(lf)) + { + terminatorLength = 2 * unit; + } + + lines.Add((encoding.GetString(data, lineStart, position - lineStart), position + terminatorLength)); + position += terminatorLength; + lineStart = position; + } + else + { + position += unit; + } + } + + if (lineStart < data.Length) + { + lines.Add((encoding.GetString(data, lineStart, data.Length - lineStart), data.Length)); + } + + return lines; + } + + private static string Shorten (string text) => text.Length <= 60 ? text : $"{text[..60]}… ({text.Length} chars)"; + + #endregion +} \ No newline at end of file diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs index 3953b04a..0235b954 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs @@ -73,4 +73,70 @@ public void SingleFileReader_OnTruncation_ReportsNewFileNotRollover () Assert.That(newFileReported.Wait(TimeSpan.FromSeconds(5)), Is.True, "Truncation did not report a new file"); Assert.That(rolloverReported, Is.False); } + + /// + /// Issue #639: a file with a binary block in the middle must grow like any other file. An overcounted + /// byte position made the file look smaller than what had been read, so every append was reported as + /// a new file and the window reloaded and jumped to the top. + /// + [Test] + [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] + public void SingleFileReader_GrowingFileWithBinaryBlock_TakesGrowthPathNotNewFile () + { + // No BOM: FileSize counts from after the preamble, so without one it equals the length on disk. + var binary = new byte[33_000]; + new Random(639).NextBytes(binary); + using (var stream = new FileStream(_logFile, FileMode.Create, FileAccess.Write)) + { + stream.Write(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Range(1, 100).Select(index => $"Line {index}\n")))); + stream.Write(binary); + stream.Write(Encoding.UTF8.GetBytes("Line after binary\n")); + } + + using var loadingFinished = new ManualResetEventSlim(false); + using var grown = new ManualResetEventSlim(false); + var newFileReported = false; + var progressReporterMock = new Mock(); + _ = progressReporterMock + .Setup(reporter => reporter.ReportLoadingFinished()) + .Callback(() => loadingFinished.Set()); + _ = progressReporterMock + .Setup(reporter => reporter.ReportNewFile(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback(() => newFileReported = true); + + using var reader = new LogfileReader( + _logFile, + new EncodingOptions { Encoding = Encoding.UTF8 }, + multiFile: false, + bufferCount: 40, + linesPerBuffer: 50, + new MultiFileOptions(), + ReaderType.SystemDirect, + PluginRegistry.PluginRegistry.Instance, + maximumLineLength: 500, + progressReporter: progressReporterMock.Object); + + reader.StartMonitoring(); + Assert.That(loadingFinished.Wait(TimeSpan.FromSeconds(5)), Is.True, "Initial load did not finish"); + Assert.That(reader.FileSize, Is.EqualTo(new FileInfo(_logFile).Length), "Size read after the initial load"); + + var linesBefore = reader.LineCount; + reader.FileSizeChanged += (_, args) => + { + if (args.LineCount > linesBefore) + { + grown.Set(); + } + }; + + File.AppendAllText(_logFile, "Appended 1\nAppended 2\n", Encoding.UTF8); + + Assert.That(grown.Wait(TimeSpan.FromSeconds(5)), Is.True, "Growth was not reported"); + Assert.Multiple(() => + { + Assert.That(newFileReported, Is.False, "Growth was mistaken for a new file"); + Assert.That(reader.LineCount, Is.EqualTo(linesBefore + 2)); + Assert.That(reader.FileSize, Is.EqualTo(new FileInfo(_logFile).Length), "Size read after the growth"); + }); + } } diff --git a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs index 19843334..cafbea20 100644 --- a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs @@ -274,6 +274,71 @@ public void ContentParity_LargeFile () #endregion + #region Allocations + + /// + /// The reader must stay allocation-free per line: lines land in pooled blocks, and only detaching + /// blocks allocates (the list handed to the buffer). Measured on the reading thread, so the result + /// is deterministic; any per-line allocation (at least 24 bytes) breaks the budget of 0.25 bytes. + /// + [Test] + public void TryReadLine_ValidText_DoesNotAllocatePerLine ([Values(65001, 1200)] int codePage) + { + const int lineCount = 200_000; + const int linesPerBuffer = 500; + var encoding = Encoding.GetEncoding(codePage); + + var sb = new StringBuilder(); + for (var i = 0; i < lineCount; i++) + { + _ = sb.Append("2026-04-23 12:00:00.").Append(i % 1000).Append(" [INFO] Thread-7 SomeClass - Log message number ").Append(i).Append("\r\n"); + } + + var data = encoding.GetBytes(sb.ToString()); + + // Warm up the JIT and the array pool, then measure one full read. + _ = ReadAllAndReturnBlocks(data, encoding, linesPerBuffer); + _ = ReadAllAndReturnBlocks(data, encoding, linesPerBuffer); + + var before = GC.GetAllocatedBytesForCurrentThread(); + var lines = ReadAllAndReturnBlocks(data, encoding, linesPerBuffer); + var allocated = GC.GetAllocatedBytesForCurrentThread() - before; + + Assert.Multiple(() => + { + Assert.That(lines, Is.EqualTo(lineCount)); + Assert.That(allocated, Is.LessThan(lineCount / 4), $"Allocated {allocated} bytes for {lines} lines"); + }); + } + + private static int ReadAllAndReturnBlocks (byte[] data, Encoding encoding, int linesPerBuffer) + { + using var stream = new MemoryStream(data, writable: false); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, 20_000); + + var lines = 0; + while (reader.TryReadLine(out _)) + { + if (++lines % linesPerBuffer == 0) + { + ReturnBlocks(reader.DetachCharBlocks()); + } + } + + ReturnBlocks(reader.DetachCharBlocks()); + return lines; + + static void ReturnBlocks (List blocks) + { + foreach (var block in blocks) + { + System.Buffers.ArrayPool.Shared.Return(block); + } + } + } + + #endregion + #region DetachCharBlocks [Test] From b85d1a5f011794378cf8b1018c6e9f80eeba0e87 Mon Sep 17 00:00:00 2001 From: BRUNER Patrick Date: Fri, 20 Nov 2026 15:02:17 +0100 Subject: [PATCH 2/3] Remove the System and Legacy stream readers and the reader choice With Direct exact for every byte sequence and line ending, the System and Legacy readers no longer cover anything Direct does not. Both read through StreamReader, which never reports the bytes it consumed, so neither could be made exact without becoming a copy of Direct. They were kept only as a one-release safety net (ADR 0006 update), and System already guessed one newline length per file. Removes PositionAwareStreamReaderSystem, PositionAwareStreamReaderLegacy, the ReaderType enum, Preferences.ReaderType (and the dead UseLegacyReader), the readerType parameter of LogfileReader, and the Settings combobox with its tooltip resource. Settings saved with "ReaderType" still load: Newtonsoft ignores the unknown member, which a new LoadOrCreateNew test pins for each stored value. Tests that used the System reader as oracle now check against the raw bytes; reader tests that ran per reader run against Direct, and duplicates are gone. The ReaderTest case that pinned System's overcount at EOF now expects the exact position. StreamReaderBenchmarks (System/Legacy only) is replaced by DirectReaderBenchmarks; the two ReadThroughput cases that only differed in reader type are dropped. --- src/LogExpert.Benchmarks/Program.cs | 8 +- .../ReadThroughputBenchmarks.cs | 51 ----- .../StreamReaderBenchmarks.cs | 152 --------------- .../Classes/Log/LogfileReader.cs | 17 +- .../PositionAwareStreamReaderBase.cs | 8 +- .../PositionAwareStreamReaderLegacy.cs | 149 --------------- .../PositionAwareStreamReaderSystem.cs | 174 ------------------ src/LogExpert.Core/Config/Preferences.cs | 10 +- src/LogExpert.Core/Enums/ReaderType.cs | 27 --- .../Interfaces/ILogStreamReader.cs | 4 +- src/LogExpert.Resources/Resources.Designer.cs | 9 - src/LogExpert.Resources/Resources.de.resx | 3 - src/LogExpert.Resources/Resources.resx | 3 - src/LogExpert.Resources/Resources.zh-CN.resx | 3 - .../Buffers/BufferShiftTest.cs | 7 +- .../ColumnizerTests/CSVColumnizerTest.cs | 43 +++-- .../ColumnizerTests/ColumnizerPickerTest.cs | 9 +- .../ColumnizerTests/JsonColumnizerTest.cs | 7 +- .../JsonCompactColumnizerTest.cs | 11 +- .../SquareBracketColumnizerTest.cs | 15 +- .../ConfigManagerTests/ConfigManagerTest.cs | 34 ++++ src/LogExpert.Tests/ReaderTest.cs | 76 ++------ .../DirectReaderExactPositionTests.cs | 2 +- .../StreamReaderTests/LogStreamReaderTest.cs | 114 +++--------- .../LogfileReaderBlockAllocationTests.cs | 8 - .../LogfileReaderEncodingTests.cs | 2 - .../LogfileReaderMultiFileFlagTests.cs | 3 - .../LogfileReaderSingleFileMonitoringTests.cs | 3 - .../PositionAwareStreamReaderDirectTests.cs | 130 +++---------- .../Services/FileOperationServiceTests.cs | 7 +- .../Controls/LogWindow/LogWindow.cs | 4 +- .../Dialogs/SettingsDialog.Designer.cs | 11 -- src/LogExpert.UI/Dialogs/SettingsDialog.cs | 16 -- 33 files changed, 151 insertions(+), 969 deletions(-) delete mode 100644 src/LogExpert.Benchmarks/StreamReaderBenchmarks.cs delete mode 100644 src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderLegacy.cs delete mode 100644 src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderSystem.cs delete mode 100644 src/LogExpert.Core/Enums/ReaderType.cs diff --git a/src/LogExpert.Benchmarks/Program.cs b/src/LogExpert.Benchmarks/Program.cs index 62f85f82..10dbe02e 100644 --- a/src/LogExpert.Benchmarks/Program.cs +++ b/src/LogExpert.Benchmarks/Program.cs @@ -12,7 +12,6 @@ public static void Main (string[] args) Console.WriteLine("No benchmarks specified. Running all benchmarks..."); // Run all benchmarks if no arguments are provided - _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); _ = BenchmarkRunner.Run(); @@ -26,7 +25,6 @@ public static void Main (string[] args) } Console.WriteLine("Replace with the name of the benchmark you want to run, e.g. "); - Console.WriteLine("StreamReaderBenchmarks: Benchmarks for stream readers"); Console.WriteLine("DirectReaderBenchmarks: Benchmarks for the default (Direct) reader, the regression gate for reader changes"); Console.WriteLine("ReadThroughputBenchmarks: Benchmarks for read throughput"); Console.WriteLine("BufferIndexBenchmarks: Benchmarks for buffer index"); @@ -44,11 +42,11 @@ public static void Main (string[] args) /* * Comment / Uncommen the benchmark to run, careful some can run longer * 1.) a dry run - * dotnet run -c Release -- --filter "StreamReaderBenchmarks" --job Dry --noOverwrite + * dotnet run -c Release -- --filter "DirectReaderBenchmarks" --job Dry --noOverwrite * 2.) a short run - * dotnet run -c Release -- --filter "StreamReaderBenchmarks" --job Short --noOverwrite + * dotnet run -c Release -- --filter "DirectReaderBenchmarks" --job Short --noOverwrite * 3.) a full baseline run - * dotnet run -c Release -- --filter "StreamReaderBenchmarks" --noOverwrite + * dotnet run -c Release -- --filter "DirectReaderBenchmarks" --noOverwrite * * The full baseline run generates a MD file * BenchmarkDotNet.Artifacts/results/*-report-github.md diff --git a/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs b/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs index 688e2bd5..2b261074 100644 --- a/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs +++ b/src/LogExpert.Benchmarks/ReadThroughputBenchmarks.cs @@ -4,7 +4,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; namespace LogExpert.Benchmarks; @@ -45,7 +44,6 @@ public int ReadWithNullReporter () bufferCount: 500, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -67,7 +65,6 @@ public int ReadWithPeriodicReporter () bufferCount: 500, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500); // No progressReporter = default PeriodicProgressReporter @@ -76,54 +73,6 @@ public int ReadWithPeriodicReporter () return reader.LineCount; } - /// - /// Post-change: read with block-based allocation (System reader uses CharBlockAllocator). - /// Compare Gen0/Gen1/Gen2 collections vs baseline to validate allocation reduction. - /// This method is identical to ReadWithNullReporter — it exists solely for explicit - /// before/after naming in benchmark reports. - /// - [Benchmark] - public int ReadWithBlockAllocation () - { - using var reader = new LogfileReader( - _tempFile, - new EncodingOptions { Encoding = Encoding.UTF8 }, - multiFile: false, - bufferCount: 500, - linesPerBuffer: 500, - new MultiFileOptions(), - ReaderType.System, - PluginRegistry.PluginRegistry.Instance, - maximumLineLength: 500, - progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); - - reader.ReadFiles(); - return reader.LineCount; - } - - /// - /// Direct-read: reads decoded chars directly into pooled blocks without per-line string allocation. - /// Compare allocations and GC counts vs ReadWithBlockAllocation. - /// - [Benchmark] - public int ReadWithDirectRead () - { - using var reader = new LogfileReader( - _tempFile, - new EncodingOptions { Encoding = Encoding.UTF8 }, - multiFile: false, - bufferCount: 500, - linesPerBuffer: 500, - new MultiFileOptions(), - ReaderType.SystemDirect, - PluginRegistry.PluginRegistry.Instance, - maximumLineLength: 500, - progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); - - reader.ReadFiles(); - return reader.LineCount; - } - [GlobalCleanup] public void Cleanup () { diff --git a/src/LogExpert.Benchmarks/StreamReaderBenchmarks.cs b/src/LogExpert.Benchmarks/StreamReaderBenchmarks.cs deleted file mode 100644 index a82f0432..00000000 --- a/src/LogExpert.Benchmarks/StreamReaderBenchmarks.cs +++ /dev/null @@ -1,152 +0,0 @@ -using System.Text; - -using BenchmarkDotNet.Attributes; - -using LogExpert.Core.Classes.Log.Streamreaders; -using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; - -namespace LogExpert.Benchmarks; - -[MemoryDiagnoser] -[RankColumn] -public class StreamReaderBenchmarks -{ - private byte[] _smallTestData; - private byte[] _mediumTestData; - private byte[] _largeTestData; - private byte[] _unicodeTestData; - - [GlobalSetup] - public void Setup () - { - // Small: 1000 lines, ~50 bytes each = ~50 KB - _smallTestData = GenerateTestData(1000, 50); - - // Medium: 10000 lines, ~100 bytes each = ~1 MB - _mediumTestData = GenerateTestData(10000, 100); - - // Large: 100000 lines, ~200 bytes each = ~20 MB - _largeTestData = GenerateTestData(100000, 200); - - // Unicode: 5000 lines with mixed ASCII and Unicode - _unicodeTestData = GenerateUnicodeTestData(5000); - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Test")] - private static byte[] GenerateTestData (int lineCount, int avgLineLength) - { - var sb = new StringBuilder(); - var random = new Random(42); // Fixed seed for reproducibility - - for (int i = 0; i < lineCount; i++) - { - var lineLength = avgLineLength + random.Next(-10, 11); // Vary line length slightly - var line = $"Line {i:D10} " + new string('X', Math.Max(0, lineLength - 20)); - _ = sb.AppendLine(line); - } - - return Encoding.UTF8.GetBytes(sb.ToString()); - } - - [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Test")] - private static byte[] GenerateUnicodeTestData (int lineCount) - { - var sb = new StringBuilder(); - var random = new Random(42); - - for (int i = 0; i < lineCount; i++) - { - var lineType = random.Next(0, 3); - var line = lineType switch - { - 0 => $"Line {i}: ASCII text only", - 1 => $"Line {i}: Hello 世界 (Chinese)", - _ => $"Line {i}: Спасибо большое (Russian)" - }; - _ = sb.AppendLine(line); - } - - return Encoding.UTF8.GetBytes(sb.ToString()); - } - - [Benchmark(Baseline = true)] - [BenchmarkCategory("Legacy", "Small", "ReadAll")] - public void Legacy_ReadAll_Small () - { - using var stream = new MemoryStream(_smallTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Small", "ReadAll")] - public void System_ReadAll_Small () - { - using var stream = new MemoryStream(_smallTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("Legacy", "Medium", "ReadAll")] - public void Legacy_ReadAll_Medium () - { - using var stream = new MemoryStream(_mediumTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Medium", "ReadAll")] - public void System_ReadAll_Medium () - { - using var stream = new MemoryStream(_mediumTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("Legacy", "Large", "ReadAll")] - public void Legacy_ReadAll_Large () - { - using var stream = new MemoryStream(_largeTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Large", "ReadAll")] - public void System_ReadAll_Large () - { - using var stream = new MemoryStream(_largeTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("System", "Unicode", "ReadAll")] - public void System_ReadAll_Unicode () - { - using var stream = new MemoryStream(_unicodeTestData); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - [Benchmark] - [BenchmarkCategory("Legacy", "Unicode", "ReadAll")] - public void Legacy_ReadAll_Unicode () - { - using var stream = new MemoryStream(_unicodeTestData); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 10000); - ReadAllLines(reader); - } - - private static void ReadAllLines (ILogStreamReader reader) - { - while (reader.ReadLine() != null) - { - // Consume the line - } - } -} \ No newline at end of file diff --git a/src/LogExpert.Core/Classes/Log/LogfileReader.cs b/src/LogExpert.Core/Classes/Log/LogfileReader.cs index 142f3ad6..d0a23ef7 100644 --- a/src/LogExpert.Core/Classes/Log/LogfileReader.cs +++ b/src/LogExpert.Core/Classes/Log/LogfileReader.cs @@ -8,7 +8,6 @@ using LogExpert.Core.Classes.Log.Streamreaders; using LogExpert.Core.Classes.xml; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.EventArguments; using LogExpert.Core.Interfaces; @@ -29,7 +28,6 @@ public partial class LogfileReader : ILogfileReader, IMultiFileNavigation, ILogf private readonly MultiFileOptions _multiFileOptions; private readonly IPluginRegistry _pluginRegistry; private readonly CancellationTokenSource _cts = new(); - private readonly ReaderType _readerType; private readonly int _maximumLineLength; private readonly LogBufferPool _bufferPool; @@ -67,7 +65,6 @@ public LogfileReader ( int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, - ReaderType readerType, IPluginRegistry pluginRegistry, int maximumLineLength, ILoadProgressReporter? progressReporter = null) @@ -78,7 +75,6 @@ public LogfileReader ( bufferCount, linesPerBuffer, multiFileOptions, - readerType, pluginRegistry, maximumLineLength, progressReporter) @@ -92,7 +88,6 @@ public LogfileReader ( int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, - ReaderType readerType, IPluginRegistry pluginRegistry, int maximumLineLength, ILoadProgressReporter? progressReporter = null, @@ -104,7 +99,6 @@ public LogfileReader ( bufferCount, linesPerBuffer, multiFileOptions, - readerType, pluginRegistry, maximumLineLength, progressReporter, @@ -121,7 +115,6 @@ private LogfileReader ( int bufferCount, int linesPerBuffer, MultiFileOptions multiFileOptions, - ReaderType readerType, IPluginRegistry pluginRegistry, int maximumLineLength, ILoadProgressReporter? progressReporter = null, @@ -140,7 +133,6 @@ private LogfileReader ( } _maximumLineLength = maximumLineLength; - _readerType = readerType; EncodingOptions = encodingOptions; _max_buffers = bufferCount; _maxLinesPerBuffer = linesPerBuffer; @@ -1709,14 +1701,7 @@ private ILogStreamReader GetLogStreamReader (Stream stream, EncodingOptions enco /// private ILogStreamReader CreateLogStreamReader (Stream stream, EncodingOptions encodingOptions) { - return _readerType switch - { - ReaderType.Legacy => new PositionAwareStreamReaderLegacy(stream, encodingOptions, _maximumLineLength), - ReaderType.System => new PositionAwareStreamReaderSystem(stream, encodingOptions, _maximumLineLength), - ReaderType.SystemDirect => new PositionAwareStreamReaderDirect(stream, encodingOptions, _maximumLineLength), - //Default will be SystemDirect, because it is the best performing reader and should be used if not explicitly overridden by user. - _ => new PositionAwareStreamReaderDirect(stream, encodingOptions, _maximumLineLength), - }; + return new PositionAwareStreamReaderDirect(stream, encodingOptions, _maximumLineLength); } /// diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs index 9121590c..7c60d3f1 100644 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs +++ b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderBase.cs @@ -148,15 +148,9 @@ protected virtual void ResetReader () _reader.DiscardBufferedData(); } - protected StreamReader GetStreamReader () - { - ObjectDisposedException.ThrowIf(IsDisposed, GetType()); - return _reader; - } - /// /// The raw byte stream, positioned where points (past the preamble), for - /// readers that decode bytes themselves instead of going through . + /// readers that decode bytes themselves; reads through the base StreamReader. /// protected Stream GetByteStream () { diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderLegacy.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderLegacy.cs deleted file mode 100644 index 2a2880be..00000000 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderLegacy.cs +++ /dev/null @@ -1,149 +0,0 @@ -using LogExpert.Core.Classes.Log.Buffers; -using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; - -namespace LogExpert.Core.Classes.Log.Streamreaders; - -public class PositionAwareStreamReaderLegacy (Stream stream, EncodingOptions encodingOptions, int maximumLineLength) : PositionAwareStreamReaderBase(stream, encodingOptions, maximumLineLength), ILogStreamReaderMemory -{ - #region Fields - - private readonly char[] _charBuffer = new char[maximumLineLength]; - - private int _charBufferPos; - private bool _crDetect; - - public override bool IsDisposed { get; protected set; } - - #endregion - - #region Properties - - private CharBlockAllocator BlockAllocator - { - get => field ??= new CharBlockAllocator(); - set; - } - - #endregion - - #region Public methods - - public bool TryReadLine (out ReadOnlyMemory lineMemory) - { - var line = ReadLine(); - - if (line is null) - { - lineMemory = default; - return false; - } - - var target = BlockAllocator.Rent(line.Length); - line.AsSpan().CopyTo(target.Span); - lineMemory = target; - return true; - } - - public void ReturnMemory (ReadOnlyMemory memory) - { - // Bulk return via DetachCharBlocks() when the LogBuffer is evicted, or Dispose(). - } - - public List DetachCharBlocks () => BlockAllocator.DetachBlocks(); - - protected override void Dispose (bool disposing) - { - if (disposing) - { - BlockAllocator?.Dispose(); - BlockAllocator = null; - } - - base.Dispose(disposing); - } - - public override string ReadLine () - { - int readInt; - - while (-1 != (readInt = ReadChar())) - { - var readChar = (char)readInt; - - switch (readChar) - { - case '\n': - { - _crDetect = false; - return GetLineAndResetCharBufferPos(); - } - case '\r': - { - if (_crDetect) - { - return GetLineAndResetCharBufferPos(); - } - - _crDetect = true; - break; - } - default: - { - if (_crDetect) - { - _crDetect = false; - var line = GetLineAndResetCharBufferPos(); - AppendToCharBuffer(readChar); - return line; - } - - AppendToCharBuffer(readChar); - break; - } - } - } - - var result = GetLineAndResetCharBufferPos(); - if (readInt == -1 && result.Length == 0 && !_crDetect) - { - return null; // EOF - } - - _crDetect = false; - return result; - } - - protected override void ResetReader () - { - ResetCharBufferPos(); - - base.ResetReader(); - } - - #endregion - - #region Private Methods - - private string GetLineAndResetCharBufferPos () - { - string result = new(_charBuffer, 0, _charBufferPos); - ResetCharBufferPos(); - return result; - } - - private void AppendToCharBuffer (char readChar) - { - if (_charBufferPos < MaximumLineLength) - { - _charBuffer[_charBufferPos++] = readChar; - } - } - - private void ResetCharBufferPos () - { - _charBufferPos = 0; - } - - #endregion -} \ No newline at end of file diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderSystem.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderSystem.cs deleted file mode 100644 index 55090406..00000000 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderSystem.cs +++ /dev/null @@ -1,174 +0,0 @@ -using System.Text; - -using LogExpert.Core.Classes.Log.Buffers; -using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; - -namespace LogExpert.Core.Classes.Log.Streamreaders; - -/// -/// This class is responsible for reading line from the log file. It also decodes characters with the appropriate charset encoding. -/// PositionAwareStreamReaderSystem tries a BOM detection to determine correct file offsets when directly seeking into the file (on re-loading flushed buffers). -/// UTF-8 handling is a bit slower, because after reading a character the byte length of the character must be determined. -/// Lines are read char-by-char. StreamReader.ReadLine() is not used because StreamReader cannot tell a file position. -/// -public class PositionAwareStreamReaderSystem : PositionAwareStreamReaderBase, ILogStreamReaderMemory -{ - #region Fields - - private const int CHAR_CR = 0x0D; - private const int CHAR_LF = 0x0A; - - private int _newLineSequenceLength; - - public override bool IsDisposed { get; protected set; } - - #endregion - - #region cTor - - public PositionAwareStreamReaderSystem (Stream stream, EncodingOptions encodingOptions, int maximumLineLength) : base(stream, encodingOptions, maximumLineLength) - { - } - - #endregion - - #region Properties - - /// - /// The block allocator used by this reader instance. Blocks are handed to the owning buffer via - /// ; ownership of the allocator does not cross the reader seam. - /// - private CharBlockAllocator BlockAllocator - { - get => field ??= new CharBlockAllocator(); - set; - } - - #endregion - - #region Public methods - - public override string ReadLine () - { - var reader = GetStreamReader(); - - if (_newLineSequenceLength == 0) - { - _newLineSequenceLength = GuessNewLineSequenceLength(reader); - } - - var line = reader.ReadLine(); - - if (line != null) - { - MovePosition(Encoding.GetByteCount(line) + _newLineSequenceLength); - - if (line.Length > MaximumLineLength) - { - line = line[..MaximumLineLength]; - } - } - - return line; - } - - /// - /// Attempts to read the next line from the stream without allocating a new string. - /// The returned Memory<char> is valid until the next call to TryReadLine or ReturnMemory. - /// - public bool TryReadLine (out ReadOnlyMemory lineMemory) - { - var reader = GetStreamReader(); - - if (_newLineSequenceLength == 0) - { - _newLineSequenceLength = GuessNewLineSequenceLength(reader); - } - - var line = reader.ReadLine(); - - if (line is null) - { - lineMemory = default; - return false; - } - - MovePosition(Encoding.GetByteCount(line) + _newLineSequenceLength); - - var length = Math.Min(line.Length, MaximumLineLength); - - // Allocate from block and copy - var allocator = BlockAllocator; - var target = allocator.Rent(length); - line.AsSpan(0, length).CopyTo(target.Span); - lineMemory = target; - return true; - } - - /// - /// Returns the memory buffer. For the block-based reader, individual returns are not tracked — blocks are returned - /// in bulk via the BlockAllocator when the LogBuffer is evicted or the reader is disposed. - /// - public void ReturnMemory (ReadOnlyMemory memory) - { - // Bulk return via DetachCharBlocks() or Dispose(). - // Individual per-line return is not needed with block-based allocation. - } - - public List DetachCharBlocks () => BlockAllocator.DetachBlocks(); - - #endregion - - #region Private Methods - - private int GuessNewLineSequenceLength (StreamReader reader) - { - var currentPos = Position; - - try - { - var line = reader.ReadLine(); - - if (line != null) - { - Position += Encoding.GetByteCount(line); - - var firstChar = reader.Read(); - if (firstChar == CHAR_CR) // check \r - { - var secondChar = reader.Read(); - if (secondChar == CHAR_LF) // check \n - { - // Use stackalloc or SpanOwner instead of string - Span newline = ['\r', '\n']; - return Encoding.GetByteCount(newline); - //return Encoding.GetByteCount("\r\n"); - } - } - - Span singleChar = [(char)firstChar]; - return Encoding.GetByteCount(singleChar); - } - - return 0; - } - finally - { - Position = currentPos; - } - } - - protected override void Dispose (bool disposing) - { - if (disposing) - { - BlockAllocator?.Dispose(); - BlockAllocator = null; - } - - base.Dispose(disposing); - } - - #endregion -} \ No newline at end of file diff --git a/src/LogExpert.Core/Config/Preferences.cs b/src/LogExpert.Core/Config/Preferences.cs index c3125158..1bcef910 100644 --- a/src/LogExpert.Core/Config/Preferences.cs +++ b/src/LogExpert.Core/Config/Preferences.cs @@ -77,8 +77,7 @@ public List HilightGroupList /// /// /// This property controls line truncation at the I/O reader level before lines are processed by columnizers. - /// It is implemented in and - /// . Related property: + /// It is implemented in . Related property: /// controls display-level truncation in UI columns, which must not exceed this /// value. Default is 20000 characters. /// @@ -134,13 +133,6 @@ public bool? DarkMode } } - [Obsolete("This setting is no longer used and will be removed in version 1.50. The 'UseLegacyReader' now works with ReaderType.Legacy")] - [System.Text.Json.Serialization.JsonIgnore] - [Newtonsoft.Json.JsonIgnore] - public bool UseLegacyReader { get; set; } - - public ReaderType ReaderType { get; set; } = ReaderType.SystemDirect; - public List ToolEntries { get; set; } = []; public DragOrientations TimestampControlDragOrientation { get; set; } = DragOrientations.Horizontal; diff --git a/src/LogExpert.Core/Enums/ReaderType.cs b/src/LogExpert.Core/Enums/ReaderType.cs deleted file mode 100644 index facd26c4..00000000 --- a/src/LogExpert.Core/Enums/ReaderType.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; - -namespace LogExpert.Core.Enums; - -/// -/// Defines the available stream reader implementations. -/// -[JsonConverter(typeof(StringEnumConverter))] -public enum ReaderType -{ - /// - /// Direct-read implementation: reads decoded chars directly into pooled blocks via - /// StreamReader.Read(char[], offset, count), eliminating per-line string allocation. - /// - SystemDirect = 0, - - /// - /// Legacy reader implementation (original). - /// - Legacy = 1, - - /// - /// System.IO.StreamReader based implementation. - /// - System = 2 -} diff --git a/src/LogExpert.Core/Interfaces/ILogStreamReader.cs b/src/LogExpert.Core/Interfaces/ILogStreamReader.cs index 8c883798..7303eee2 100644 --- a/src/LogExpert.Core/Interfaces/ILogStreamReader.cs +++ b/src/LogExpert.Core/Interfaces/ILogStreamReader.cs @@ -15,9 +15,7 @@ namespace LogExpert.Core.Interfaces; /// /// Implementations include: /// -/// PositionAwareStreamReaderDirect - Default. Reads decoded chars into pooled blocks and returns zero-copy slices; fastest, assumes \n/\r\n line endings -/// PositionAwareStreamReaderSystem - Uses .NET's StreamReader.ReadLine(); also splits bare \r (classic-Mac) line endings -/// PositionAwareStreamReaderLegacy - Character-by-character reading; exact byte position on mixed/pathological line endings, slowest +/// PositionAwareStreamReaderDirect - Finds lines in the raw bytes, decodes them into pooled blocks and returns zero-copy slices; exact byte position for any content and line ending (ADR 0010) /// XmlLogReader - Decorator for reading structured XML log blocks (e.g., Log4j XML format) /// /// diff --git a/src/LogExpert.Resources/Resources.Designer.cs b/src/LogExpert.Resources/Resources.Designer.cs index e877eb2e..99ba15df 100644 --- a/src/LogExpert.Resources/Resources.Designer.cs +++ b/src/LogExpert.Resources/Resources.Designer.cs @@ -5960,15 +5960,6 @@ public static string SettingsDialog_UI_CheckBox_ToolTip_toolTipPortableMode { } } - /// - /// 查找类似 File Reader algorithm 的本地化字符串。 - /// - public static string SettingsDialog_UI_CheckBox_ToolTip_toolTipReaderTyp { - get { - return ResourceManager.GetString("SettingsDialog_UI_CheckBox_ToolTip_toolTipReaderTyp", resourceCulture); - } - } - /// /// 查找类似 HeaderName 的本地化字符串。 /// diff --git a/src/LogExpert.Resources/Resources.de.resx b/src/LogExpert.Resources/Resources.de.resx index 594df5d8..09297e16 100644 --- a/src/LogExpert.Resources/Resources.de.resx +++ b/src/LogExpert.Resources/Resources.de.resx @@ -900,9 +900,6 @@ Damit die anderen Fenster beim Auswählen einer Zeile (Mausklick oder Pfeiltaste Wähle ein Verzeichnis für die LogExpert Sessiondateien - - Dateiauslese algorithmus - Bei Aktivierung des Modus, wird die gespeicherte Datei aus dem Verzeichnis der Executable geladen diff --git a/src/LogExpert.Resources/Resources.resx b/src/LogExpert.Resources/Resources.resx index 078363a3..f1c16575 100644 --- a/src/LogExpert.Resources/Resources.resx +++ b/src/LogExpert.Resources/Resources.resx @@ -1030,9 +1030,6 @@ Checked tools will appear in the icon bar. All other tools are available in the If this mode is activated, the save file will be loaded from the Executable Location - - File Reader algorithm - Settings diff --git a/src/LogExpert.Resources/Resources.zh-CN.resx b/src/LogExpert.Resources/Resources.zh-CN.resx index 9ed0b1dd..7b93aec6 100644 --- a/src/LogExpert.Resources/Resources.zh-CN.resx +++ b/src/LogExpert.Resources/Resources.zh-CN.resx @@ -919,9 +919,6 @@ 如果启用此模式,保存文件将从可执行文件所在位置加载 - - 文件读取算法 - 设置 diff --git a/src/LogExpert.Tests/Buffers/BufferShiftTest.cs b/src/LogExpert.Tests/Buffers/BufferShiftTest.cs index 8233da59..bf34f19c 100644 --- a/src/LogExpert.Tests/Buffers/BufferShiftTest.cs +++ b/src/LogExpert.Tests/Buffers/BufferShiftTest.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.PluginRegistry.FileSystem; using NUnit.Framework; @@ -25,13 +24,11 @@ public void Boot () } [Test] - [TestCase(ReaderType.System)] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Test")] - //[TestCase(ReaderType.Legacy)] Legacy Reader does not Support this //TO Test real life scenario, use the LogRotator tool, in the src/Tools/LogRotator directory, //to create files and perform rollovers while watching the files in LogExpert with MultiFile enabled //(pattern: *$J(.)) - public void TestShiftBuffers1 (ReaderType readerType) + public void TestShiftBuffers1 () { var linesPerFile = 10; MultiFileOptions options = new() @@ -48,7 +45,7 @@ public void TestShiftBuffers1 (ReaderType readerType) }; _ = PluginRegistry.PluginRegistry.Create(TestDirectory.FullName, 500); - LogfileReader reader = new(files.Last.Value, encodingOptions, true, 40, 50, options, readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(files.Last.Value, encodingOptions, true, 40, 50, options, PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); var lil = reader.GetLogFileInfoList(); diff --git a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs index 0e03e196..106a74ae 100644 --- a/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/CSVColumnizerTest.cs @@ -6,7 +6,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using Moq; @@ -282,14 +281,14 @@ public void SplitLine_BadCsvData_ReturnsSingleColumnInsteadOfCrash () Assert.That(result.ColumnValues[0].FullValue.ToString(), Does.Contain("6774DC1dB00BD11")); } - [TestCase(@".\TestData\organizations-10000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" }, ReaderType.System)] - [TestCase(@".\TestData\organizations-1000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" }, ReaderType.System)] - [TestCase(@".\TestData\people-10000.csv", new[] { "Index", "User Id", "First Name", "Last Name", "Sex", "Email", "Phone", "Date of birth", "Job Title" }, ReaderType.System)] - public void Instantiat_CSVFile_BuildCorrectColumnizer (string filename, string[] expectedHeaders, ReaderType readerType) + [TestCase(@".\TestData\organizations-10000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" })] + [TestCase(@".\TestData\organizations-1000.csv", new[] { "Index", "Organization Id", "Name", "Website", "Country", "Description", "Founded", "Industry", "Number of employees" })] + [TestCase(@".\TestData\people-10000.csv", new[] { "Index", "User Id", "First Name", "Last Name", "Sex", "Email", "Phone", "Date of birth", "Job Title" })] + public void Instantiat_CSVFile_BuildCorrectColumnizer (string filename, string[] expectedHeaders) { CsvColumnizer.CsvColumnizer csvColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, filename); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); var line = reader.GetLogLineMemory(0); IColumnizedLogLineMemory logline = new ColumnizedLogLine(); @@ -309,7 +308,7 @@ public void LogfileReader_SemicolonCsv_ReadsAllLines () { // semicolon.csv: CRLF between lines, no trailing newline on last line var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\semicolon.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); Assert.That(reader.LineCount, Is.EqualTo(2), $"semicolon.csv should have 2 lines, got {reader.LineCount}"); @@ -320,7 +319,7 @@ public void LogfileReader_TabCsv_ReadsAllLines () { // tab.csv: header + 1 data line, CRLF between lines var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\tab.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); Assert.That(reader.LineCount, Is.EqualTo(2), $"tab.csv should have 2 lines, got {reader.LineCount}"); @@ -331,7 +330,7 @@ public void LogfileReader_CsvTest01_ReadsAllLines () { // CsvTest_01.csv: CR line endings, no trailing newline on last line var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\CsvTest_01.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); Assert.That(reader.LineCount, Is.EqualTo(31), $"CsvTest_01.csv should have 31 lines (header + 30 data), got {reader.LineCount}"); @@ -343,7 +342,7 @@ public void LogfileReader_SemicolonCsv_WithPreProcess_DataLinesVisible () // With CsvColumnizer as PreProcessColumnizer, header should be dropped, data line should remain CsvColumnizer.CsvColumnizer csvColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\semicolon.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -360,7 +359,7 @@ public void LogfileReader_TabCsv_WithPreProcess_DataLinesVisible () { CsvColumnizer.CsvColumnizer csvColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\tab.csv"); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -382,7 +381,7 @@ public void LogfileReader_SemicolonCsv_SimulateGuiReload_DataLinesVisible () CsvColumnizer.CsvColumnizer csvColumnizer = new(); // The GUI Reload creates a brand new LogfileReader with PreProcess already set - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -416,7 +415,7 @@ public void LogfileReader_SemicolonCsv_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -450,7 +449,7 @@ public void LogfileReader_CsvTest01_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -483,7 +482,7 @@ public void LogfileReader_TabCsv_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -505,7 +504,7 @@ public void LogfileReader_CommaCsv_StartMonitoring_LoadsCorrectly () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -550,7 +549,7 @@ public void LogfileReader_SemicolonCsv_SplitLine_ReturnsCorrectValues () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -595,7 +594,7 @@ public void LogfileReader_CommaCsv_ReloadWithPreProcess_DataLineNotEmpty () using ManualResetEventSlim loadingDone = new(false); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); // NO PreProcessColumnizer set initially reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -609,7 +608,7 @@ public void LogfileReader_CommaCsv_ReloadWithPreProcess_DataLineNotEmpty () reader.StopMonitoring(); using ManualResetEventSlim reloadDone = new(false); - LogfileReader reader2 = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader2 = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader2.PreProcessColumnizer = csvColumnizer; reader2.LoadingFinished += (_, _) => reloadDone.Set(); reader2.StartMonitoring(); @@ -658,7 +657,7 @@ public void LogfileReader_CommaCsv_SingleFile_WithPreProcess_DataLineNotEmpty () using ManualResetEventSlim loadingDone = new(false); // multiFile=FALSE, like the real GUI - LogfileReader reader = new(path, new EncodingOptions(), false, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), false, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.LoadingFinished += (_, _) => loadingDone.Set(); reader.StartMonitoring(); @@ -695,7 +694,7 @@ public void LogfileReader_CsvNoTrailingNewline_HeaderDropped_DataLineVisible () CsvColumnizer.CsvColumnizer csvColumnizer = new(); - LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); @@ -727,7 +726,7 @@ public void LogfileReader_CsvWithTrailingNewline_HeaderDropped_DataLineVisible ( CsvColumnizer.CsvColumnizer csvColumnizer = new(); - LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(tempFile, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.PreProcessColumnizer = csvColumnizer; reader.ReadFiles(); diff --git a/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs b/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs index 3d69741e..975806e0 100644 --- a/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/ColumnizerPickerTest.cs @@ -5,7 +5,6 @@ using LogExpert.Core.Classes.Columnizer; using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using Moq; @@ -105,14 +104,14 @@ public void FindColumnizer_ReturnCorrectColumnizer (string expectedColumnizerNam Assert.That(result.GetName(), Is.EqualTo(expectedColumnizerName)); } - [TestCase(@".\TestData\JsonColumnizerTest_01.txt", typeof(JsonCompactColumnizer.JsonCompactColumnizer), ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", typeof(SquareBracketColumnizer), ReaderType.System)] - public void FindReplacementForAutoColumnizer_ValidTextFile_ReturnCorrectColumnizer (string fileName, Type columnizerType, ReaderType readerType) + [TestCase(@".\TestData\JsonColumnizerTest_01.txt", typeof(JsonCompactColumnizer.JsonCompactColumnizer))] + [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", typeof(SquareBracketColumnizer))] + public void FindReplacementForAutoColumnizer_ValidTextFile_ReturnCorrectColumnizer (string fileName, Type columnizerType) { var pluginRegistry = PluginRegistry.PluginRegistry.Instance; var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, pluginRegistry, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), pluginRegistry, 500); reader.ReadFiles(); Mock autoColumnizer = new(); diff --git a/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs index fc344da5..d6b3dacc 100644 --- a/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/JsonColumnizerTest.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -44,12 +43,12 @@ private static void ResetPluginRegistrySingleton () instanceField?.SetValue(null, null); } - [TestCase(@".\TestData\JsonColumnizerTest_01.txt", "time @m level", ReaderType.System)] - public void GetColumnNames_HappyFile_ColumnNameMatches (string fileName, string expectedHeaders, ReaderType readerType) + [TestCase(@".\TestData\JsonColumnizerTest_01.txt", "time @m level")] + public void GetColumnNames_HappyFile_ColumnNameMatches (string fileName, string expectedHeaders) { var jsonColumnizer = new JsonColumnizer.JsonColumnizer(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader reader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); reader.ReadFiles(); var line = reader.GetLogLineMemory(0); diff --git a/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs index e16118ab..21f912c5 100644 --- a/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/JsonCompactColumnizerTest.cs @@ -4,7 +4,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -47,14 +46,14 @@ private static void ResetPluginRegistrySingleton () } // As long as the json file contains one of the pre-defined key, it's perfectly supported. - [TestCase(@".\TestData\JsonCompactColumnizerTest_01.json", Priority.PerfectlySupport, ReaderType.System)] - [TestCase(@".\TestData\JsonCompactColumnizerTest_02.json", Priority.PerfectlySupport, ReaderType.System)] - [TestCase(@".\TestData\JsonCompactColumnizerTest_03.json", Priority.WellSupport, ReaderType.System)] - public void GetPriority_HappyFile_PriorityMatches (string fileName, Priority priority, ReaderType readerType) + [TestCase(@".\TestData\JsonCompactColumnizerTest_01.json", Priority.PerfectlySupport)] + [TestCase(@".\TestData\JsonCompactColumnizerTest_02.json", Priority.PerfectlySupport)] + [TestCase(@".\TestData\JsonCompactColumnizerTest_03.json", Priority.WellSupport)] + public void GetPriority_HappyFile_PriorityMatches (string fileName, Priority priority) { var jsonCompactColumnizer = new JsonCompactColumnizer.JsonCompactColumnizer(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); logFileReader.ReadFiles(); List loglines = [ diff --git a/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs b/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs index d1112a93..2285af0e 100644 --- a/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs +++ b/src/LogExpert.Tests/ColumnizerTests/SquareBracketColumnizerTest.cs @@ -5,7 +5,6 @@ using LogExpert.Core.Classes.Columnizer; using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -47,16 +46,16 @@ private static void ResetPluginRegistrySingleton () instanceField?.SetValue(null, null); } - [TestCase(@".\TestData\SquareBracketColumnizerTest_01.txt", 5, ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", 5, ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_03.txt", 6, ReaderType.System)] - [TestCase(@".\TestData\SquareBracketColumnizerTest_05.txt", 3, ReaderType.System)] - public void GetPriority_HappyFile_ColumnCountMatches (string fileName, int count, ReaderType readerType) + [TestCase(@".\TestData\SquareBracketColumnizerTest_01.txt", 5)] + [TestCase(@".\TestData\SquareBracketColumnizerTest_02.txt", 5)] + [TestCase(@".\TestData\SquareBracketColumnizerTest_03.txt", 6)] + [TestCase(@".\TestData\SquareBracketColumnizerTest_05.txt", 3)] + public void GetPriority_HappyFile_ColumnCountMatches (string fileName, int count) { SquareBracketColumnizer squareBracketColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, fileName); - LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), readerType, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); logFileReader.ReadFiles(); List loglines = [ @@ -85,7 +84,7 @@ public void SplitLine_NestedBrackets_KeepsInnerBracketsInOneColumn () SquareBracketColumnizer squareBracketColumnizer = new(); var path = Path.Join(AppDomain.CurrentDomain.BaseDirectory, @".\TestData\SquareBracketColumnizerTest_06.txt"); - LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500); + LogfileReader logFileReader = new(path, new EncodingOptions(), true, 40, 50, new MultiFileOptions(), PluginRegistry.PluginRegistry.Instance, 500); logFileReader.ReadFiles(); List loglines = diff --git a/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs b/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs index a6ffa31e..dca26bb4 100644 --- a/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs +++ b/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs @@ -437,6 +437,40 @@ public void LoadOrCreateNew_ValidFile_LoadsSuccessfully () Assert.That(loadResult.Settings.FilterList[0].SearchText, Is.EqualTo("VALID_FILTER_TEST")); } + /// + /// ADR 0010 removed the reader choice. Settings saved by earlier versions still carry + /// "ReaderType"; they must load as they are, with everything else intact, and not fall + /// back to the backup or to defaults. + /// + [Test] + [Category("DeserializationRecovery")] + [TestCase("SystemDirect")] + [TestCase("System")] + [TestCase("Legacy")] + public void LoadOrCreateNew_RemovedReaderTypeSetting_LoadsTheRestOfTheSettings (string storedReaderType) + { + // Arrange + Settings settings = CreatePopulatedSettings(); + settings.FilterList.Clear(); + settings.FilterList.Add(new FilterParams { SearchText = "KEPT_FILTER" }); + settings.Preferences.MaxLineLength = 12_345; + var json = Newtonsoft.Json.Linq.JObject.Parse(JsonConvert.SerializeObject(settings)); + json["Preferences"]!["ReaderType"] = storedReaderType; + File.WriteAllText(_testSettingsFile.FullName, json.ToString()); + + // Act + LoadResult loadResult = InvokePrivateInstanceMethod("LoadOrCreateNew", _testSettingsFile); + + // Assert + Assert.Multiple(() => + { + Assert.That(loadResult.CriticalFailure, Is.False); + Assert.That(loadResult.LoadedFromBackup, Is.False); + Assert.That(loadResult.Settings.FilterList.Select(filter => filter.SearchText), Is.EqualTo(new[] { "KEPT_FILTER" })); + Assert.That(loadResult.Settings.Preferences.MaxLineLength, Is.EqualTo(12_345)); + }); + } + [Test] [Category("DeserializationRecovery")] [Description("LoadOrCreateNew should handle missing file by creating new settings")] diff --git a/src/LogExpert.Tests/ReaderTest.cs b/src/LogExpert.Tests/ReaderTest.cs index f03feda5..b6dc04d5 100644 --- a/src/LogExpert.Tests/ReaderTest.cs +++ b/src/LogExpert.Tests/ReaderTest.cs @@ -1,8 +1,5 @@ -using System.Text; - using LogExpert.Core.Classes.Log.Streamreaders; using LogExpert.Core.Entities; -using LogExpert.Core.Interfaces; using NUnit.Framework; @@ -21,7 +18,7 @@ public void Boot () { } - #region GuessNewLineSequenceLength / TryReadLine tests + #region TryReadLine line endings [Test] public void TryReadLine_CrLf_NoTrailingNewline_ReadsAllLines () @@ -29,7 +26,7 @@ public void TryReadLine_CrLf_NoTrailingNewline_ReadsAllLines () // File with CRLF between lines but NO trailing newline on last line var content = "line1\r\nline2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -44,7 +41,7 @@ public void TryReadLine_CrLf_WithTrailingNewline_ReadsAllLines () // File with CRLF between lines AND trailing newline var content = "line1\r\nline2\r\n"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -59,7 +56,7 @@ public void TryReadLine_CrOnly_NoTrailingNewline_ReadsAllLines () // File with CR-only line endings (like old Mac), no trailing newline var content = "line1\rline2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -74,7 +71,7 @@ public void TryReadLine_LfOnly_NoTrailingNewline_ReadsAllLines () // File with LF-only line endings (Unix), no trailing newline var content = "line1\nline2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -89,7 +86,7 @@ public void TryReadLine_CrLf_ThreeLines_NoTrailingNewline_ReadsAllLines () // 3 lines with CRLF, no trailing newline var content = "header\r\ndata1\r\ndata2"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -105,7 +102,7 @@ public void TryReadLine_CrLf_Position_TracksCorrectly () // Verify position tracking for CRLF file var content = "AB\r\nCD\r\n"u8.ToArray(); // 4 + 4 = 8 bytes using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.ToString(), Is.EqualTo("AB")); @@ -124,7 +121,7 @@ public void TryReadLine_CrLf_NoTrailingNewline_Position_TracksCorrectly () // Verify position tracking for CRLF file without trailing newline var content = "AB\r\nCD"u8.ToArray(); // 4 + 2 = 6 bytes using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.ToString(), Is.EqualTo("AB")); @@ -132,10 +129,9 @@ public void TryReadLine_CrLf_NoTrailingNewline_Position_TracksCorrectly () Assert.That(reader.TryReadLine(out var line2), Is.True); Assert.That(line2.ToString(), Is.EqualTo("CD")); - // Last line has no newline, but current implementation adds _newLineSequenceLength anyway. - // BUG: position is 8 (adds 2 for nonexistent CRLF), should be 6. - // This causes incorrect seeking on buffer re-read. - Assert.That(reader.Position, Is.EqualTo(8), "Known issue: overcounts by newline length on last line without trailing newline"); + // The last line has no newline, so only its content bytes count (the removed System reader added + // a guessed newline length here, which broke seeking on buffer re-read). + Assert.That(reader.Position, Is.EqualTo(6), "After 'CD' at EOF, position should be 6"); } [Test] @@ -144,7 +140,7 @@ public void TryReadLine_SingleLine_NoNewline_ReadsLine () // Single line file with no newline at all var content = "onlyline"u8.ToArray(); using var stream = new MemoryStream(content); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lines = ReadAllLines(reader); @@ -152,7 +148,7 @@ public void TryReadLine_SingleLine_NoNewline_ReadsLine () Assert.That(lines[0], Is.EqualTo("onlyline")); } - private static List ReadAllLines (PositionAwareStreamReaderSystem reader) + private static List ReadAllLines (PositionAwareStreamReaderDirect reader) { List lines = []; while (reader.TryReadLine(out var lineMemory)) @@ -164,50 +160,4 @@ private static List ReadAllLines (PositionAwareStreamReaderSystem reader } #endregion - - //TODO reimplement - private void CompareReaderImplementationsInternal (string fileName, Encoding enc, int maxPosition) - { - var path = Environment.CurrentDirectory + "\\data\\"; - EncodingOptions encOpts = new() - { - Encoding = enc - }; - - using Stream s1 = new FileStream(path + fileName, FileMode.Open, FileAccess.Read); - using Stream s2 = new FileStream(path + fileName, FileMode.Open, FileAccess.Read); - using ILogStreamReader r1 = new PositionAwareStreamReaderLegacy(s1, encOpts, 500); - using ILogStreamReader r2 = new PositionAwareStreamReaderSystem(s2, encOpts, 500); - for (var lineNum = 0; ; lineNum++) - { - var line1 = r1.ReadLine(); - var line2 = r2.ReadLine(); - if (line1 == null && line2 == null) - { - break; - } - - Assert.That(line1, Is.EqualTo(line2), "File " + fileName); - - if (r1.Position != maxPosition) - { - Assert.That(r2.Position, Is.EqualTo(r1.Position), "Line " + lineNum + ", File: " + fileName); - } - else - { - //Its desired that the position of the new implementation is 2 bytes ahead to fix the problem of empty lines every time a new line is appended. - Assert.That(r2.Position - 2, Is.EqualTo(r1.Position), "Line " + lineNum + ", File: " + fileName); - } - } - } - - //TODO find out why it does not work with appveyor, but works fine with normal environment! - //[Test] - //[TestCase("50 MB.txt", "Windows-1252", 50000000)] - //[TestCase("50 MB UTF16.txt", "Unicode", 49999998)] - //[TestCase("50 MB UTF8.txt", "UTF-8", 50000000)] - //public void CompareReaderImplementations(string fileName, string encoding, int maxPosition) - //{ - // CompareReaderImplementationsInternal(fileName, Encoding.GetEncoding(encoding), maxPosition); - //} } \ No newline at end of file diff --git a/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs index cf319fb0..87e1e347 100644 --- a/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs @@ -248,7 +248,7 @@ private static byte[] TextWithBinaryBlock (int lineCount, int binaryLength, int #region Oracle - private static void AssertMatchesOracle (byte[] data, Encoding encoding) + internal static void AssertMatchesOracle (byte[] data, Encoding encoding) { var expected = ExpectedLines(data, encoding); var actual = ReadAll(data, encoding); diff --git a/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs b/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs index 37ee01e3..84b343b4 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogStreamReaderTest.cs @@ -3,7 +3,6 @@ using LogExpert.Core.Classes.Log.Streamreaders; using LogExpert.Core.Entities; using LogExpert.Core.Helpers; -using LogExpert.Core.Interfaces; using NUnit.Framework; @@ -19,10 +18,10 @@ public class LogStreamReaderTest [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n", 3)] [TestCase("Line 1\rLine 2\rLine 3", 3)] [TestCase("Line 1\rLine 2\rLine 3\r", 3)] - public void ReadLinesWithSystemNewLine (string text, int expectedLines) + public void ReadLine_AllLineEndings_ReadsAllLines (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (true) { @@ -44,10 +43,10 @@ public void ReadLinesWithSystemNewLine (string text, int expectedLines) [TestCase("\n\n\n", 3)] [TestCase("\r\n\r\n\r\n", 3)] [TestCase("\r\r\r", 3)] - public void CountLinesWithSystemNewLine (string text, int expectedLines) + public void ReadLine_OnlyTerminators_CountsEmptyLines (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (reader.ReadLine() != null) { @@ -64,54 +63,10 @@ public void CountLinesWithSystemNewLine (string text, int expectedLines) [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n", 3)] [TestCase("Line 1\rLine 2\rLine 3", 3)] [TestCase("Line 1\rLine 2\rLine 3\r", 3)] - public void ReadLinesWithLegacyNewLine (string text, int expectedLines) + public void TryReadLine_ReturnsCorrectContent (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 500); - var lineCount = 0; - while (true) - { - var line = reader.ReadLine(); - if (line == null) - { - break; - } - - lineCount += 1; - - Assert.That(line.StartsWith($"Line {lineCount}", StringComparison.OrdinalIgnoreCase), $"Invalid line: {line}"); - } - - Assert.That(expectedLines, Is.EqualTo(lineCount), $"Unexpected lines:\n{text}"); - } - [Test] - [TestCase("\n\n\n", 3)] - [TestCase("\r\n\r\n\r\n", 3)] - [TestCase("\r\r\r", 3)] - public void CountLinesWithLegacyNewLine (string text, int expectedLines) - { - using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 500); - var lineCount = 0; - while (reader.ReadLine() != null) - { - lineCount += 1; - } - - Assert.That(expectedLines, Is.EqualTo(lineCount), $"Unexpected lines:\n{text}"); - } - - [Test] - [TestCase("Line 1\nLine 2\nLine 3", 3)] - [TestCase("Line 1\nLine 2\nLine 3\n", 3)] - [TestCase("Line 1\r\nLine 2\r\nLine 3", 3)] - [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n", 3)] - [TestCase("Line 1\rLine 2\rLine 3", 3)] - [TestCase("Line 1\rLine 2\rLine 3\r", 3)] - public void TryReadLine_ReturnsCorrectContent_SystemReader (string text, int expectedLines) - { - using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (reader.TryReadLine(out var lineMemory)) { @@ -127,10 +82,10 @@ public void TryReadLine_ReturnsCorrectContent_SystemReader (string text, int exp [TestCase("\n\n\n", 3)] [TestCase("\r\n\r\n\r\n", 3)] [TestCase("\r\r\r", 3)] - public void TryReadLine_CountsEmptyLines_SystemReader (string text, int expectedLines) + public void TryReadLine_CountsEmptyLines (string text, int expectedLines) { using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); var lineCount = 0; while (reader.TryReadLine(out _)) { @@ -141,11 +96,11 @@ public void TryReadLine_CountsEmptyLines_SystemReader (string text, int expected } [Test] - public void TryReadLine_TracksPositionCorrectly_SystemReader () + public void TryReadLine_TracksPositionCorrectly () { var text = "Line 1\nLine 2\nLine 3\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.Span.ToString(), Is.EqualTo("Line 1")); @@ -167,7 +122,7 @@ public void TryReadLine_ReturnsBlockBackedMemory_NotStringBacked () { var text = "Hello World\nSecond Line\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var lineMemory), Is.True); @@ -183,7 +138,7 @@ public void TryReadLine_DetachCharBlocks_ReturnsFilledBlocks_AfterReading () { var text = "Line 1\nLine 2\nLine 3\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); while (reader.TryReadLine(out _)) { @@ -199,7 +154,7 @@ public void TryReadLine_DetachCharBlocks_TransfersOwnership () { var text = "Line 1\nLine 2\nLine 3\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); while (reader.TryReadLine(out _)) { @@ -222,8 +177,8 @@ public void TryReadLine_MatchesReadLine_ContentAndPosition () using var stream1 = new MemoryStream(Encoding.UTF8.GetBytes(text)); using var stream2 = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var readLineReader = new PositionAwareStreamReaderSystem(stream1, new EncodingOptions(), 500); - using var tryReadLineReader = new PositionAwareStreamReaderSystem(stream2, new EncodingOptions(), 500); + using var readLineReader = new PositionAwareStreamReaderDirect(stream1, new EncodingOptions(), 500); + using var tryReadLineReader = new PositionAwareStreamReaderDirect(stream2, new EncodingOptions(), 500); while (true) { @@ -248,7 +203,7 @@ public void TryReadLine_RespectsMaximumLineLength () var longLine = new string('X', 1000); var text = $"{longLine}\nShort\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions(), 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions(), 500); Assert.That(reader.TryReadLine(out var lineMemory), Is.True); Assert.That(lineMemory.Length, Is.EqualTo(500), "Line should be truncated to MaximumLineLength"); @@ -263,7 +218,7 @@ public void TryReadLine_UTF8_MultiByteCharacters () // Japanese characters: 3 bytes each in UTF-8 var text = "日本語テスト\nLine 2\n"; using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - using var reader = new PositionAwareStreamReaderSystem(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); Assert.That(reader.TryReadLine(out var line1), Is.True); Assert.That(line1.Span.ToString(), Is.EqualTo("日本語テスト")); @@ -273,45 +228,22 @@ public void TryReadLine_UTF8_MultiByteCharacters () Assert.That(line2.Span.ToString(), Is.EqualTo("Line 2")); } - [Test] - [TestCase("Line 1\nLine 2\nLine 3", 3)] - [TestCase("Line 1\r\nLine 2\r\nLine 3", 3)] - [TestCase("Line 1\rLine 2\rLine 3", 3)] - public void TryReadLine_LegacyReader_ReadsAllLines (string text, int expectedLines) - { - using var stream = new MemoryStream(Encoding.ASCII.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions(), 500); - - var memoryReader = reader as ILogStreamReaderMemory; - Assert.That(memoryReader, Is.Not.Null, "Legacy reader must implement ILogStreamReaderMemory"); - - var lineCount = 0; - while (memoryReader!.TryReadLine(out var lineMemory)) - { - lineCount++; - Assert.That(lineMemory.Span.ToString(), Does.StartWith($"Line {lineCount}")); - } - - Assert.That(lineCount, Is.EqualTo(expectedLines)); - } - /// - /// The legacy reader advances its position per character read, by a per-encoding step - /// (GetPosIncPrecomputed). GB2312 (issue #688) is the first offered encoding that is neither - /// single-byte nor Unicode — one byte per ASCII character, two per Chinese one — so a step of "1 - /// byte unless UTF-8 or UTF-16" drifted the position on the first Chinese character and every - /// subsequent line started at the wrong offset. + /// GB2312 (issue #688) is the first offered encoding that is neither single-byte nor Unicode — one + /// byte per ASCII character, two per Chinese one — so a position that assumed a fixed width per + /// character drifted on the first Chinese character and every subsequent line started at the wrong + /// offset. /// [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] - public void TryReadLine_LegacyReader_Gb2312_TracksTheBytePositionPerLine () + public void TryReadLine_Gb2312_TracksTheBytePositionPerLine () { var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); string[] lines = ["错误: 连接失败", "INFO 启动完成", "plain ascii", "警告"]; var text = string.Join("\n", lines) + "\n"; using var stream = new MemoryStream(gb2312.GetBytes(text)); - using var reader = new PositionAwareStreamReaderLegacy(stream, new EncodingOptions { Encoding = gb2312 }, 500); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = gb2312 }, 500); var expectedPosition = 0; diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs index 6bd1ae97..1040c97d 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderBlockAllocationTests.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -45,7 +44,6 @@ public void ReadFiles_AllLinesCorrect_WithBlockAllocation (int lineCount) bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -76,7 +74,6 @@ public void ReadFiles_AllLinesCorrect_WithDirectRead (int lineCount) bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.SystemDirect, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -103,7 +100,6 @@ public void ReadFiles_EmptyLines_PreservedCorrectly () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -133,7 +129,6 @@ public void ReadFiles_UTF8MultiByte_ContentPreserved () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -162,7 +157,6 @@ public void ReadFiles_LongLine_TruncatedToMaxLength () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -185,7 +179,6 @@ public void ReadFiles_LineContentBackedByPooledMemory () bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); @@ -213,7 +206,6 @@ public void ReadFiles_MultipleBufferRotations_AllLinesCorrect () bufferCount: 100, linesPerBuffer: 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: Core.Classes.Log.ProgressReporters.NullProgressReporter.Instance); diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs index 9d791bf4..87dc2087 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderEncodingTests.cs @@ -3,7 +3,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Classes.Log.ProgressReporters; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.Helpers; using NUnit.Framework; @@ -156,7 +155,6 @@ private LogfileReader CreateReader (EncodingOptions encodingOptions) bufferCount: 100, linesPerBuffer: 500, new MultiFileOptions(), - ReaderType.SystemDirect, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: NullProgressReporter.Instance); diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs index 00e36f73..39d62052 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderMultiFileFlagTests.cs @@ -3,7 +3,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Classes.Log.ProgressReporters; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using NUnit.Framework; @@ -95,7 +94,6 @@ public void MultiFileCtor_AlwaysMultiFile () bufferCount: 40, linesPerBuffer: 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: NullProgressReporter.Instance); @@ -118,7 +116,6 @@ private LogfileReader CreateSingleFileReader (bool multiFile, MultiFileOptions? bufferCount: 40, linesPerBuffer: 50, options ?? new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: NullProgressReporter.Instance); diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs index 0235b954..4bd0657b 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs @@ -2,7 +2,6 @@ using LogExpert.Core.Classes.Log; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.Interfaces; using Moq; @@ -57,7 +56,6 @@ public void SingleFileReader_OnTruncation_ReportsNewFileNotRollover () bufferCount: 40, linesPerBuffer: 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: progressReporterMock.Object); @@ -111,7 +109,6 @@ public void SingleFileReader_GrowingFileWithBinaryBlock_TakesGrowthPathNotNewFil bufferCount: 40, linesPerBuffer: 50, new MultiFileOptions(), - ReaderType.SystemDirect, PluginRegistry.PluginRegistry.Instance, maximumLineLength: 500, progressReporter: progressReporterMock.Object); diff --git a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs index cafbea20..b203af27 100644 --- a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs @@ -159,117 +159,76 @@ public void TryReadLine_LineCrossesBlockBoundary_Correct () #endregion - #region Position parity with System reader + #region Position matches the raw bytes [Test] [TestCase("Line 1\nLine 2\nLine 3\n")] [TestCase("Line 1\r\nLine 2\r\nLine 3\r\n")] [TestCase("Short\nMedium line here\nA much longer line with more content\n")] - public void Position_MatchesSystemReader_UTF8 (string text) + public void Position_MatchesRawBytes_UTF8 (string text) { - ComparePositions(text, Encoding.UTF8); + DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.UTF8.GetBytes(text), Encoding.UTF8); } [Test] [TestCase("Line 1\nLine 2\nLine 3\n")] - public void Position_MatchesSystemReader_ASCII (string text) + public void Position_MatchesRawBytes_ASCII (string text) { - ComparePositions(text, Encoding.ASCII); + DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.ASCII.GetBytes(text), Encoding.ASCII); } [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] - public void Position_MatchesSystemReader_UTF8_MultibyteChars () + public void Position_MatchesRawBytes_UTF8_MultibyteChars () { var text = "Héllo wörld\nCafé résumé\nこんにちは\n"; - ComparePositions(text, Encoding.UTF8); + DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.UTF8.GetBytes(text), Encoding.UTF8); } /// /// GB2312 (issue #688) is the first offered encoding that is neither single-byte nor Unicode: two - /// bytes per Chinese character, one per ASCII character, mixed within a line. The direct reader - /// advances its byte position by Encoding.GetByteCount over the decoded chars, so a wrong - /// assumption there would drift the position from the second line on — and every buffer boundary, + /// bytes per Chinese character, one per ASCII character, mixed within a line. A wrong assumption + /// about the width would drift the position from the second line on — and every buffer boundary, /// rollover check and .lxp scroll position is computed from it. /// [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] - public void Position_MatchesSystemReader_Gb2312_MixedAsciiAndChinese () + public void Position_MatchesRawBytes_Gb2312_MixedAsciiAndChinese () { + var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); var text = "2026-07-30 错误: 连接失败\nINFO 启动完成\nplain ascii line\n警告\n"; - ComparePositions(text, EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312)); - } - - private static void ComparePositions (string text, Encoding encoding) - { - var bytes = encoding.GetBytes(text); - - using var s1 = new MemoryStream(bytes); - using var s2 = new MemoryStream(bytes); - using var systemReader = new PositionAwareStreamReaderSystem(s1, new EncodingOptions { Encoding = encoding }, 500); - using var directReader = new PositionAwareStreamReaderDirect(s2, new EncodingOptions { Encoding = encoding }, 500); - - var lineNum = 0; - while (true) - { - var systemHasLine = systemReader.TryReadLine(out var systemLine); - var directHasLine = directReader.TryReadLine(out var directLine); - - Assert.That(directHasLine, Is.EqualTo(systemHasLine), $"Line {lineNum}: EOF mismatch"); - - if (!systemHasLine) - { - break; - } - - Assert.That(directLine.Span.ToString(), Is.EqualTo(systemLine.Span.ToString()), - $"Line {lineNum}: content mismatch"); - Assert.That(directReader.Position, Is.EqualTo(systemReader.Position), - $"Line {lineNum}: position mismatch (System={systemReader.Position}, Direct={directReader.Position})"); - - lineNum++; - } + DirectReaderExactPositionTests.AssertMatchesOracle(gb2312.GetBytes(text), gb2312); } #endregion - #region Content parity with System reader + #region Content of a large file [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1305:Specify IFormatProvider", Justification = "Unit Tests")] - public void ContentParity_LargeFile () + public void TryReadLine_LargeFile_ReturnsEveryLine () { + var expected = new List(); var sb = new StringBuilder(); var rng = new Random(42); for (var i = 0; i < 10_000; i++) { - _ = sb.Append("2026-04-23 12:00:00."); - _ = sb.Append(i % 1000); - _ = sb.Append(" [INFO] Thread-"); - _ = sb.Append(rng.Next(1, 32)); - _ = sb.Append(" SomeClass - Log message number "); - _ = sb.AppendLine(i.ToString()); + var line = $"2026-04-23 12:00:00.{i % 1000} [INFO] Thread-{rng.Next(1, 32)} SomeClass - Log message number {i}"; + expected.Add(line); + _ = sb.AppendLine(line); } - var text = sb.ToString(); - var bytes = Encoding.UTF8.GetBytes(text); - - using var s1 = new MemoryStream(bytes); - using var s2 = new MemoryStream(bytes); - using var systemReader = new PositionAwareStreamReaderSystem(s1, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); - using var directReader = new PositionAwareStreamReaderDirect(s2, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); + using var stream = CreateStream(sb.ToString()); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, 500); - var lineNum = 0; - while (systemReader.TryReadLine(out var systemLine)) + var actual = new List(); + while (reader.TryReadLine(out var line)) { - Assert.That(directReader.TryReadLine(out var directLine), Is.True, $"Direct reader ended early at line {lineNum}"); - Assert.That(directLine.Span.ToString(), Is.EqualTo(systemLine.Span.ToString()), $"Line {lineNum}: content mismatch"); - lineNum++; + actual.Add(line.Span.ToString()); } - Assert.That(directReader.TryReadLine(out _), Is.False, "Direct reader has extra lines"); - Assert.That(lineNum, Is.EqualTo(10_000)); + Assert.That(actual, Is.EqualTo(expected)); } #endregion @@ -511,51 +470,14 @@ public void TryReadLine_LineMultipleTimesBlockSize_ReadsCorrectly () } [Test] - public void TryReadLine_LineLongerThanBlockSize_PositionParityWithSystemReader () + public void TryReadLine_LineLongerThanBlockSize_PositionMatchesRawBytes () { RunWithTimeout(() => { - // Verify byte positions match between Direct and System readers for long lines const int lineLength = 50_000; var text = $"short\n{new string('Q', lineLength)}\nend\n"; - var bytes = Encoding.UTF8.GetBytes(text); - - using var s1 = new MemoryStream(bytes); - using var s2 = new MemoryStream(bytes); - // Note: System reader has a pre-existing bug in CharBlockAllocator.ReturnAll() for - // oversized lines (allocates with new[] but tries to return to ArrayPool). Suppress - // disposal to avoid that unrelated failure. - var systemReader = new PositionAwareStreamReaderSystem(s1, new EncodingOptions { Encoding = Encoding.UTF8 }, lineLength + 100); - using var directReader = new PositionAwareStreamReaderDirect(s2, new EncodingOptions { Encoding = Encoding.UTF8 }, lineLength + 100); - - try - { - var lineNum = 0; - while (true) - { - var systemHasLine = systemReader.TryReadLine(out var systemLine); - var directHasLine = directReader.TryReadLine(out var directLine); - - Assert.That(directHasLine, Is.EqualTo(systemHasLine), $"Line {lineNum}: EOF mismatch"); - if (!systemHasLine) - { - break; - } - - Assert.That(directLine.Span.ToString(), Is.EqualTo(systemLine.Span.ToString()), $"Line {lineNum}: content mismatch"); - Assert.That(directReader.Position, Is.EqualTo(systemReader.Position), $"Line {lineNum}: position mismatch (System={systemReader.Position}, Direct={directReader.Position})"); - - lineNum++; - } - Assert.That(lineNum, Is.EqualTo(3)); - } - finally - { - // Don't call Dispose on systemReader — CharBlockAllocator has a pre-existing - // bug returning oversized (new[]) arrays to ArrayPool. - // The GC will reclaim the memory. - } + DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.UTF8.GetBytes(text), Encoding.UTF8); }); } diff --git a/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs b/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs index a954e6f7..21602d54 100644 --- a/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs +++ b/src/LogExpert.UI.Tests/Services/FileOperationServiceTests.cs @@ -6,7 +6,6 @@ using LogExpert.Core.Classes.Log.ProgressReporters; using LogExpert.Core.Config; using LogExpert.Core.Entities; -using LogExpert.Core.Enums; using LogExpert.Core.Interfaces; using LogExpert.UI.Controls.LogWindow; using LogExpert.UI.Interface; @@ -527,7 +526,7 @@ public void LoadDroppedFiles_CombinedChoice_ReadsAccessibleSelectedFiles (int un // Exercise the same reader endpoint used by LogWindow.LoadFilesAsMulti. using var reader = new LogfileReader(e.MultiFileNames, new EncodingOptions { Encoding = Encoding.UTF8 }, 40, 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); + PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); reader.ReadFiles(); lineCount = reader.GetLogLineMemories(0, 10).Length; }; @@ -587,7 +586,7 @@ public void LoadDroppedFiles_CombinedSessionAndLog_DoesNotOpenUnselectedRotation { using var reader = new LogfileReader(e.MultiFileNames!, new EncodingOptions { Encoding = Encoding.UTF8 }, 40, 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); + PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); reader.ReadFiles(); observed = reader.GetLogLineMemories(0, 10).Select(line => line.FullLine.ToString()).ToArray(); }; @@ -620,7 +619,7 @@ public void LoadDroppedFiles_CombinedSelectionAfterTruncation_KeepsOnlySelectedP { using var reader = new LogfileReader(e.MultiFileNames!, new EncodingOptions { Encoding = Encoding.UTF8 }, 40, 50, new MultiFileOptions(), - ReaderType.System, PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); + PluginRegistry.PluginRegistry.Instance, 500, NullProgressReporter.Instance, useExplicitFileList: true); reader.ReadFiles(); File.WriteAllText(second, "short\n"); _ = reader.ShiftBuffers(); diff --git a/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs b/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs index 56174ee8..2ee38b6d 100644 --- a/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs +++ b/src/LogExpert.UI/Controls/LogWindow/LogWindow.cs @@ -6071,7 +6071,7 @@ public void LoadFile (string fileName, EncodingOptions encodingOptions) try { - _logFileReader = new(fileName, EncodingOptions, IsMultiFile, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, Preferences.ReaderType, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength); + _logFileReader = new(fileName, EncodingOptions, IsMultiFile, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength); } catch (LogFileException lfe) { @@ -6146,7 +6146,7 @@ private void StartMultiFileReader (string[] fileNames, EncodingOptions encodingO _columnCache = new ColumnCache(); _filterColumnCache = new ColumnCache(); - _logFileReader = new(fileNames, EncodingOptions, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, Preferences.ReaderType, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength, useExplicitFileList: true); + _logFileReader = new(fileNames, EncodingOptions, Preferences.BufferCount, Preferences.LinesPerBuffer, _multiFileOptions, _pluginRegistry, ConfigManager.Settings.Preferences.MaxLineLength, useExplicitFileList: true); RegisterLogFileReaderEvents(); _logFileReader.StartMonitoring(); diff --git a/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs b/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs index a404a21a..6d871e77 100644 --- a/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs +++ b/src/LogExpert.UI/Dialogs/SettingsDialog.Designer.cs @@ -163,7 +163,6 @@ private void InitializeComponent () checkBoxSaveSessions = new CheckBox(); tabPageMemory = new TabPage(); groupBoxCPUAndStuff = new GroupBox(); - comboBoxReaderType = new ComboBox(); checkBoxMultiThread = new CheckBox(); labelFilePollingInterval = new Label(); upDownPollingInterval = new NumericUpDown(); @@ -1917,7 +1916,6 @@ private void InitializeComponent () // // groupBoxCPUAndStuff // - groupBoxCPUAndStuff.Controls.Add(comboBoxReaderType); groupBoxCPUAndStuff.Controls.Add(checkBoxMultiThread); groupBoxCPUAndStuff.Controls.Add(labelFilePollingInterval); groupBoxCPUAndStuff.Controls.Add(upDownPollingInterval); @@ -1930,14 +1928,6 @@ private void InitializeComponent () groupBoxCPUAndStuff.TabStop = false; groupBoxCPUAndStuff.Text = "CPU and stuff"; // - // comboBoxReaderType - // - comboBoxReaderType.FormattingEnabled = true; - comboBoxReaderType.Location = new Point(14, 76); - comboBoxReaderType.Name = "comboBoxReaderType"; - comboBoxReaderType.Size = new Size(262, 23); - comboBoxReaderType.TabIndex = 10; - // // checkBoxMultiThread // checkBoxMultiThread.AutoSize = true; @@ -2366,7 +2356,6 @@ private void InitializeComponent () private System.Windows.Forms.Label labelMaxDisplayLength; private Label labelLanguage; private ComboBox comboBoxLanguage; - private ComboBox comboBoxReaderType; private DataGridViewCheckBoxColumn columnControlCharEnabled; private DataGridViewTextBoxColumn columnControlCharHex; private DataGridViewTextBoxColumn columnControlCharAbbr; diff --git a/src/LogExpert.UI/Dialogs/SettingsDialog.cs b/src/LogExpert.UI/Dialogs/SettingsDialog.cs index c04df562..87b61f40 100644 --- a/src/LogExpert.UI/Dialogs/SettingsDialog.cs +++ b/src/LogExpert.UI/Dialogs/SettingsDialog.cs @@ -286,7 +286,6 @@ private void FillDialog () FillMultifileSettings(); FillEncodingList(); FillLanguageList(); - FillReaderTypeList(); FillControlCharsTab(); FillMarkerBarTab(); @@ -314,19 +313,6 @@ private void FillDialog () checkBoxShowErrorMessageOnlyOneInstance.Checked = Preferences.ShowErrorMessageAllowOnlyOneInstances; } - private void FillReaderTypeList () - { - foreach (var readerType in Enum.GetValues()) - { - if (!comboBoxReaderType.Items.Contains(readerType)) - { - _ = comboBoxReaderType.Items.Add(readerType); - } - } - - comboBoxReaderType.SelectedItem = Preferences.ReaderType; - } - private void FillColorModeList () { foreach (var colorMode in Enum.GetValues().Where(cm => !comboBoxColorMode.Items.Contains(cm))) @@ -827,7 +813,6 @@ private void OnBtnOkClick (object sender, EventArgs e) Preferences.DefaultEncoding = comboBoxEncoding.SelectedItem is Encoding selectedEncoding ? selectedEncoding.HeaderName : FallbackEncoding.HeaderName; Preferences.DefaultLanguage = comboBoxLanguage.SelectedItem != null ? (comboBoxLanguage.SelectedItem as string) : CultureInfo.GetCultureInfo("en-US").Name; Preferences.ShowColumnFinder = checkBoxColumnFinder.Checked; - Preferences.ReaderType = comboBoxReaderType.SelectedItem != null ? (ReaderType)comboBoxReaderType.SelectedItem : ReaderType.SystemDirect; Preferences.MaximumFilterEntries = (int)upDownMaximumFilterEntries.Value; Preferences.MaximumFilterEntriesDisplayed = (int)upDownMaximumFilterEntriesDisplayed.Value; @@ -1400,7 +1385,6 @@ private Dictionary GetToolTipMap () { comboBoxEncoding, Resources.SettingsDialog_UI_ComboBox_ToolTip_toolTipEncoding }, { checkBoxPortableMode, Resources.SettingsDialog_UI_CheckBox_ToolTip_toolTipPortableMode }, { radioButtonSessionApplicationStartupDir, Resources.SettingsDialog_UI_RadioButton_ToolTip_toolTipSessionApplicationStartupDir }, - { comboBoxReaderType, Resources.SettingsDialog_UI_CheckBox_ToolTip_toolTipReaderTyp }, { upDownMultifileDays, Resources.SettingsDialog_UI_NumericUpDown_ToolTip_upDownMultifileDays }, { upDownMultifileMaxFileAgeDays, Resources.SettingsDialog_UI_NumericUpDown_ToolTip_upDownMultifileMaxFileAgeDays } }; From 65608a3df7a2e1e38d770b25d767a4619658c396 Mon Sep 17 00:00:00 2001 From: BRUNER Patrick Date: Fri, 20 Nov 2026 15:13:47 +0100 Subject: [PATCH 3/3] Address review: comment rules, shared byte oracle, non-ASCII allocation case - Drop XML doc comments from private members and tests (docs/CODING_STANDARDS.md) and shorten the remaining comments. - Move the raw-byte oracle into RawByteLineOracle instead of one fixture calling into another. - Cover valid non-ASCII UTF-8 and UTF-16 in the per-line allocation test, which goes through the encoding's decoder instead of the ASCII fast path. - Rename _charFill to _usedChars and drop the big-endian UTF-16 branch (Windows only runs little-endian). --- .../DirectReaderBenchmarks.cs | 5 +- .../PositionAwareStreamReaderDirect.cs | 100 ++++++--------- .../ConfigManagerTests/ConfigManagerTest.cs | 6 +- .../DirectReaderExactPositionTests.cs | 119 +++--------------- .../LogfileReaderSingleFileMonitoringTests.cs | 6 +- .../PositionAwareStreamReaderDirectTests.cs | 23 ++-- .../StreamReaderTests/RawByteLineOracle.cs | 87 +++++++++++++ 7 files changed, 150 insertions(+), 196 deletions(-) create mode 100644 src/LogExpert.Tests/StreamReaderTests/RawByteLineOracle.cs diff --git a/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs b/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs index f577cd68..19c726c0 100644 --- a/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs +++ b/src/LogExpert.Benchmarks/DirectReaderBenchmarks.cs @@ -137,10 +137,7 @@ private static string GenerateMixedLineEndings (int lineCount) return sb.ToString(); } - /// - /// CSV-like text with a binary block every 1000 lines: NUL runs, stray continuation bytes and - /// truncated multi-byte sequences, the shape of the sample file from issue #639. - /// + // CSV-like text with a random binary block every 1000 lines, the shape of the issue #639 sample. [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Benchmark data generation")] private static byte[] GenerateInvalidBytes (int lineCount) { diff --git a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs index c90db3f8..bee4c10b 100644 --- a/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs +++ b/src/LogExpert.Core/Classes/Log/Streamreaders/PositionAwareStreamReaderDirect.cs @@ -12,23 +12,18 @@ namespace LogExpert.Core.Classes.Log.Streamreaders; /// ReadOnlyMemory<char> slices without allocating per-line strings. /// /// -/// The byte advances by the bytes a line really -/// occupies, so it stays exact for mixed line endings and for byte sequences that are invalid in the -/// encoding (issue #639): re-encoding decoded text would count each U+FFFD as the bytes of a replacement -/// character instead of the bytes it replaced. Line boundaries are searched as code units of the -/// encoding (one byte, or two or four for UTF-16 and UTF-32); in every encoding LogExpert offers, a -/// \r or \n unit never occurs inside a multi-byte character. -/// still reads through the base ; it and -/// must not be interleaved without setting the position in between. +/// advances by the raw bytes of each line, so it +/// stays exact for any line ending and for bytes that are invalid in the encoding (ADR 0010). +/// reads through the base ; do not interleave it with +/// without setting the position in between. /// public class PositionAwareStreamReaderDirect : PositionAwareStreamReaderBase, ILogStreamReaderMemory { #region Constants private const int BLOCK_SIZE = 32_768; // 64 KB (32K chars × 2 bytes), under LOH threshold - private const int BYTE_BUFFER_SIZE = 65_536; // raw bytes read from the stream per refill + private const int BYTE_BUFFER_SIZE = 65_536; private const int CODE_PAGE_UTF16_LE = 1200; - private const int CODE_PAGE_UTF16_BE = 1201; private static readonly SearchValues _surrogates = SearchValues.Create(Enumerable.Range(0xD800, 0x800).Select(code => (char)code).ToArray()); @@ -37,19 +32,19 @@ public class PositionAwareStreamReaderDirect : PositionAwareStreamReaderBase, IL #region Fields private char[] _readBlock; - private int _charFill; // chars of _readBlock already handed out as lines - private byte[] _byteBuffer; // rented on the first read + private int _usedChars; // chars of _readBlock already handed out as lines + private byte[] _byteBuffer; private int _byteStart; // first byte of the line in progress - private int _byteEnd; // end of the valid bytes in _byteBuffer + private int _byteEnd; private int _scannedBytes; // bytes after _byteStart already searched without a terminator private bool _eof; - private bool _initialized; // terminator units resolved for the encoding + private bool _initialized; private int _unitSize; // bytes per code unit: 1, 2 or 4 private uint _crUnit; private uint _lfUnit; - private bool _asciiCompatible; // bytes 0x00–0x7F decode to the same chars - private bool _nativeUtf16; // UTF-16 in the machine's byte order + private bool _asciiCompatible; + private bool _isUtf16LittleEndian; private readonly List _completedBlocks = []; @@ -109,8 +104,7 @@ public bool TryReadLine (out ReadOnlyMemory lineMemory) } else { - // \r is the last unit read so far but more data follows. Refill so the next unit - // becomes available, then re-scan from the \r to classify it. + // \r is the last unit read so far: refill, then re-scan from the \r to classify it. _scannedBytes = hitIndex; Refill(); continue; @@ -135,7 +129,7 @@ public bool TryReadLine (out ReadOnlyMemory lineMemory) return false; } - // Everything read so far is searched; only whole units count, a partial one is completed by the refill. + // Only whole units are searched; a partial one is completed by the refill. _scannedBytes = available - (available % _unitSize); Refill(); } @@ -152,11 +146,11 @@ public void ReturnMemory (ReadOnlyMemory memory) /// public List DetachCharBlocks () { - if (_charFill > 0) + if (_usedChars > 0) { _completedBlocks.Add(_readBlock); _readBlock = ArrayPool.Shared.Rent(BLOCK_SIZE); - _charFill = 0; + _usedChars = 0; } if (_completedBlocks.Count == 0) @@ -189,10 +183,8 @@ private void EnsureInitialized () } } - /// - /// Reads how \r and \n are encoded: one byte in single-byte and multi-byte code pages - /// and UTF-8, a 2-byte or 4-byte unit in UTF-16 and UTF-32 of either byte order. - /// + // In every encoding LogExpert offers, \r and \n are single code units (1 byte, or 2/4 for + // UTF-16/UTF-32) that never occur inside a multi-byte character, so lines can be split on bytes. private void ResolveTerminatorUnits () { var cr = Encoding.GetBytes("\r"); @@ -207,13 +199,10 @@ private void ResolveTerminatorUnits () _crUnit = ReadUnit(cr, _unitSize); _lfUnit = ReadUnit(lf, _unitSize); _asciiCompatible = _unitSize == 1 && DecodesAsciiUnchanged(Encoding); - _nativeUtf16 = Encoding.CodePage == (BitConverter.IsLittleEndian ? CODE_PAGE_UTF16_LE : CODE_PAGE_UTF16_BE); + _isUtf16LittleEndian = Encoding.CodePage == CODE_PAGE_UTF16_LE; } - /// - /// Whether every byte 0x00–0x7F decodes to the same char: true for UTF-8, ASCII, Latin-1, the - /// Windows code pages and GB2312; false for UTF-7 or EBCDIC, which take the encoding's decoder. - /// + // False for encodings like UTF-7 or EBCDIC, whose 0x00–0x7F bytes are not plain ASCII. private static bool DecodesAsciiUnchanged (Encoding encoding) { Span ascii = stackalloc byte[128]; @@ -257,10 +246,6 @@ private uint UnitAt (int offset) : ReadUnit(_byteBuffer.AsSpan(_byteStart + offset, _unitSize), _unitSize); } - /// - /// Byte offset of the first \r or \n unit in , which starts on a - /// unit boundary, or -1. A trailing partial unit is not searched. - /// private int FindTerminator (ReadOnlySpan bytes) { switch (_unitSize) @@ -282,11 +267,7 @@ private int FindTerminator (ReadOnlySpan bytes) } } - /// - /// Decodes the first bytes of the line in progress into the current - /// block and returns the text, capped at the maximum line length. A line is decoded on its own: - /// a sequence cut short by the terminator becomes U+FFFD in this line, as a continuous decode does. - /// + // A line is decoded on its own: a sequence cut short by the terminator becomes U+FFFD in this line. private ReadOnlyMemory DecodeLine (int byteCount) { var bytes = _byteBuffer.AsSpan(_byteStart, byteCount); @@ -294,20 +275,19 @@ private ReadOnlyMemory DecodeLine (int byteCount) int charCount; if (_asciiCompatible) { - // Most log lines are pure ASCII: widen them directly, which skips the per-call cost of the - // encoding's decoder. A line with any other byte is decoded by the encoding instead. + // Pure-ASCII lines skip the per-call cost of the encoding's decoder. EnsureCharCapacity(byteCount); - if (Ascii.ToUtf16(bytes, _readBlock.AsSpan(_charFill), out charCount) != OperationStatus.Done) + if (Ascii.ToUtf16(bytes, _readBlock.AsSpan(_usedChars), out charCount) != OperationStatus.Done) { charCount = DecodeWithEncoding(bytes); } } - else if (_nativeUtf16 && (byteCount & 1) == 0 && !MemoryMarshal.Cast(bytes).ContainsAny(_surrogates)) + else if (_isUtf16LittleEndian && (byteCount & 1) == 0 && !MemoryMarshal.Cast(bytes).ContainsAny(_surrogates)) { - // UTF-16 in the machine's byte order without surrogates decodes to its own code units. + // UTF-16 LE without surrogates decodes to its own code units. var source = MemoryMarshal.Cast(bytes); EnsureCharCapacity(source.Length); - source.CopyTo(_readBlock.AsSpan(_charFill)); + source.CopyTo(_readBlock.AsSpan(_usedChars)); charCount = source.Length; } else @@ -317,15 +297,15 @@ private ReadOnlyMemory DecodeLine (int byteCount) var cappedLength = Math.Min(charCount, MaximumLineLength); - var lineMemory = _readBlock.AsMemory(_charFill, cappedLength); - _charFill += cappedLength; + var lineMemory = _readBlock.AsMemory(_usedChars, cappedLength); + _usedChars += cappedLength; return lineMemory; } private int DecodeWithEncoding (ReadOnlySpan bytes) { EnsureCharCapacity(Encoding.GetMaxCharCount(bytes.Length)); - return Encoding.GetChars(bytes, _readBlock.AsSpan(_charFill)); + return Encoding.GetChars(bytes, _readBlock.AsSpan(_usedChars)); } private void ConsumeBytes (int byteCount) @@ -335,19 +315,15 @@ private void ConsumeBytes (int byteCount) _scannedBytes = 0; } - /// - /// Makes room for chars after the lines already in the current block. - /// A block that holds lines is kept for ; an unused one goes back to the pool. - /// Lines longer than BLOCK_SIZE (e.g. huge XML payloads) get a block of their own. - /// + // A block that already holds lines is kept for DetachCharBlocks; lines longer than BLOCK_SIZE get a block of their own. private void EnsureCharCapacity (int charCount) { - if (_readBlock.Length - _charFill >= charCount) + if (_readBlock.Length - _usedChars >= charCount) { return; } - if (_charFill > 0) + if (_usedChars > 0) { _completedBlocks.Add(_readBlock); } @@ -363,13 +339,10 @@ private void EnsureCharCapacity (int charCount) } _readBlock = ArrayPool.Shared.Rent(blockSize); - _charFill = 0; + _usedChars = 0; } - /// - /// Reads more bytes after the line in progress. When the buffer is full, the line in progress is - /// moved to the front, or the buffer doubles if that line already fills it. - /// + // When the buffer is full, the line in progress moves to the front, or the buffer doubles if that line fills it. private void Refill () { if (_byteEnd == _byteBuffer.Length) @@ -401,11 +374,8 @@ private void Refill () } } - /// - /// Discards the bytes read ahead so the next read starts at the (just seeked) stream position. - /// Only touches value-type fields, which is safe under the base constructor's virtual call to this - /// method. Lines already handed out stay valid: their chars are kept until . - /// + // Only value-type fields: the base constructor calls this before the fields are initialized. + // Lines already handed out stay valid until DetachCharBlocks. protected override void ResetReader () { _byteStart = 0; diff --git a/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs b/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs index dca26bb4..589afab3 100644 --- a/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs +++ b/src/LogExpert.Tests/ConfigManagerTests/ConfigManagerTest.cs @@ -437,11 +437,7 @@ public void LoadOrCreateNew_ValidFile_LoadsSuccessfully () Assert.That(loadResult.Settings.FilterList[0].SearchText, Is.EqualTo("VALID_FILTER_TEST")); } - /// - /// ADR 0010 removed the reader choice. Settings saved by earlier versions still carry - /// "ReaderType"; they must load as they are, with everything else intact, and not fall - /// back to the backup or to defaults. - /// + // ADR 0010 removed the reader choice; settings saved by earlier versions still carry "ReaderType". [Test] [Category("DeserializationRecovery")] [TestCase("SystemDirect")] diff --git a/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs index 87e1e347..a86633e2 100644 --- a/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/DirectReaderExactPositionTests.cs @@ -8,19 +8,11 @@ namespace LogExpert.Tests.StreamReaderTests; -/// -/// Issue #639: the byte must equal the raw bytes -/// consumed, whatever the bytes are. Re-encoding decoded text overcounts invalid sequences (each U+FFFD -/// re-encodes as 3 UTF-8 bytes), so a file with a binary block looked larger than it was and every -/// growth was mistaken for a recreated file. The oracle here is the raw byte stream itself: a line ends -/// at the next \n, \r\n or bare \r unit, and its text is that line's bytes decoded on -/// their own. -/// +// Issue #639: Position must equal the raw bytes consumed, whatever they are. Re-encoding decoded text +// counted each U+FFFD as 3 UTF-8 bytes, so a file with a binary block looked larger than it was. [TestFixture] public class DirectReaderExactPositionTests { - private const int MAX_LINE_LENGTH = 500_000; - #region Invalid byte sequences private static IEnumerable InvalidUtf8Cases () @@ -46,7 +38,7 @@ static TestCaseData Case (string name, string before, byte[] invalid, string aft [TestCaseSource(nameof(InvalidUtf8Cases))] public void Position_InvalidUtf8_MatchesRawBytes (byte[] data) { - AssertMatchesOracle(data, Encoding.UTF8); + RawByteLineOracle.AssertMatches(data, Encoding.UTF8); } [Test] @@ -54,7 +46,7 @@ public void Position_InvalidUtf8_ContentShowsReplacementCharacters () { byte[] data = [.. "abc"u8, 0x80, .. "def\n"u8]; - var lines = ReadAll(data, Encoding.UTF8); + var lines = RawByteLineOracle.ReadAll(data, Encoding.UTF8); Assert.That(lines.Select(line => line.Content), Is.EqualTo(new[] { "abc�def" })); } @@ -87,7 +79,7 @@ public void Position_Utf16WithLoneSurrogateAndOddTrailingByte_MatchesRawBytes () { byte[] data = [.. Encoding.Unicode.GetBytes("ab"), 0x00, 0xD8, .. Encoding.Unicode.GetBytes("\r\ncd\n"), 0x41]; - AssertMatchesOracle(data, Encoding.Unicode); + RawByteLineOracle.AssertMatches(data, Encoding.Unicode); } [Test] @@ -96,7 +88,7 @@ public void Position_Utf16WithSurrogatePairs_MatchesRawBytes () { var data = Encoding.Unicode.GetBytes("plain\r\nemoji 🙂 and 𝄞\npair at end 🙂"); - AssertMatchesOracle(data, Encoding.Unicode); + RawByteLineOracle.AssertMatches(data, Encoding.Unicode); } [Test] @@ -104,7 +96,7 @@ public void Position_Utf16BigEndian_MatchesRawBytes () { var data = Encoding.BigEndianUnicode.GetBytes("first\r\nsecond\nthird\rfourth"); - AssertMatchesOracle(data, Encoding.BigEndianUnicode); + RawByteLineOracle.AssertMatches(data, Encoding.BigEndianUnicode); } [Test] @@ -113,18 +105,14 @@ public void Position_Gb2312WithInvalidLeadBytes_MatchesRawBytes () var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); byte[] data = [.. gb2312.GetBytes("错误 "), 0x81, .. "\n"u8, 0xFF, .. gb2312.GetBytes("警告\n")]; - AssertMatchesOracle(data, gb2312); + RawByteLineOracle.AssertMatches(data, gb2312); } #endregion #region Buffer boundaries - /// - /// A multi-byte character, an invalid sequence or a \r\n pair that straddles the reader's - /// internal buffer must not shift the position. The boundaries cover the old 32 K char block and - /// common byte buffer sizes, on both short-line and single-long-line layouts. - /// + // Boundaries cover the old 32 K char block and common byte buffer sizes. [Test] public void Position_SequenceStraddlesBufferBoundary_MatchesRawBytes ( [Values(16_383, 16_384, 32_767, 32_768, 65_535, 65_536, 131_071, 131_072)] int boundary, @@ -144,7 +132,7 @@ public void Position_SequenceStraddlesBufferBoundary_MatchesRawBytes ( var prefix = Filler(boundary - startOffset, shortLines); byte[] data = [.. prefix, .. sequence, .. "after\nlast"u8]; - AssertMatchesOracle(data, Encoding.UTF8); + RawByteLineOracle.AssertMatches(data, Encoding.UTF8); } static byte[] Filler (int length, bool shortLines) @@ -170,7 +158,7 @@ public void Position_RecordedLineStarts_SeekBackToTheSameLines () var expected = new List<(long Start, string Content)>(); using (var stream = new MemoryStream(data)) - using (var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, MAX_LINE_LENGTH)) + using (var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = Encoding.UTF8 }, RawByteLineOracle.MAX_LINE_LENGTH)) { var start = reader.Position; while (reader.TryReadLine(out var line)) @@ -181,7 +169,7 @@ public void Position_RecordedLineStarts_SeekBackToTheSameLines () } using var seekStream = new MemoryStream(data); - using var seekReader = new PositionAwareStreamReaderDirect(seekStream, new EncodingOptions { Encoding = Encoding.UTF8 }, MAX_LINE_LENGTH); + using var seekReader = new PositionAwareStreamReaderDirect(seekStream, new EncodingOptions { Encoding = Encoding.UTF8 }, RawByteLineOracle.MAX_LINE_LENGTH); // Walk backwards so every seek moves the stream and discards what the reader had buffered. for (var index = expected.Count - 1; index >= 0; index -= 37) @@ -200,7 +188,7 @@ public void Position_RecordedLineStarts_SeekBackToTheSameLines () [Test] public void Position_TextWithBinaryBlock_MatchesRawBytes () { - AssertMatchesOracle(TextWithBinaryBlock(lineCount: 3_000, binaryLength: 33_000, seed: 42), Encoding.UTF8); + RawByteLineOracle.AssertMatches(TextWithBinaryBlock(lineCount: 3_000, binaryLength: 33_000, seed: 42), Encoding.UTF8); } [Test] @@ -220,7 +208,7 @@ public void Position_RandomBytes_MatchesRawBytes ([Range(0, 49)] int seed) }; } - AssertMatchesOracle(data, Encoding.UTF8); + RawByteLineOracle.AssertMatches(data, Encoding.UTF8); } [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] @@ -245,83 +233,4 @@ private static byte[] TextWithBinaryBlock (int lineCount, int binaryLength, int } #endregion - - #region Oracle - - internal static void AssertMatchesOracle (byte[] data, Encoding encoding) - { - var expected = ExpectedLines(data, encoding); - var actual = ReadAll(data, encoding); - - Assert.That(actual.Count, Is.EqualTo(expected.Count), "Line count"); - for (var i = 0; i < expected.Count; i++) - { - if (actual[i] != expected[i]) - { - Assert.Fail($"Line {i}: expected end {expected[i].End} \"{Shorten(expected[i].Content)}\", got end {actual[i].End} \"{Shorten(actual[i].Content)}\""); - } - } - } - - private static List<(string Content, long End)> ReadAll (byte[] data, Encoding encoding) - { - using var stream = new MemoryStream(data); - using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, MAX_LINE_LENGTH); - - var lines = new List<(string, long)>(); - while (reader.TryReadLine(out var line)) - { - lines.Add((line.ToString(), reader.Position)); - } - - return lines; - } - - /// - /// Splits the raw bytes at \n, \r\n and bare \r code units of the encoding and - /// decodes each line's bytes on its own. A trailing line without terminator is a line; an empty - /// remainder after the last terminator is not. - /// - private static List<(string Content, long End)> ExpectedLines (byte[] data, Encoding encoding) - { - var cr = encoding.GetBytes("\r"); - var lf = encoding.GetBytes("\n"); - var unit = lf.Length; - - var lines = new List<(string, long)>(); - var lineStart = 0; - var position = 0; - - while (position + unit <= data.Length) - { - var span = data.AsSpan(position, unit); - if (span.SequenceEqual(lf) || span.SequenceEqual(cr)) - { - var terminatorLength = unit; - if (span.SequenceEqual(cr) && position + (2 * unit) <= data.Length && data.AsSpan(position + unit, unit).SequenceEqual(lf)) - { - terminatorLength = 2 * unit; - } - - lines.Add((encoding.GetString(data, lineStart, position - lineStart), position + terminatorLength)); - position += terminatorLength; - lineStart = position; - } - else - { - position += unit; - } - } - - if (lineStart < data.Length) - { - lines.Add((encoding.GetString(data, lineStart, data.Length - lineStart), data.Length)); - } - - return lines; - } - - private static string Shorten (string text) => text.Length <= 60 ? text : $"{text[..60]}… ({text.Length} chars)"; - - #endregion } \ No newline at end of file diff --git a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs index 4bd0657b..ca0a58de 100644 --- a/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/LogfileReaderSingleFileMonitoringTests.cs @@ -72,11 +72,7 @@ public void SingleFileReader_OnTruncation_ReportsNewFileNotRollover () Assert.That(rolloverReported, Is.False); } - /// - /// Issue #639: a file with a binary block in the middle must grow like any other file. An overcounted - /// byte position made the file look smaller than what had been read, so every append was reported as - /// a new file and the window reloaded and jumped to the top. - /// + // Issue #639: an overcounted position made every append look like a new file, so the window reloaded. [Test] [System.Diagnostics.CodeAnalysis.SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Unit Tests")] public void SingleFileReader_GrowingFileWithBinaryBlock_TakesGrowthPathNotNewFile () diff --git a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs index b203af27..c0e6d6b7 100644 --- a/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs +++ b/src/LogExpert.Tests/StreamReaderTests/PositionAwareStreamReaderDirectTests.cs @@ -167,14 +167,14 @@ public void TryReadLine_LineCrossesBlockBoundary_Correct () [TestCase("Short\nMedium line here\nA much longer line with more content\n")] public void Position_MatchesRawBytes_UTF8 (string text) { - DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.UTF8.GetBytes(text), Encoding.UTF8); + RawByteLineOracle.AssertMatches(Encoding.UTF8.GetBytes(text), Encoding.UTF8); } [Test] [TestCase("Line 1\nLine 2\nLine 3\n")] public void Position_MatchesRawBytes_ASCII (string text) { - DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.ASCII.GetBytes(text), Encoding.ASCII); + RawByteLineOracle.AssertMatches(Encoding.ASCII.GetBytes(text), Encoding.ASCII); } [Test] @@ -182,7 +182,7 @@ public void Position_MatchesRawBytes_ASCII (string text) public void Position_MatchesRawBytes_UTF8_MultibyteChars () { var text = "Héllo wörld\nCafé résumé\nこんにちは\n"; - DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.UTF8.GetBytes(text), Encoding.UTF8); + RawByteLineOracle.AssertMatches(Encoding.UTF8.GetBytes(text), Encoding.UTF8); } /// @@ -197,7 +197,7 @@ public void Position_MatchesRawBytes_Gb2312_MixedAsciiAndChinese () { var gb2312 = EncodingRegistry.GetEncoding(EncodingRegistry.CODE_PAGE_GB2312); var text = "2026-07-30 错误: 连接失败\nINFO 启动完成\nplain ascii line\n警告\n"; - DirectReaderExactPositionTests.AssertMatchesOracle(gb2312.GetBytes(text), gb2312); + RawByteLineOracle.AssertMatches(gb2312.GetBytes(text), gb2312); } #endregion @@ -235,22 +235,21 @@ public void TryReadLine_LargeFile_ReturnsEveryLine () #region Allocations - /// - /// The reader must stay allocation-free per line: lines land in pooled blocks, and only detaching - /// blocks allocates (the list handed to the buffer). Measured on the reading thread, so the result - /// is deterministic; any per-line allocation (at least 24 bytes) breaks the budget of 0.25 bytes. - /// + // Only detaching blocks may allocate (the list handed to the buffer). Measured on the reading thread, + // so the result is deterministic; any per-line allocation (at least 24 bytes) breaks 0.25 bytes per line. [Test] - public void TryReadLine_ValidText_DoesNotAllocatePerLine ([Values(65001, 1200)] int codePage) + [System.Diagnostics.CodeAnalysis.SuppressMessage("Globalization", "CA1303:Do not pass literals as localized parameters", Justification = "Unit Tests")] + public void TryReadLine_ValidText_DoesNotAllocatePerLine ([Values(65001, 1200)] int codePage, [Values(false, true)] bool nonAscii) { const int lineCount = 200_000; const int linesPerBuffer = 500; var encoding = Encoding.GetEncoding(codePage); + var message = nonAscii ? " [INFO] Größe 日志 🙂 - Log message number " : " [INFO] Thread-7 SomeClass - Log message number "; var sb = new StringBuilder(); for (var i = 0; i < lineCount; i++) { - _ = sb.Append("2026-04-23 12:00:00.").Append(i % 1000).Append(" [INFO] Thread-7 SomeClass - Log message number ").Append(i).Append("\r\n"); + _ = sb.Append("2026-04-23 12:00:00.").Append(i % 1000).Append(message).Append(i).Append("\r\n"); } var data = encoding.GetBytes(sb.ToString()); @@ -477,7 +476,7 @@ public void TryReadLine_LineLongerThanBlockSize_PositionMatchesRawBytes () const int lineLength = 50_000; var text = $"short\n{new string('Q', lineLength)}\nend\n"; - DirectReaderExactPositionTests.AssertMatchesOracle(Encoding.UTF8.GetBytes(text), Encoding.UTF8); + RawByteLineOracle.AssertMatches(Encoding.UTF8.GetBytes(text), Encoding.UTF8); }); } diff --git a/src/LogExpert.Tests/StreamReaderTests/RawByteLineOracle.cs b/src/LogExpert.Tests/StreamReaderTests/RawByteLineOracle.cs new file mode 100644 index 00000000..df063fcf --- /dev/null +++ b/src/LogExpert.Tests/StreamReaderTests/RawByteLineOracle.cs @@ -0,0 +1,87 @@ +using System.Text; + +using LogExpert.Core.Classes.Log.Streamreaders; +using LogExpert.Core.Entities; + +using NUnit.Framework; + +namespace LogExpert.Tests.StreamReaderTests; + +// The raw bytes as oracle for the Direct reader (ADR 0010): a line ends at the next \n, \r\n or bare +// \r code unit of the encoding, its text is that line's bytes decoded on their own, and the position +// after it is the byte offset after its terminator. +internal static class RawByteLineOracle +{ + public const int MAX_LINE_LENGTH = 500_000; + + public static void AssertMatches (byte[] data, Encoding encoding) + { + var expected = ExpectedLines(data, encoding); + var actual = ReadAll(data, encoding); + + Assert.That(actual.Count, Is.EqualTo(expected.Count), "Line count"); + for (var i = 0; i < expected.Count; i++) + { + if (actual[i] != expected[i]) + { + Assert.Fail($"Line {i}: expected end {expected[i].End} \"{Shorten(expected[i].Content)}\", got end {actual[i].End} \"{Shorten(actual[i].Content)}\""); + } + } + } + + public static List<(string Content, long End)> ReadAll (byte[] data, Encoding encoding) + { + using var stream = new MemoryStream(data); + using var reader = new PositionAwareStreamReaderDirect(stream, new EncodingOptions { Encoding = encoding }, MAX_LINE_LENGTH); + + var lines = new List<(string, long)>(); + while (reader.TryReadLine(out var line)) + { + lines.Add((line.ToString(), reader.Position)); + } + + return lines; + } + + // A trailing line without terminator is a line; an empty remainder after the last terminator is not. + private static List<(string Content, long End)> ExpectedLines (byte[] data, Encoding encoding) + { + var cr = encoding.GetBytes("\r"); + var lf = encoding.GetBytes("\n"); + var unit = lf.Length; + + var lines = new List<(string, long)>(); + var lineStart = 0; + var position = 0; + + while (position + unit <= data.Length) + { + var span = data.AsSpan(position, unit); + if (span.SequenceEqual(lf) || span.SequenceEqual(cr)) + { + var terminatorLength = unit; + if (span.SequenceEqual(cr) && position + (2 * unit) <= data.Length && data.AsSpan(position + unit, unit).SequenceEqual(lf)) + { + terminatorLength = 2 * unit; + } + + lines.Add((encoding.GetString(data, lineStart, position - lineStart), position + terminatorLength)); + position += terminatorLength; + lineStart = position; + } + else + { + position += unit; + } + } + + if (lineStart < data.Length) + { + lines.Add((encoding.GetString(data, lineStart, data.Length - lineStart), data.Length)); + } + + return lines; + } + + private static string Shorten (string text) => text.Length <= 60 ? text : $"{text[..60]}… ({text.Length} chars)"; +} \ No newline at end of file