From 88b8c1d3a38ef6bd0bcad13d3f7745676fcbfd20 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 16 Nov 2026 07:28:23 +0100 Subject: [PATCH 1/3] fix(reader): read vortex.varbin offsets of every integer width Rust narrows varbin offsets to the smallest integer type holding the data buffer's length: u8 under 256 bytes, u16 under 64 KiB. VarBinOffsetArray read every type but i32/u32 as 64-bit, so a small Rust-written binary column decoded garbage offsets and failed ("varbin element bytes [...) out of range"). Found exporting #721's mnist image struct to Parquet. The fix sits in VarBinOffsetArray, which every varbin producer (varbin, zstd, dict) constructs from a file-declared offsets type; a non-integer type is now rejected there. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + ...inNarrowOffsetsInteropIntegrationTest.java | 100 ++++++++++++++++++ .../reader/array/VarBinOffsetArray.java | 48 ++++++--- .../vortex/reader/array/VarBinArrayTest.java | 53 ++++++++++ 4 files changed, 189 insertions(+), 13 deletions(-) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/VarBinNarrowOffsetsInteropIntegrationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c57d5077..12291eb0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `Chunk.prefetch` and `VortexHandle.prefetch` start the range requests of several columns together over HTTP, merged into few requests, and an HTTP handle keeps the segments it has read. ([#744](https://github.com/dfa1/vortex-java/issues/744)) ### Fixed +- A Rust-written `vortex.varbin` column with 8- or 16-bit offsets, as Rust writes a data buffer under 64 KiB, reads its values instead of failing. ([#721](https://github.com/dfa1/vortex-java/issues/721)) - An unexpected HTTP status now reports the server's own explanation and `Retry-After`, so a Hugging Face rate limit (`429`) says what to do instead of just the code. ([#743](https://github.com/dfa1/vortex-java/issues/743)) - `vortex inspect` over HTTP fetches only the end of each segment, several at a time, instead of the whole file, and shows a progress bar. ([#743](https://github.com/dfa1/vortex-java/issues/743)) - `vortex view` no longer freezes while it fetches data over HTTP: it shows a loading spinner, `q` quits at once, and it fetches only the columns on screen. ([#744](https://github.com/dfa1/vortex-java/issues/744)) diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/VarBinNarrowOffsetsInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/VarBinNarrowOffsetsInteropIntegrationTest.java new file mode 100644 index 000000000..6ef1628de --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/VarBinNarrowOffsetsInteropIntegrationTest.java @@ -0,0 +1,100 @@ +package io.github.dfa1.vortex.integration; + +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexReader; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/// Rust narrows `vortex.varbin` offsets to the smallest integer type holding the data buffer's +/// length: u8 under 256 bytes, u16 under 64 KiB. Java read every offset type but i32/u32 as 64-bit, +/// so a small Rust-written binary column decoded as garbage offsets and failed (#721). +class VarBinNarrowOffsetsInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final ColumnName B = ColumnName.of("b"); + + static { + NativeLoader.loadJni(); + } + + /// 3-byte values: 50 rows fill a 150-byte buffer (u8 offsets), 1000 a 3000-byte one (u16). + @ParameterizedTest(name = "{0} rows") + @ValueSource(ints = {50, 1000}) + void javaReadsTheNarrowOffsetsJniWrote(int rows, @TempDir Path tmp) throws Exception { + // Given + Path file = tmp.resolve("jni.vortex"); + List expected = new ArrayList<>(); + for (int i = 0; i < rows; i++) { + expected.add(HexFormat.of().formatHex(value(i))); + } + writeWithJni(file, rows); + + // When + List result = new ArrayList<>(); + try (VortexReader reader = VortexReader.open(file); + ScanIterator scan = reader.scan(ScanOptions.all())) { + while (scan.hasNext()) { + try (Chunk chunk = scan.next()) { + VarBinArray values = chunk.column(B); + for (int i = 0; i < chunk.rowCount(); i++) { + result.add(HexFormat.of().formatHex(values.getBytes(i))); + } + } + } + } + + // Then — the file must really hold a varbin, or the test proves nothing + assertThat(String.join("\n", EncodingTrees.of(file))).contains("vortex.varbin"); + assertThat(result).isEqualTo(expected); + } + + private static byte[] value(int row) { + return new byte[]{(byte) row, (byte) (row >> 8), 7}; + } + + private static void writeWithJni(Path path, int rows) throws Exception { + Schema schema = new Schema(List.of(Field.notNullable("b", ArrowType.Binary.INSTANCE))); + try (VortexWriter writer = VortexWriter.builder(SESSION, path.toAbsolutePath().toUri().toString(), schema, + ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, ALLOCATOR)) { + VarBinaryVector vector = (VarBinaryVector) root.getVector("b"); + vector.allocateNew(); + for (int i = 0; i < rows; i++) { + vector.setSafe(i, value(i)); + } + vector.setValueCount(rows); + root.setRowCount(rows); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, arrowSchema); + writer.writeBatch(array.memoryAddress(), arrowSchema.memoryAddress()); + } + } + } +} diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java index 45cbe496b..fd05c05e5 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/array/VarBinOffsetArray.java @@ -6,6 +6,7 @@ import io.github.dfa1.vortex.core.io.VortexFormat; import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; import java.nio.charset.StandardCharsets; import java.util.function.IntConsumer; @@ -17,12 +18,20 @@ /// @param length number of variable-length elements /// @param bytesSegment concatenated raw byte data for all elements /// @param offsetsSegment offsets segment of length `length + 1` -/// @param offsetsPtype physical type of the offsets values (I32/U32 or I64/U64) +/// @param offsetsPtype physical type of the offsets values, any integer width: Rust narrows them +/// to the smallest one holding the data buffer's length @SuppressWarnings("java:S6218") // internal data carrier; record components are arrays of immutable primitives or refs that flow through pipelines without ever being compared. public record VarBinOffsetArray(DType dtype, long length, MemorySegment bytesSegment, MemorySegment offsetsSegment, PType offsetsPtype) implements VarBinArray { + /// Rejects a non-integer offsets type, which comes from untrusted metadata. + public VarBinOffsetArray { + if (!offsetsPtype.isInteger()) { + throw new VortexException("varbin offsets must be integers, got " + offsetsPtype); + } + } + @Override public byte[] getBytes(long i) { long start = readOffset(i); @@ -53,11 +62,22 @@ public void forEachByteLength(IntConsumer c) { long n = length; // The loop reads offsets[0..n] at a fixed stride and must stay uniform to // vectorize (AGENTS.md hot-loop rule), so the untrusted offsets segment is - // sized once here rather than bounds-checked per row. A non-monotonic pair - // still yields a negative length to the consumer; the typed accessors + // sized once here rather than bounds-checked per row. A non-monotonic pair or a + // negative narrow offset still yields a wrong length to the consumer; the typed accessors // ([#getBytes(long)], [#getByteLength(long)]) reject it when the row is read. checkOffsetsExtent(n); - if (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) { + // A narrow offset widens unsigned: exact for every valid (non-negative) one, signed or not + if (offsetWidth() == Byte.BYTES) { + for (long i = 0; i < n; i++) { + c.accept(Byte.toUnsignedInt(offsetsSegment.get(ValueLayout.JAVA_BYTE, i + 1)) + - Byte.toUnsignedInt(offsetsSegment.get(ValueLayout.JAVA_BYTE, i))); + } + } else if (offsetWidth() == Short.BYTES) { + for (long i = 0; i < n; i++) { + c.accept(Short.toUnsignedInt(offsetsSegment.getAtIndex(VortexFormat.LE_SHORT, i + 1)) + - Short.toUnsignedInt(offsetsSegment.getAtIndex(VortexFormat.LE_SHORT, i))); + } + } else if (offsetWidth() == Integer.BYTES) { for (long i = 0; i < n; i++) { c.accept(offsetsSegment.getAtIndex(VortexFormat.LE_INT, i + 1) - offsetsSegment.getAtIndex(VortexFormat.LE_INT, i)); @@ -99,7 +119,7 @@ private void checkOffsetsExtent(long rows) { } private int offsetWidth() { - return (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) ? Integer.BYTES : Long.BYTES; + return offsetsPtype.byteSize(); } /// Reads offset `i` at the width of [#offsetsPtype]. @@ -109,17 +129,19 @@ private int offsetWidth() { /// `IndexOutOfBoundsException` (ADR 0003). /// /// @param i zero-based offset index, in `[0, length]` - /// @return the offset value widened to a long, unsigned for `U32` + /// @return the offset value widened to a long, unsigned for an unsigned type private long readOffset(long i) { try { - if (offsetsPtype == PType.U32) { + return switch (offsetsPtype) { + case U8 -> Byte.toUnsignedLong(offsetsSegment.get(ValueLayout.JAVA_BYTE, i)); + case I8 -> offsetsSegment.get(ValueLayout.JAVA_BYTE, i); + case U16 -> Short.toUnsignedLong(offsetsSegment.getAtIndex(VortexFormat.LE_SHORT, i)); + case I16 -> offsetsSegment.getAtIndex(VortexFormat.LE_SHORT, i); // a u32 offset passes Integer.MAX_VALUE once the data buffer does (#633) - return Integer.toUnsignedLong(offsetsSegment.getAtIndex(VortexFormat.LE_INT, i)); - } - if (offsetsPtype == PType.I32) { - return offsetsSegment.getAtIndex(VortexFormat.LE_INT, i); - } - return offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i); + case U32 -> Integer.toUnsignedLong(offsetsSegment.getAtIndex(VortexFormat.LE_INT, i)); + case I32 -> offsetsSegment.getAtIndex(VortexFormat.LE_INT, i); + default -> offsetsSegment.getAtIndex(VortexFormat.LE_LONG, i); + }; } catch (IndexOutOfBoundsException e) { throw new VortexException("varbin offset index " + i + " (" + offsetsPtype + ") out of range for an offsets segment of " diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java index 7f5a22e09..9574f243b 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/array/VarBinArrayTest.java @@ -84,6 +84,59 @@ void getByteLength_unsignedOffsetPastTheDataBuffer_throwsVortexException() { } } + /// Rust narrows varbin offsets to the smallest integer holding the data buffer's length, so a + /// Rust-written file carries u8/u16 offsets; read at 64 bits they came back as garbage (#721). + @Nested + class OffsetWidths { + + @ParameterizedTest + @EnumSource(value = PType.class, names = {"U8", "I8", "U16", "I16", "U32", "I32", "U64", "I64"}) + void everyIntegerWidth_readsItsOffsets(PType ptype) { + // Given — "ab", "", "cde" over offsets 0, 2, 2, 5 at the width of `ptype` + MemorySegment offsets = MemorySegment.ofArray(new byte[4 * ptype.byteSize()]); + long[] values = {0, 2, 2, 5}; + for (int i = 0; i < values.length; i++) { + for (int b = 0; b < ptype.byteSize(); b++) { + offsets.set(java.lang.foreign.ValueLayout.JAVA_BYTE, (long) i * ptype.byteSize() + b, + (byte) (values[i] >>> (8 * b))); + } + } + var sut = new VarBinOffsetArray(UTF8, 3, MemorySegment.ofArray("abcde".getBytes(StandardCharsets.UTF_8)), + offsets, ptype); + List lengths = new ArrayList<>(); + + // When + sut.forEachByteLength(lengths::add); + + // Then + assertThat(List.of(sut.getString(0), sut.getString(1), sut.getString(2))).containsExactly("ab", "", "cde"); + assertThat(lengths).containsExactly(2, 0, 3); + assertThat(sut.limited(2).getString(1)).isEmpty(); + } + + @Test + void u8OffsetPast127_readsUnsigned() { + // Given — 200 bytes, the second offset 0xC8 negative as a signed byte + var sut = new VarBinOffsetArray(DTypes.BINARY, 1, MemorySegment.ofArray(new byte[200]), + MemorySegment.ofArray(new byte[]{0, (byte) 200}), PType.U8); + + // When + int result = sut.getByteLength(0); + + // Then + assertThat(result).isEqualTo(200); + } + + @Test + void floatOffsets_throwVortexException() { + // Given / When / Then — the offsets ptype comes from untrusted metadata + assertThatThrownBy(() -> new VarBinOffsetArray(DTypes.BINARY, 0, MemorySegment.NULL, + MemorySegment.ofArray(new byte[4]), PType.F32)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("F32"); + } + } + @Nested class Regular { From 6b7ecc2d5b61e2a54394caf192f4562fa5d7c5e4 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 16 Nov 2026 07:28:23 +0100 Subject: [PATCH 2/3] fix(parquet): import a null struct holding a required list or struct A null struct pushes null into every field builder. The leaf builder takes it as a placeholder the struct's validity masks, but the list and struct builders rejected it when their own dtype was non-nullable, so a Parquet file with an optional struct over a required list or struct failed to import. They now append an empty list / null fields instead. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + .../dfa1/vortex/parquet/ColumnBuilder.java | 18 ++++++----- .../vortex/parquet/ColumnBuilderTest.java | 31 ++++++++++++------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12291eb0c..fe28057bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - A Rust-written `vortex.varbin` column with 8- or 16-bit offsets, as Rust writes a data buffer under 64 KiB, reads its values instead of failing. ([#721](https://github.com/dfa1/vortex-java/issues/721)) +- A Parquet import no longer fails on a null struct holding a required list or struct field. ([#721](https://github.com/dfa1/vortex-java/issues/721)) - An unexpected HTTP status now reports the server's own explanation and `Retry-After`, so a Hugging Face rate limit (`429`) says what to do instead of just the code. ([#743](https://github.com/dfa1/vortex-java/issues/743)) - `vortex inspect` over HTTP fetches only the end of each segment, several at a time, instead of the whole file, and shows a progress bar. ([#743](https://github.com/dfa1/vortex-java/issues/743)) - `vortex view` no longer freezes while it fetches data over HTTP: it shows a loading spinner, `q` quits at once, and it fetches only the columns on screen. ([#744](https://github.com/dfa1/vortex-java/issues/744)) diff --git a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ColumnBuilder.java b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ColumnBuilder.java index 5383c7132..93b38777e 100644 --- a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ColumnBuilder.java +++ b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ColumnBuilder.java @@ -84,14 +84,15 @@ public long bytes() { @Override public void append(Object value) { + // A null reaching a non-nullable struct comes from a null struct above it, whose validity + // masks the row, as LeafColumnBuilder#append explains if (value == null) { - if (validity == null) { - throw new IllegalArgumentException("null value for non-nullable struct column"); - } for (ColumnBuilder fieldBuilder : fieldBuilders) { fieldBuilder.append(null); } - validity.add(false); + if (validity != null) { + validity.add(false); + } return; } PqStruct struct = (PqStruct) value; @@ -137,12 +138,13 @@ public long bytes() { @Override public void append(Object value) { + // A null reaching a non-nullable list comes from a null struct above it, whose validity + // masks the row; it stays an empty list, as LeafColumnBuilder#append explains if (value == null) { - if (validity == null) { - throw new IllegalArgumentException("null value for non-nullable list column"); - } offsets.add(runningCount); - validity.add(false); + if (validity != null) { + validity.add(false); + } return; } PqList list = (PqList) value; diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java index dca94d8e6..c0a3d9287 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ColumnBuilderTest.java @@ -307,16 +307,20 @@ void nullableStruct_recursesIntoFields_andWrapsWholeStructOnNullRow() { } @Test - void nonNullableStruct_rejectsNullRow() { - // Given + void nonNullableStruct_takesTheNullOfANullStructAbove_asNullFields() { + // Given — a REQUIRED struct inside an OPTIONAL one receives null where the outer is + // null (#721); the outer struct's validity masks the row, so it must not throw DType.Struct dtype = new DType.Struct( List.of(ColumnName.of("bytes")), List.of(new DType.Binary(true)), false); ColumnBuilder sut = ColumnBuilder.forType(dtype); - // When / Then - assertThatThrownBy(() -> sut.append(null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("null value for non-nullable struct column"); + // When + sut.append(null); + Object result = sut.build(); + + // Then + StructData struct = (StructData) result; + assertThat(((NullableData) struct.fieldArrays().getFirst()).validity()).containsExactly(false); } } @@ -379,14 +383,17 @@ void listOfNullableStruct_propagatesElementLevelNulls() { } @Test - void nonNullableList_rejectsNullRow() { - // Given + void nonNullableList_takesTheNullOfANullStructAbove_asEmptyList() { + // Given — a REQUIRED list inside an OPTIONAL struct receives null where the struct is + // null (#721); the struct's validity masks the row, so it must not throw ColumnBuilder sut = ColumnBuilder.forType(new DType.List(new DType.Utf8(false), false)); - // When / Then - assertThatThrownBy(() -> sut.append(null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("null value for non-nullable list column"); + // When + sut.append(null); + Object result = sut.build(); + + // Then + assertThat(((ListData) result).offsets()).containsExactly(0L, 0L); } } } From f1b49f8a07eb272b46c069fab155b030751d3985 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 16 Nov 2026 07:28:23 +0100 Subject: [PATCH 3/3] feat(parquet): export struct, list and map columns (#721) ParquetExporter refused every nested column, so 4 of 10 public Raincloud datasets failed `vortex export x.vortex out.parquet`. Hardwood's columnar writer already takes Arrow-shaped nested input (struct validity, list offsets), so the exporter now declares the schema recursively (one Slot adapter over Hardwood's root/struct/element builders) and writes each leaf from gathered row indices: a list's elements, a list view's ranges and a fixed-size list's slices need no copy. Parquet stores no entries under a null list or null struct, while a Vortex null may still span some, so those rows get zero entries. Struct, List, FixedSizeList and Map map to a group, LIST, LIST and MAP; a FixedSizeList re-imports as a List. An unsupported type now throws VortexException naming the column path (e.g. emb.list.element) instead of UnsupportedOperationException. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 1 + docs/how-to.md | 5 +- docs/reference.md | 11 +- ...edParquetExportInteropIntegrationTest.java | 240 +++++++ .../dfa1/vortex/parquet/ParquetExporter.java | 619 ++++++++++++++---- .../vortex/parquet/ParquetExporterTest.java | 261 +++++++- 6 files changed, 997 insertions(+), 140 deletions(-) create mode 100644 integration/src/test/java/io/github/dfa1/vortex/integration/NestedParquetExportInteropIntegrationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index fe28057bd..89e4d8d88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- Parquet export writes `Struct`, `List`, `FixedSizeList` and `Map` columns as Parquet groups, and an unsupported column fails with a `VortexException` naming it. ([#721](https://github.com/dfa1/vortex-java/issues/721)) - `vortex rewrite` re-encodes a file with the current writer. ([#749](https://github.com/dfa1/vortex-java/pull/749)) - `Chunk.prefetch` and `VortexHandle.prefetch` start the range requests of several columns together over HTTP, merged into few requests, and an HTTP handle keeps the segments it has read. ([#744](https://github.com/dfa1/vortex-java/issues/744)) diff --git a/docs/how-to.md b/docs/how-to.md index 485b4e090..d7f9226e4 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -410,8 +410,9 @@ this project does not configure, so give such a file 8 GB. Files with narrow col ## Convert Vortex to Parquet -Flat schemas only (`Bool`, `Primitive` including `F16`, `Utf8`, `Binary`, `vortex.timestamp`); a -`Struct`/`List`/`Map` top-level column throws `UnsupportedOperationException`. +Leaves of `Bool`, `Primitive` including `F16`, `Decimal`, `Utf8`, `Binary` and `vortex.timestamp`, +flat or nested in `Struct`, `List`, `FixedSizeList` and `Map` columns (exported as Parquet groups, +`LIST` and `MAP`); any other type throws `VortexException` naming the column. **API:** diff --git a/docs/reference.md b/docs/reference.md index 11e78bd96..5a4b5a2ad 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -570,11 +570,14 @@ Final class (private constructor); `chunkSize` and `maxChunkBytes` bound the row ### `ParquetExporter` (`io.github.dfa1.vortex.parquet.ParquetExporter`) -Supports flat schemas only: `Bool`, `Primitive` (`F16` as Parquet's `FLOAT16`), `Utf8`, `Binary`, `Decimal` (Parquet `DECIMAL` on +Supports leaves of `Bool`, `Primitive` (`F16` as Parquet's `FLOAT16`), `Utf8`, `Binary`, `Decimal` (Parquet `DECIMAL` on `INT32`, `INT64` or `FIXED_LEN_BYTE_ARRAY` by precision), and the -`vortex.timestamp` extension over MILLIS/MICROS/NANOS resolution. `Struct`/`List`/`Map` top-level -columns throw `UnsupportedOperationException` — the inverse of `ParquetImporter`'s nested -`LIST`/`STRUCT` support does not exist yet. +`vortex.timestamp` extension over MILLIS/MICROS/NANOS resolution, flat or nested in `Struct` (a Parquet group), +`List` and `FixedSizeList` (`LIST`) and `Map` (`MAP`, its key `REQUIRED`) columns, composable to any depth. Any other +type throws `VortexException` naming the column (e.g. `emb.list.element`). A `FixedSizeList` re-imports as a `List`, +since Parquet has no fixed-size list. A present struct whose every field is null exports as a null struct: the reader +folds a struct's validity into its fields, so the two read the same. A struct, list or map whose name holds a `.` +throws, since Parquet writers address those groups by dotted path. | Method | Notes | |------------------------------------------------------------|-----------------------------------------------------------| diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/NestedParquetExportInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/NestedParquetExportInteropIntegrationTest.java new file mode 100644 index 000000000..79c121ca7 --- /dev/null +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/NestedParquetExportInteropIntegrationTest.java @@ -0,0 +1,240 @@ +package io.github.dfa1.vortex.integration; + +import dev.hardwood.InputFile; +import dev.hardwood.reader.ParquetFileReader; +import dev.hardwood.reader.RowReader; +import dev.hardwood.row.PqList; +import dev.hardwood.row.PqStruct; +import dev.vortex.api.Session; +import dev.vortex.api.VortexWriter; +import dev.vortex.arrow.ArrowAllocation; +import dev.vortex.jni.NativeLoader; +import io.github.dfa1.vortex.parquet.ParquetExporter; +import org.apache.arrow.c.ArrowArray; +import org.apache.arrow.c.ArrowSchema; +import org.apache.arrow.c.Data; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.vector.Float4Vector; +import org.apache.arrow.vector.VarBinaryVector; +import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.complex.FixedSizeListVector; +import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.StructVector; +import org.apache.arrow.vector.types.FloatingPointPrecision; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HexFormat; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +/// #721: a Rust-written file with the nested column shapes of the Raincloud datasets whose Parquet +/// export failed — `mnist`'s `struct`, `cohere-wikipedia-simple-embed`'s +/// fixed-size float embedding, `coig`'s `list>` — exports to Parquet with +/// every row, null and empty list intact. The list of structs is decoded from a `vortex.struct` +/// encoding inside the list's segment, the path a Java-written test file need not take. +class NestedParquetExportInteropIntegrationTest { + + private static final Session SESSION = Session.create(); + private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator(); + private static final int ROWS = 1000; + private static final int DIM = 4; + + static { + NativeLoader.loadJni(); + } + + @Test + void javaExportsTheNestedColumnsJniWrote(@TempDir Path tmp) throws Exception { + // Given + Path vortex = tmp.resolve("jni.vortex"); + writeWithJni(vortex); + Path parquet = tmp.resolve("out.parquet"); + + // When + ParquetExporter.exportParquet(vortex, parquet); + + // Then + List> result = new ArrayList<>(); + try (ParquetFileReader reader = ParquetFileReader.open(InputFile.of(parquet)); + RowReader rows = reader.buildRowReader().build()) { + while (rows.hasNext()) { + rows.next(); + result.add(Arrays.asList(image(rows), embedding(rows), turns(rows))); + } + } + List> expected = new ArrayList<>(); + for (int i = 0; i < ROWS; i++) { + expected.add(Arrays.asList(expectedImage(i), expectedEmbedding(i), expectedTurns(i))); + } + assertThat(result).isEqualTo(expected); + } + + // ── expected values: every fifth image and seventh embedding null, a third of the paths and a + // quarter of the answers null, and row i holding i % 3 turns, so empty lists recur ────────── + + private static List expectedImage(int row) { + if (row % 5 == 0) { + return null; + } + return Arrays.asList(HexFormat.of().formatHex(imageBytes(row)), row % 3 == 0 ? null : "img/" + row + ".png"); + } + + private static byte[] imageBytes(int row) { + return new byte[]{(byte) row, (byte) (row >> 8), 7}; + } + + private static List expectedEmbedding(int row) { + if (row % 7 == 0) { + return null; + } + List values = new ArrayList<>(DIM); + for (int j = 0; j < DIM; j++) { + values.add(row * 0.5f + j); + } + return values; + } + + private static List> expectedTurns(int row) { + List> turns = new ArrayList<>(); + for (int t = 0; t < row % 3; t++) { + int element = 3 * row + t; + turns.add(Arrays.asList("q" + element, element % 4 == 0 ? null : "a" + element)); + } + return turns; + } + + // ── read back from Parquet in the same shape ────────────────────────────────────────────── + + private static List image(RowReader rows) { + if (rows.isNull("img")) { + return null; + } + PqStruct img = rows.getStruct("img"); + return Arrays.asList(HexFormat.of().formatHex(img.getBinary("bytes")), img.getString("path")); + } + + private static List embedding(RowReader rows) { + if (rows.isNull("emb")) { + return null; + } + return rows.getList("emb").floats(); + } + + private static List> turns(RowReader rows) { + PqList conv = rows.getList("conv"); + List> turns = new ArrayList<>(); + for (PqStruct turn : conv.structs()) { + turns.add(Arrays.asList(turn.getString("question"), turn.getString("answer"))); + } + return turns; + } + + // ── vortex-jni writer ───────────────────────────────────────────────────────────────────── + + private static void writeWithJni(Path path) throws Exception { + Field img = new Field("img", FieldType.nullable(ArrowType.Struct.INSTANCE), List.of( + Field.notNullable("bytes", ArrowType.Binary.INSTANCE), + Field.nullable("path", ArrowType.Utf8.INSTANCE))); + Field emb = new Field("emb", FieldType.nullable(new ArrowType.FixedSizeList(DIM)), List.of( + Field.notNullable("item", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)))); + Field turn = new Field("item", FieldType.notNullable(ArrowType.Struct.INSTANCE), List.of( + Field.notNullable("question", ArrowType.Utf8.INSTANCE), + Field.nullable("answer", ArrowType.Utf8.INSTANCE))); + Field conv = new Field("conv", FieldType.notNullable(ArrowType.List.INSTANCE), List.of(turn)); + Schema schema = new Schema(List.of(img, emb, conv)); + try (VortexWriter writer = VortexWriter.builder(SESSION, path.toAbsolutePath().toUri().toString(), schema, + ALLOCATOR).build(); + VectorSchemaRoot root = VectorSchemaRoot.create(schema, ALLOCATOR)) { + fillImages((StructVector) root.getVector("img")); + fillEmbeddings((FixedSizeListVector) root.getVector("emb")); + fillTurns((ListVector) root.getVector("conv")); + root.setRowCount(ROWS); + try (ArrowArray array = ArrowArray.allocateNew(ALLOCATOR); + ArrowSchema arrowSchema = ArrowSchema.allocateNew(ALLOCATOR)) { + Data.exportVectorSchemaRoot(ALLOCATOR, root, null, array, arrowSchema); + writer.writeBatch(array.memoryAddress(), arrowSchema.memoryAddress()); + } + } + } + + private static void fillImages(StructVector images) { + images.allocateNew(); + VarBinaryVector bytes = (VarBinaryVector) images.getChild("bytes"); + VarCharVector paths = (VarCharVector) images.getChild("path"); + for (int i = 0; i < ROWS; i++) { + List image = expectedImage(i); + // a null struct's required field still holds a value in Arrow; only the struct is null + bytes.setSafe(i, imageBytes(i)); + if (image == null) { + images.setNull(i); + paths.setNull(i); + continue; + } + images.setIndexDefined(i); + if (image.get(1) == null) { + paths.setNull(i); + } else { + paths.setSafe(i, ((String) image.get(1)).getBytes(StandardCharsets.UTF_8)); + } + } + bytes.setValueCount(ROWS); + paths.setValueCount(ROWS); + images.setValueCount(ROWS); + } + + private static void fillEmbeddings(FixedSizeListVector embeddings) { + embeddings.allocateNew(); + Float4Vector items = (Float4Vector) embeddings.getDataVector(); + for (int i = 0; i < ROWS; i++) { + List embedding = expectedEmbedding(i); + for (int j = 0; j < DIM; j++) { + items.setSafe(i * DIM + j, embedding == null ? 0f : embedding.get(j)); + } + if (embedding == null) { + embeddings.setNull(i); + } else { + embeddings.setNotNull(i); + } + } + items.setValueCount(ROWS * DIM); + embeddings.setValueCount(ROWS); + } + + private static void fillTurns(ListVector conversations) { + conversations.allocateNew(); + StructVector turns = (StructVector) conversations.getDataVector(); + VarCharVector questions = (VarCharVector) turns.getChild("question"); + VarCharVector answers = (VarCharVector) turns.getChild("answer"); + int element = 0; + for (int i = 0; i < ROWS; i++) { + conversations.startNewValue(i); + List> rowTurns = expectedTurns(i); + for (List rowTurn : rowTurns) { + turns.setIndexDefined(element); + questions.setSafe(element, rowTurn.get(0).getBytes(StandardCharsets.UTF_8)); + if (rowTurn.get(1) == null) { + answers.setNull(element); + } else { + answers.setSafe(element, rowTurn.get(1).getBytes(StandardCharsets.UTF_8)); + } + element++; + } + conversations.endValue(i, rowTurns.size()); + } + questions.setValueCount(element); + answers.setValueCount(element); + turns.setValueCount(element); + conversations.setValueCount(ROWS); + } +} diff --git a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java index 7d99e91d3..083da8739 100644 --- a/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java +++ b/parquet/src/main/java/io/github/dfa1/vortex/parquet/ParquetExporter.java @@ -1,6 +1,7 @@ package io.github.dfa1.vortex.parquet; import dev.hardwood.OutputFile; +import dev.hardwood.Validity; import dev.hardwood.metadata.LogicalType; import dev.hardwood.metadata.PhysicalType; import dev.hardwood.metadata.RepetitionType; @@ -8,6 +9,7 @@ import dev.hardwood.writer.ColumnBatch; import dev.hardwood.writer.ColumnWriter; import dev.hardwood.writer.ParquetFileWriter; +import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.io.IoBounds; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; @@ -25,12 +27,17 @@ import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.DecimalArray; import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FixedSizeListArray; import io.github.dfa1.vortex.reader.array.Float16Array; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.ListArray; +import io.github.dfa1.vortex.reader.array.ListViewArray; import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MapArray; import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.ShortArray; +import io.github.dfa1.vortex.reader.array.StructArray; import io.github.dfa1.vortex.reader.array.VarBinArray; import java.io.IOException; @@ -43,14 +50,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Consumer; /// Reads a Vortex file and writes a Parquet file. /// -/// Supports flat schemas only: every top-level column must be `Bool`, a `Primitive`, -/// `Utf8`, `Binary`, or a `vortex.timestamp` extension over millisecond/microsecond/nanosecond -/// resolution. `Struct`, `List`, `Map`, and any extension other than `vortex.timestamp` -/// throw [UnsupportedOperationException] — the inverse direction ([ParquetImporter]) supports -/// nested `LIST`/`STRUCT`; export does not yet. +/// Every leaf must be `Bool`, a `Primitive`, `Decimal`, `Utf8`, `Binary`, or a `vortex.timestamp` +/// extension over millisecond/microsecond/nanosecond resolution, at the top level or nested in +/// `Struct`, `List`, `FixedSizeList` and `Map` columns, which export as Parquet groups. Any other +/// type throws [VortexException] naming the column. /// /// Supported Vortex → Parquet type mapping: /// - Bool → BOOLEAN @@ -62,12 +69,19 @@ /// - Decimal → DECIMAL on INT32 (precision up to 9), INT64 (up to 18) or FIXED_LEN_BYTE_ARRAY /// - `vortex.timestamp` → INT64 (TIMESTAMP, `isAdjustedToUTC` set from whether the column /// carries a timezone) +/// - Struct → group, List and FixedSizeList → `LIST`, Map → `MAP` (the key `REQUIRED`, as Parquet +/// requires) public final class ParquetExporter { // the most digits an unscaled decimal holds in a Parquet INT32 / INT64 private static final int MAX_INT32_DIGITS = 9; private static final int MAX_INT64_DIGITS = 18; + // where Hardwood puts a list element and a map key and value under their group + private static final String LIST_ELEMENT = ".list.element"; + private static final String MAP_KEY = ".key_value.key"; + private static final String MAP_VALUE = ".key_value.value"; + private ParquetExporter() { } @@ -137,10 +151,13 @@ public static void exportParquet(VortexHandle vortex, Path parquetPath, ExportOp while (scan.hasNext()) { try (Chunk chunk = scan.next()) { int rowCount = IoBounds.checkCount(chunk.rowCount()); + int[] rows = new int[rowCount]; + Arrays.setAll(rows, i -> i); columnWriter.writeBatch(batch -> { + int leaf = 0; for (int c = 0; c < names.size(); c++) { Array array = chunk.column(names.get(c)); - writeColumn(batch, c, types.get(c), array, rowCount); + leaf = writeNode(batch, leaf, names.get(c).value(), types.get(c), array, rows, null); } }); rowsDone += rowCount; @@ -166,36 +183,76 @@ static List resolveTypes(List allNames, List allTypes, return result; } + + /// Declares top-level column `name` of `type` on `builder`. static void addColumn(FileSchema.Builder builder, String name, DType type) { + requireUndotted(name, type); + declare(new RootSlot(builder, name), name, type); + } + + /// Hardwood addresses a struct, list or map by its dotted path, so a `.` in the name of one would + /// point at another node. A leaf is addressed by index, so its name may hold one. + private static void requireUndotted(String name, DType type) { + boolean group = type instanceof DType.Struct || type instanceof DType.List + || type instanceof DType.FixedSizeList || type instanceof DType.Map; + if (group && name.contains(".")) { + throw new VortexException("Parquet export cannot address a nested column whose name holds a '.': " + name); + } + } + + /// Declares the Parquet node of `type` in `slot`; `path` is its dotted Parquet path, the address + /// [#writeNode] fills its groups by, and the column an unsupported type is reported against. + private static void declare(Slot slot, String path, DType type) { RepetitionType rep = type.nullable() ? RepetitionType.OPTIONAL : RepetitionType.REQUIRED; switch (type) { - case DType.Bool _ -> builder.addColumn(name, PhysicalType.BOOLEAN, rep); - case DType.Utf8 _ -> - builder.addColumn(name, PhysicalType.BYTE_ARRAY, rep, new LogicalType.StringType()); - case DType.Binary _ -> builder.addColumn(name, PhysicalType.BYTE_ARRAY, rep); - case DType.Primitive p -> addPrimitiveColumn(builder, name, p.ptype(), rep); - case DType.Decimal d -> addDecimalColumn(builder, name, d, rep); - case DType.Extension ext -> addTimestampColumn(builder, name, ext, rep); - default -> throw new UnsupportedOperationException( - "unsupported column type for Parquet export (STRUCT/LIST/MAP not yet supported): " + type - + " (column: " + name + ")"); + case DType.Struct s -> slot.struct(rep, fields -> { + for (int f = 0; f < s.fieldNames().size(); f++) { + String name = s.fieldNames().get(f).value(); + requireUndotted(name, s.fieldTypes().get(f)); + declare(new FieldSlot(fields, name), path + "." + name, s.fieldTypes().get(f)); + } + }); + case DType.List l -> + slot.list(rep, element -> declare(new ElementSlot(element), path + LIST_ELEMENT, l.elementType())); + case DType.FixedSizeList l -> + slot.list(rep, element -> declare(new ElementSlot(element), path + LIST_ELEMENT, l.elementType())); + case DType.Map m -> slot.map(leafOf(path + MAP_KEY, m.keyType()), rep, + value -> declare(new ElementSlot(value), path + MAP_VALUE, m.valueType())); + default -> slot.leaf(leafOf(path, type), rep); } } + /// A Parquet primitive: its physical type, the length of a `FIXED_LEN_BYTE_ARRAY` (else `0`) + /// and its annotation (`null` for none). + private record Leaf(PhysicalType type, int typeLength, LogicalType logical) { + } + + private static Leaf leafOf(String path, DType type) { + return switch (type) { + case DType.Bool _ -> new Leaf(PhysicalType.BOOLEAN, 0, null); + case DType.Utf8 _ -> new Leaf(PhysicalType.BYTE_ARRAY, 0, new LogicalType.StringType()); + case DType.Binary _ -> new Leaf(PhysicalType.BYTE_ARRAY, 0, null); + case DType.Primitive p -> primitiveLeaf(p.ptype()); + case DType.Decimal d -> decimalLeaf(d); + case DType.Extension ext -> timestampLeaf(path, ext); + default -> throw new VortexException( + "unsupported column type for Parquet export: " + type + " (column: " + path + ")"); + }; + } + /// Maps a decimal to Parquet's `DECIMAL` annotation on the narrowest physical type holding its /// unscaled value: `INT32` up to 9 digits, `INT64` up to 18, else a big-endian two's-complement /// `FIXED_LEN_BYTE_ARRAY` just wide enough for the precision. The importer reads each of them back /// as the same Vortex decimal. - private static void addDecimalColumn(FileSchema.Builder builder, String name, DType.Decimal d, - RepetitionType rep) { + private static Leaf decimalLeaf(DType.Decimal d) { LogicalType.DecimalType logical = new LogicalType.DecimalType(d.precision(), d.scale()); if (d.precision() <= MAX_INT32_DIGITS) { - builder.addColumn(name, PhysicalType.INT32, rep, logical); - } else if (d.precision() <= MAX_INT64_DIGITS) { - builder.addColumn(name, PhysicalType.INT64, rep, logical); - } else { - builder.addColumn(name, PhysicalType.FIXED_LEN_BYTE_ARRAY, rep, decimalByteWidth(d.precision()), logical); + return new Leaf(PhysicalType.INT32, 0, logical); + } + if (d.precision() <= MAX_INT64_DIGITS) { + return new Leaf(PhysicalType.INT64, 0, logical); } + return new Leaf(PhysicalType.FIXED_LEN_BYTE_ARRAY, decimalByteWidth(d.precision()), logical); } /// The fewest bytes whose two's-complement range holds every `precision`-digit unscaled value. @@ -204,90 +261,410 @@ private static int decimalByteWidth(int precision) { return (bits + 7) / 8; } - private static void addPrimitiveColumn(FileSchema.Builder builder, String name, PType ptype, RepetitionType rep) { - switch (ptype) { - case I8 -> builder.addColumn(name, PhysicalType.INT32, rep, new LogicalType.IntType(8, true)); - case U8 -> builder.addColumn(name, PhysicalType.INT32, rep, new LogicalType.IntType(8, false)); - case I16 -> builder.addColumn(name, PhysicalType.INT32, rep, new LogicalType.IntType(16, true)); - case U16 -> builder.addColumn(name, PhysicalType.INT32, rep, new LogicalType.IntType(16, false)); - case I32 -> builder.addColumn(name, PhysicalType.INT32, rep); - case U32 -> builder.addColumn(name, PhysicalType.INT32, rep, new LogicalType.IntType(32, false)); - case I64 -> builder.addColumn(name, PhysicalType.INT64, rep); - case U64 -> builder.addColumn(name, PhysicalType.INT64, rep, new LogicalType.IntType(64, false)); - case F32 -> builder.addColumn(name, PhysicalType.FLOAT, rep); - case F64 -> builder.addColumn(name, PhysicalType.DOUBLE, rep); + private static Leaf primitiveLeaf(PType ptype) { + return switch (ptype) { + case I8 -> new Leaf(PhysicalType.INT32, 0, new LogicalType.IntType(8, true)); + case U8 -> new Leaf(PhysicalType.INT32, 0, new LogicalType.IntType(8, false)); + case I16 -> new Leaf(PhysicalType.INT32, 0, new LogicalType.IntType(16, true)); + case U16 -> new Leaf(PhysicalType.INT32, 0, new LogicalType.IntType(16, false)); + case I32 -> new Leaf(PhysicalType.INT32, 0, null); + case U32 -> new Leaf(PhysicalType.INT32, 0, new LogicalType.IntType(32, false)); + case I64 -> new Leaf(PhysicalType.INT64, 0, null); + case U64 -> new Leaf(PhysicalType.INT64, 0, new LogicalType.IntType(64, false)); + case F32 -> new Leaf(PhysicalType.FLOAT, 0, null); + case F64 -> new Leaf(PhysicalType.DOUBLE, 0, null); // Parquet's FLOAT16 is a 2-byte FIXED_LEN_BYTE_ARRAY holding the little-endian half - case F16 -> builder.addColumn(name, PhysicalType.FIXED_LEN_BYTE_ARRAY, rep, 2, new LogicalType.Float16Type()); - } + case F16 -> new Leaf(PhysicalType.FIXED_LEN_BYTE_ARRAY, 2, new LogicalType.Float16Type()); + }; } /// Maps a `vortex.timestamp` extension to Parquet's `TIMESTAMP` logical type. `Seconds` and /// `Days` have no Parquet `TIMESTAMP` equivalent (the format only defines MILLIS/MICROS/NANOS /// resolution) and throw; any other extension id throws, since only `vortex.timestamp` is /// supported for export. - private static void addTimestampColumn(FileSchema.Builder builder, String name, DType.Extension ext, - RepetitionType rep) { + private static Leaf timestampLeaf(String path, DType.Extension ext) { if (ext.extensionId() != ExtensionId.VORTEX_TIMESTAMP) { - throw new UnsupportedOperationException( - "unsupported extension type for Parquet export: " + ext.extensionId() + " (column: " + name + ")"); + throw new VortexException( + "unsupported extension type for Parquet export: " + ext.extensionId() + " (column: " + path + ")"); } TimeUnit unit = TimestampDtype.readUnit(ext); LogicalType.TimeUnit parquetUnit = switch (unit) { case Milliseconds -> LogicalType.TimeUnit.MILLIS; case Microseconds -> LogicalType.TimeUnit.MICROS; case Nanoseconds -> LogicalType.TimeUnit.NANOS; - case Seconds, Days -> throw new UnsupportedOperationException( - "Parquet TIMESTAMP has no " + unit + " resolution (column: " + name + ")"); + case Seconds, Days -> throw new VortexException( + "Parquet TIMESTAMP has no " + unit + " resolution (column: " + path + ")"); }; boolean isAdjustedToUtc = TimestampDtype.timezone(ext).isPresent(); - builder.addColumn(name, PhysicalType.INT64, rep, new LogicalType.TimestampType(isAdjustedToUtc, parquetUnit)); + return new Leaf(PhysicalType.INT64, 0, new LogicalType.TimestampType(isAdjustedToUtc, parquetUnit)); + } + + /// Where [#declare] puts one Parquet node, so one walk serves Hardwood's three builders: a + /// top-level column, a struct field, and a list element or map value. + private interface Slot { + + void leaf(Leaf leaf, RepetitionType rep); + + void struct(RepetitionType rep, Consumer fields); + + void list(RepetitionType rep, Consumer element); + + void map(Leaf key, RepetitionType rep, Consumer value); + } + + private record RootSlot(FileSchema.Builder builder, String name) implements Slot { + + @Override + public void leaf(Leaf leaf, RepetitionType rep) { + if (leaf.typeLength() == 0) { + builder.addColumn(name, leaf.type(), rep, leaf.logical()); + } else { + builder.addColumn(name, leaf.type(), rep, leaf.typeLength(), leaf.logical()); + } + } + + @Override + public void struct(RepetitionType rep, Consumer fields) { + builder.struct(name, rep, fields); + } + + @Override + public void list(RepetitionType rep, Consumer element) { + builder.list(name, rep, element); + } + + @Override + public void map(Leaf key, RepetitionType rep, Consumer value) { + if (key.typeLength() == 0) { + builder.map(name, rep, key.type(), key.logical(), value); + } else { + builder.map(name, rep, key.type(), key.typeLength(), key.logical(), value); + } + } + } + + private record FieldSlot(FileSchema.StructBuilder builder, String name) implements Slot { + + @Override + public void leaf(Leaf leaf, RepetitionType rep) { + if (leaf.typeLength() == 0) { + builder.addColumn(name, leaf.type(), rep, leaf.logical()); + } else { + builder.addColumn(name, leaf.type(), rep, leaf.typeLength(), leaf.logical()); + } + } + + @Override + public void struct(RepetitionType rep, Consumer fields) { + builder.struct(name, rep, fields); + } + + @Override + public void list(RepetitionType rep, Consumer element) { + builder.list(name, rep, element); + } + + @Override + public void map(Leaf key, RepetitionType rep, Consumer value) { + if (key.typeLength() == 0) { + builder.map(name, rep, key.type(), key.logical(), value); + } else { + builder.map(name, rep, key.type(), key.typeLength(), key.logical(), value); + } + } + } + + private record ElementSlot(FileSchema.ElementBuilder builder) implements Slot { + + @Override + public void leaf(Leaf leaf, RepetitionType rep) { + if (leaf.typeLength() == 0) { + builder.primitive(leaf.type(), rep, leaf.logical()); + } else { + builder.primitive(leaf.type(), rep, leaf.typeLength(), leaf.logical()); + } + } + + @Override + public void struct(RepetitionType rep, Consumer fields) { + builder.struct(rep, fields); + } + + @Override + public void list(RepetitionType rep, Consumer element) { + builder.list(rep, element); + } + + @Override + public void map(Leaf key, RepetitionType rep, Consumer value) { + if (key.typeLength() == 0) { + builder.map(rep, key.type(), key.logical(), value); + } else { + builder.map(rep, key.type(), key.typeLength(), key.logical(), value); + } + } } - /// Writes one column's values for the current batch, addressed by leaf index `idx` - /// (matching the schema order [#addColumn] built it in). A nullable column decodes as a - /// [MaskedArray]; its per-row nulls are read once into a `boolean[]` mask and its inner - /// (unmasked) array supplies the values — the value at a null row is still read rather than - /// skipped (Hardwood ignores it), keeping the read loop branch-free per row. - static void writeColumn(ColumnBatch batch, int idx, DType type, Array array, int rowCount) { - Array target = array; + /// Writes the columns of the node [#declare] built for `type` at `path`: the values at `rows` of + /// `array`, its leaves numbered from `leaf` in schema order, plus the nulls of its structs and the + /// offsets of its lists. `absent` marks the rows under a null struct (`null` for none), where + /// Parquet allows no list entries. + /// + /// Rows are gathered by index, so a list's elements, a list view's out-of-order ranges and the + /// entries a null list skips need no copy of the arrays. + /// + /// @return the index of the leaf after this node's + private static int writeNode(ColumnBatch batch, int leaf, String path, DType type, Array array, int[] rows, + boolean[] absent) { + return switch (type) { + case DType.Struct s -> writeStruct(batch, leaf, path, s, array, rows, absent); + case DType.List l -> writeList(batch, leaf, path, l, l.elementType(), array, rows, absent); + case DType.FixedSizeList l -> writeList(batch, leaf, path, l, l.elementType(), array, rows, absent); + case DType.Map m -> writeMap(batch, leaf, path, m, array, rows, absent); + default -> { + writeLeaf(batch, leaf, type, array, rows); + yield leaf + 1; + } + }; + } + + private static int writeStruct(ColumnBatch batch, int leaf, String path, DType.Struct type, Array array, + int[] rows, boolean[] absent) { + StructArray struct = structOf(type, array); + boolean[] fieldAbsent = absent; + if (type.nullable()) { + boolean[] nulls = new boolean[rows.length]; + for (int i = 0; i < rows.length; i++) { + nulls[i] = !isValid(array, rows[i]) || allFieldsNull(struct, rows[i]); + } + batch.struct(path, Validity.ofNulls(nulls)); + // a copy: Hardwood keeps `nulls` as the struct's own validity until the batch is written + fieldAbsent = nulls.clone(); + if (absent != null) { + for (int i = 0; i < rows.length; i++) { + fieldAbsent[i] |= absent[i]; + } + } + } + int next = leaf; + for (int f = 0; f < struct.fieldCount(); f++) { + next = writeNode(batch, next, path + "." + type.fieldNames().get(f).value(), type.fieldTypes().get(f), + struct.field(f), rows, fieldAbsent); + } + return next; + } + + /// The fields of a struct column. The `vortex.struct` decoder returns a one-field struct as its + /// lone field (masked with the struct's validity), so an array of another shape is that field. + private static StructArray structOf(DType.Struct type, Array array) { + if (unmasked(array) instanceof StructArray struct + && ((DType.Struct) struct.dtype()).fieldNames().equals(type.fieldNames())) { + return struct; + } + if (type.fieldTypes().size() != 1) { + throw new VortexException("struct column decoded to " + array.getClass().getSimpleName()); + } + return new StructArray(type, array.length(), List.of(array)); + } + + /// Whether struct row `row` is null. A [StructArray] carries no validity of its own: the decoder + /// masks every field with it, so a row where every field is masked invalid is a null struct. A + /// row whose fields are each null on their own reads the same; `CsvExporter` takes the same + /// reading. + private static boolean allFieldsNull(StructArray struct, int row) { + for (int f = 0; f < struct.fieldCount(); f++) { + if (!(struct.field(f) instanceof MaskedArray) || isValid(struct.field(f), row)) { + return false; + } + } + return struct.fieldCount() > 0; + } + + private static int writeList(ColumnBatch batch, int leaf, String path, DType type, DType elementType, + Array array, int[] rows, boolean[] absent) { + boolean[] nulls = type.nullable() ? nullRows(array, rows) : null; + Array lists = unmasked(array); + int[] offsets = new int[rows.length + 1]; + int[] elementRows = entryRows(lists, rows, nulls, absent, offsets); + if (nulls == null) { + batch.list(path, offsets); + } else { + batch.list(path, offsets, Validity.ofNulls(nulls)); + } + return writeNode(batch, leaf, path + LIST_ELEMENT, elementType, elementsOf(lists), elementRows, null); + } + + /// A map is a list of `{key, value}` entry structs ([MapArray]), null where either the map or its + /// entries list is masked. + private static int writeMap(ColumnBatch batch, int leaf, String path, DType.Map type, Array array, int[] rows, + boolean[] absent) { + if (!(unmasked(array) instanceof MapArray map)) { + throw new VortexException("map column decoded to " + array.getClass().getSimpleName() + " (column: " + path + ")"); + } + Array entries = map.entries(); boolean[] nulls = null; - if (array instanceof MaskedArray masked) { - target = masked.inner(); - nulls = readNulls(masked, rowCount); + if (type.nullable()) { + nulls = nullRows(array, rows); + for (int i = 0; i < rows.length; i++) { + nulls[i] |= !isValid(entries, rows[i]); + } } - switch (type) { - case DType.Bool _ -> writeBooleans(batch, idx, target, rowCount, nulls); - case DType.Utf8 _ -> writeBytes(batch, idx, target, rowCount, nulls); - case DType.Binary _ -> writeBytes(batch, idx, target, rowCount, nulls); - case DType.Primitive p -> writePrimitive(batch, idx, p.ptype(), target, rowCount, nulls); - case DType.Decimal d -> writeDecimals(batch, idx, d, (DecimalArray) target, rowCount, nulls); - case DType.Extension _ -> writeLongs(batch, idx, target, rowCount, nulls); - default -> throw new UnsupportedOperationException("unsupported column type for Parquet export: " + type); + Array lists = unmasked(entries); + int[] offsets = new int[rows.length + 1]; + int[] entryRows = entryRows(lists, rows, nulls, absent, offsets); + if (nulls == null) { + batch.map(path, offsets); + } else { + batch.map(path, offsets, Validity.ofNulls(nulls)); + } + if (!(unmasked(elementsOf(lists)) instanceof StructArray keyValue) || keyValue.fieldCount() != 2) { + throw new VortexException("map entries are not {key, value} structs (column: " + path + ")"); } + int next = writeNode(batch, leaf, path + MAP_KEY, type.keyType(), keyValue.field(0), + entryRows, null); + return writeNode(batch, next, path + MAP_VALUE, type.valueType(), keyValue.field(1), entryRows, null); } - private static boolean[] readNulls(MaskedArray masked, int rowCount) { - boolean[] nulls = new boolean[rowCount]; - for (int i = 0; i < rowCount; i++) { - nulls[i] = !masked.isValid(i); + /// The element rows of the lists at `rows`, in order, recording in `offsets` where each list's + /// entries start. A list that is null or `absent` gets none: Parquet stores no entries under a + /// null, while a Vortex null list may still span some. The ranges come from the file, so one + /// outside the elements throws here rather than as an index error while reading them. + private static int[] entryRows(Array lists, int[] rows, boolean[] nulls, boolean[] absent, int[] offsets) { + long elementCount = elementsOf(lists).length(); + long[] starts = new long[rows.length]; + long total = 0; + for (int i = 0; i < rows.length; i++) { + offsets[i] = IoBounds.checkCount(total); + if ((nulls == null || !nulls[i]) && (absent == null || !absent[i])) { + long start = listStart(lists, rows[i]); + long size = listSize(lists, rows[i]); + if (start < 0 || size < 0 || start + size > elementCount) { + throw new VortexException("list row " + rows[i] + " spans elements [" + start + ", " + + (start + size) + ") outside the " + elementCount + " it has"); + } + IoBounds.checkCount(start + size); + starts[i] = start; + total += size; + } + } + offsets[rows.length] = IoBounds.checkCount(total); + int[] elementRows = new int[offsets[rows.length]]; + for (int i = 0; i < rows.length; i++) { + int base = (int) starts[i] - offsets[i]; + for (int j = offsets[i]; j < offsets[i + 1]; j++) { + elementRows[j] = base + j; + } + } + return elementRows; + } + + private static long listStart(Array lists, int row) { + return switch (lists) { + case ListArray l -> offsetAt(l.offsets(), row); + case ListViewArray l -> offsetAt(l.offsets(), row); + case FixedSizeListArray l -> (long) row * l.fixedSize(); + default -> throw unexpectedList(lists); + }; + } + + private static long listSize(Array lists, int row) { + return switch (lists) { + case ListArray l -> offsetAt(l.offsets(), row + 1L) - offsetAt(l.offsets(), row); + case ListViewArray l -> offsetAt(l.sizes(), row); + case FixedSizeListArray l -> l.fixedSize(); + default -> throw unexpectedList(lists); + }; + } + + private static Array elementsOf(Array lists) { + return switch (lists) { + case ListArray l -> l.elements(); + case ListViewArray l -> l.elements(); + case FixedSizeListArray l -> l.elements(); + default -> throw unexpectedList(lists); + }; + } + + private static VortexException unexpectedList(Array lists) { + return new VortexException("unexpected list array: " + lists.getClass().getSimpleName()); + } + + /// Reads list offset `idx` as a non-negative `long`. The encoder picks the narrowest ptype that + /// holds the largest offset, so any integer width can appear; `ByteArray`/`ShortArray#getInt` + /// already zero-extend U8/U16. + private static long offsetAt(Array offsets, long idx) { + return switch (offsets) { + case LongArray la -> la.getLong(idx); + case IntArray ia -> Integer.toUnsignedLong(ia.getInt(idx)); + case ShortArray sa -> sa.getInt(idx); + case ByteArray ba -> ba.getInt(idx); + default -> throw new VortexException("unexpected list offsets type: " + offsets.getClass().getSimpleName()); + }; + } + + /// Whether row `row` of `array` is valid under every [MaskedArray] wrapping it: a struct field + /// carries its struct's validity over its own. + private static boolean isValid(Array array, long row) { + Array current = array; + while (current instanceof MaskedArray masked) { + if (!masked.isValid(row)) { + return false; + } + current = masked.inner(); + } + return true; + } + + private static Array unmasked(Array array) { + Array current = array; + while (current instanceof MaskedArray masked) { + current = masked.inner(); + } + return current; + } + + private static boolean[] nullRows(Array array, int[] rows) { + boolean[] nulls = new boolean[rows.length]; + for (int i = 0; i < rows.length; i++) { + nulls[i] = !isValid(array, rows[i]); } return nulls; } - private static void writePrimitive(ColumnBatch batch, int idx, PType ptype, Array target, int rowCount, + /// Writes one leaf column's values at `rows` of `array`, addressed by leaf index `idx` (matching + /// the schema order [#declare] built it in). A nullable leaf decodes as a [MaskedArray]; its nulls + /// are read once into a `boolean[]` mask and its inner (unmasked) array supplies the values — the + /// value at a null row is still read rather than skipped (Hardwood ignores it), keeping the read + /// loop branch-free per row. A `REQUIRED` leaf may be masked too, by a nullable struct above it, + /// whose nulls Hardwood takes from the struct instead. + private static void writeLeaf(ColumnBatch batch, int idx, DType type, Array array, int[] rows) { + Array target = unmasked(array); + boolean[] nulls = type.nullable() && array instanceof MaskedArray ? nullRows(array, rows) : null; + switch (type) { + case DType.Bool _ -> writeBooleans(batch, idx, target, rows, nulls); + case DType.Utf8 _ -> writeBytes(batch, idx, target, rows, nulls); + case DType.Binary _ -> writeBytes(batch, idx, target, rows, nulls); + case DType.Primitive p -> writePrimitive(batch, idx, p.ptype(), target, rows, nulls); + case DType.Decimal d -> writeDecimals(batch, idx, d, (DecimalArray) target, rows, nulls); + case DType.Extension _ -> writeLongs(batch, idx, target, rows, nulls); + default -> throw new VortexException("unsupported column type for Parquet export: " + type); + } + } + + private static void writePrimitive(ColumnBatch batch, int idx, PType ptype, Array target, int[] rows, boolean[] nulls) { switch (ptype) { case I8, U8, I16, U16, I32, U32 -> { - int[] values = readInts(target, rowCount); + int[] values = readInts(target, rows); if (nulls == null) { batch.ints(idx, values); } else { batch.ints(idx, values, nulls); } } - case I64, U64 -> writeLongs(batch, idx, target, rowCount, nulls); + case I64, U64 -> writeLongs(batch, idx, target, rows, nulls); case F32 -> { - float[] values = readFloats((FloatArray) target, rowCount); + float[] values = readFloats((FloatArray) target, rows); if (nulls == null) { batch.floats(idx, values); } else { @@ -295,7 +672,7 @@ private static void writePrimitive(ColumnBatch batch, int idx, PType ptype, Arra } } case F64 -> { - double[] values = readDoubles((DoubleArray) target, rowCount); + double[] values = readDoubles((DoubleArray) target, rows); if (nulls == null) { batch.doubles(idx, values); } else { @@ -303,7 +680,7 @@ private static void writePrimitive(ColumnBatch batch, int idx, PType ptype, Arra } } case F16 -> { - byte[][] values = readHalves((Float16Array) target, rowCount); + byte[][] values = readHalves((Float16Array) target, rows); if (nulls == null) { batch.fixed(idx, values); } else { @@ -313,20 +690,21 @@ private static void writePrimitive(ColumnBatch batch, int idx, PType ptype, Arra } } - /// Writes a decimal column as its unscaled integer, on the physical type [#addDecimalColumn] chose + /// Writes a decimal column as its unscaled integer, on the physical type [#decimalLeaf] chose /// for the precision. A null row writes a zero instead of reading the masked slot. private static void writeDecimals(ColumnBatch batch, int idx, DType.Decimal type, DecimalArray arr, - int rowCount, boolean[] nulls) { + int[] rows, boolean[] nulls) { + int count = rows.length; if (type.precision() <= MAX_INT64_DIGITS) { - long[] unscaled = new long[rowCount]; - for (int i = 0; i < rowCount; i++) { + long[] unscaled = new long[count]; + for (int i = 0; i < count; i++) { if (nulls == null || !nulls[i]) { - unscaled[i] = unscaledValue(arr.getDecimal(i), type).longValueExact(); + unscaled[i] = unscaledValue(arr.getDecimal(rows[i]), type).longValueExact(); } } if (type.precision() <= MAX_INT32_DIGITS) { - int[] values = new int[rowCount]; - for (int i = 0; i < rowCount; i++) { + int[] values = new int[count]; + for (int i = 0; i < count; i++) { values[i] = (int) unscaled[i]; } if (nulls == null) { @@ -342,11 +720,11 @@ private static void writeDecimals(ColumnBatch batch, int idx, DType.Decimal type return; } int width = decimalByteWidth(type.precision()); - byte[][] values = new byte[rowCount][]; - for (int i = 0; i < rowCount; i++) { + byte[][] values = new byte[count][]; + for (int i = 0; i < count; i++) { BigInteger unscaled = BigInteger.ZERO; if (nulls == null || !nulls[i]) { - unscaled = unscaledValue(arr.getDecimal(i), type); + unscaled = unscaledValue(arr.getDecimal(rows[i]), type); } values[i] = signExtend(unscaled.toByteArray(), width); } @@ -374,8 +752,8 @@ private static byte[] signExtend(byte[] bytes, int width) { return result; } - private static void writeBooleans(ColumnBatch batch, int idx, Array target, int rowCount, boolean[] nulls) { - boolean[] values = readBooleans((BoolArray) target, rowCount); + private static void writeBooleans(ColumnBatch batch, int idx, Array target, int[] rows, boolean[] nulls) { + boolean[] values = readBooleans((BoolArray) target, rows); if (nulls == null) { batch.booleans(idx, values); } else { @@ -383,8 +761,8 @@ private static void writeBooleans(ColumnBatch batch, int idx, Array target, int } } - private static void writeBytes(ColumnBatch batch, int idx, Array target, int rowCount, boolean[] nulls) { - byte[][] values = readBytes((VarBinArray) target, rowCount); + private static void writeBytes(ColumnBatch batch, int idx, Array target, int[] rows, boolean[] nulls) { + byte[][] values = readBytes((VarBinArray) target, rows); if (nulls == null) { batch.bytes(idx, values); } else { @@ -392,8 +770,8 @@ private static void writeBytes(ColumnBatch batch, int idx, Array target, int row } } - private static void writeLongs(ColumnBatch batch, int idx, Array target, int rowCount, boolean[] nulls) { - long[] values = readLongs((LongArray) target, rowCount); + private static void writeLongs(ColumnBatch batch, int idx, Array target, int[] rows, boolean[] nulls) { + long[] values = readLongs((LongArray) target, rows); if (nulls == null) { batch.longs(idx, values); } else { @@ -406,22 +784,22 @@ private static void writeLongs(ColumnBatch batch, int idx, Array target, int row /// correct value for the column's own signedness (sign-extended for a signed narrow type, /// zero-extended for unsigned), so the raw widened value is exactly what Parquet's matching /// `IntType` annotation expects — no further conversion needed. - private static int[] readInts(Array arr, int rowCount) { - int[] values = new int[rowCount]; + private static int[] readInts(Array arr, int[] rows) { + int[] values = new int[rows.length]; switch (arr) { case IntArray ia -> { - for (int i = 0; i < rowCount; i++) { - values[i] = ia.getInt(i); + for (int i = 0; i < rows.length; i++) { + values[i] = ia.getInt(rows[i]); } } case ShortArray sa -> { - for (int i = 0; i < rowCount; i++) { - values[i] = sa.getInt(i); + for (int i = 0; i < rows.length; i++) { + values[i] = sa.getInt(rows[i]); } } case ByteArray ba -> { - for (int i = 0; i < rowCount; i++) { - values[i] = ba.getInt(i); + for (int i = 0; i < rows.length; i++) { + values[i] = ba.getInt(rows[i]); } } default -> throw new IllegalStateException( @@ -430,55 +808,56 @@ private static int[] readInts(Array arr, int rowCount) { return values; } - private static long[] readLongs(LongArray arr, int rowCount) { - long[] values = new long[rowCount]; - for (int i = 0; i < rowCount; i++) { - values[i] = arr.getLong(i); + private static long[] readLongs(LongArray arr, int[] rows) { + long[] values = new long[rows.length]; + for (int i = 0; i < rows.length; i++) { + values[i] = arr.getLong(rows[i]); } return values; } /// One 2-byte little-endian entry per row, copied from the half's raw bits so a NaN payload or /// `-0.0` is written as it is held, not rounded through `float`. - private static byte[][] readHalves(Float16Array arr, int rowCount) { - byte[][] values = new byte[rowCount][]; + private static byte[][] readHalves(Float16Array arr, int[] rows) { + byte[][] values = new byte[rows.length][]; try (Arena arena = Arena.ofConfined()) { MemorySegment bits = arr.materialize(arena); - for (int i = 0; i < rowCount; i++) { - values[i] = new byte[]{bits.get(ValueLayout.JAVA_BYTE, i * 2L), bits.get(ValueLayout.JAVA_BYTE, i * 2L + 1)}; + for (int i = 0; i < rows.length; i++) { + long at = rows[i] * 2L; + values[i] = new byte[]{bits.get(ValueLayout.JAVA_BYTE, at), bits.get(ValueLayout.JAVA_BYTE, at + 1)}; } } return values; } - private static float[] readFloats(FloatArray arr, int rowCount) { - float[] values = new float[rowCount]; - for (int i = 0; i < rowCount; i++) { - values[i] = arr.getFloat(i); + private static float[] readFloats(FloatArray arr, int[] rows) { + float[] values = new float[rows.length]; + for (int i = 0; i < rows.length; i++) { + values[i] = arr.getFloat(rows[i]); } return values; } - private static double[] readDoubles(DoubleArray arr, int rowCount) { - double[] values = new double[rowCount]; - for (int i = 0; i < rowCount; i++) { - values[i] = arr.getDouble(i); + private static double[] readDoubles(DoubleArray arr, int[] rows) { + double[] values = new double[rows.length]; + for (int i = 0; i < rows.length; i++) { + values[i] = arr.getDouble(rows[i]); } return values; } - private static boolean[] readBooleans(BoolArray arr, int rowCount) { - boolean[] values = new boolean[rowCount]; - for (int i = 0; i < rowCount; i++) { - values[i] = arr.getBoolean(i); + private static boolean[] readBooleans(BoolArray arr, int[] rows) { + boolean[] values = new boolean[rows.length]; + for (int i = 0; i < rows.length; i++) { + values[i] = arr.getBoolean(rows[i]); } return values; } - private static byte[][] readBytes(VarBinArray arr, int rowCount) { - byte[][] values = new byte[rowCount][]; - for (int i = 0; i < rowCount; i++) { - values[i] = arr.getBytes(i); + private static byte[][] readBytes(VarBinArray arr, int[] rows) { + byte[][] values = new byte[rows.length][]; + for (int i = 0; i < rows.length; i++) { + values[i] = arr.getBytes(rows[i]); } return values; } diff --git a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java index 036c532a6..385f60f9f 100644 --- a/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java +++ b/parquet/src/test/java/io/github/dfa1/vortex/parquet/ParquetExporterTest.java @@ -8,6 +8,7 @@ import dev.hardwood.reader.RowReader; import dev.hardwood.schema.ColumnSchema; import dev.hardwood.schema.FileSchema; +import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.core.io.VortexFormat; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; @@ -31,6 +32,11 @@ import io.github.dfa1.vortex.reader.array.VarBinArray; import io.github.dfa1.vortex.writer.VortexWriter; import io.github.dfa1.vortex.writer.WriteOptions; +import io.github.dfa1.vortex.writer.encode.FixedSizeListData; +import io.github.dfa1.vortex.writer.encode.ListData; +import io.github.dfa1.vortex.writer.encode.ListViewData; +import io.github.dfa1.vortex.writer.encode.NullableData; +import io.github.dfa1.vortex.writer.encode.StructData; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -207,7 +213,7 @@ void timestamp_secondsOrDaysResolution_throws(TimeUnit unit) { DType.Extension ts = TimestampDtype.of(unit, null, false); // When / Then - assertThatThrownBy(() -> schemaOf("ts", ts)).isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> schemaOf("ts", ts)).isInstanceOf(VortexException.class).hasMessageContaining("column: ts"); } @Test @@ -217,25 +223,90 @@ void nonTimestampExtension_throws() { io.github.dfa1.vortex.core.model.OpaqueMetadata.EMPTY, false); // When / Then - assertThatThrownBy(() -> schemaOf("u", ext)).isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> schemaOf("u", ext)).isInstanceOf(VortexException.class).hasMessageContaining("column: u"); } @Test - void struct_throws() { - // Given — nested schemas are out of scope for this exporter - DType.Struct nested = new DType.Struct(List.of(ColumnName.of("x")), List.of(DTypes.I32), false); + void nullableStruct_mapsToOptionalGroup_ofItsFields() { + // Given + DType.Struct struct = new DType.Struct(List.of(ColumnName.of("x"), ColumnName.of("y")), + List.of(DTypes.I32, new DType.Utf8(true)), true); - // When / Then - assertThatThrownBy(() -> schemaOf("s", nested)).isInstanceOf(UnsupportedOperationException.class); + // When + FileSchema result = schemaOf("s", struct); + + // Then — the group's nullability raises the leaves' definition level, not their repetition + assertThat(result.getColumn("s.x").repetitionType()).isEqualTo(RepetitionType.REQUIRED); + assertThat(result.getColumn("s.x").maxDefinitionLevel()).isEqualTo(1); + assertThat(result.getColumn("s.y").logicalType()).isEqualTo(new LogicalType.StringType()); + assertThat(result.getColumn("s.y").maxDefinitionLevel()).isEqualTo(2); + } + + @Test + void listAndFixedSizeList_mapToRepeatedListElement() { + // Given — Parquet has no fixed-size list, so both become LIST + DType.List list = new DType.List(DTypes.F32, true); + DType.FixedSizeList fixed = new DType.FixedSizeList(DTypes.I32, 4, false); + FileSchema.Builder builder = FileSchema.builder("schema"); + ParquetExporter.addColumn(builder, "l", list); + ParquetExporter.addColumn(builder, "f", fixed); + + // When + FileSchema result = builder.build(); + + // Then + assertThat(result.getColumn("l.list.element").type()).isEqualTo(PhysicalType.FLOAT); + assertThat(result.getColumn("l.list.element").maxRepetitionLevel()).isEqualTo(1); + assertThat(result.getColumn("f.list.element").type()).isEqualTo(PhysicalType.INT32); + assertThat(result.getColumn("f.list.element").maxRepetitionLevel()).isEqualTo(1); } @Test - void list_throws() { + void map_mapsToRequiredKeyAndValue() { // Given - DType.List list = new DType.List(DTypes.I32, false); + DType.Map map = new DType.Map(DTypes.UTF8, DTypes.I64.asNullable(), false, true); + + // When + FileSchema result = schemaOf("m", map); + + // Then + assertThat(result.getColumn("m.key_value.key").repetitionType()).isEqualTo(RepetitionType.REQUIRED); + assertThat(result.getColumn("m.key_value.key").logicalType()).isEqualTo(new LogicalType.StringType()); + assertThat(result.getColumn("m.key_value.value").repetitionType()).isEqualTo(RepetitionType.OPTIONAL); + } + + @Test + void dottedNestedColumnName_throws() { + // Given — Hardwood addresses a struct by its dotted path, which "a.b" would misdirect + DType.Struct struct = new DType.Struct(List.of(ColumnName.of("x")), List.of(DTypes.I32), false); // When / Then - assertThatThrownBy(() -> schemaOf("l", list)).isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> schemaOf("a.b", struct)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("a.b"); + } + + @Test + void dottedLeafName_isKept() { + // Given — a leaf is addressed by index, so a dot in its name is harmless + DType.Struct struct = new DType.Struct(List.of(ColumnName.of("x.y")), List.of(DTypes.I32), false); + + // When + FileSchema result = schemaOf("s", struct); + + // Then + assertThat(result.getColumn(0).fieldPath().elements()).containsExactly("s", "x.y"); + } + + @Test + void unsupportedNestedLeaf_throws_namingItsPath() { + // Given — a Null dtype has no Parquet leaf; the message must point at the nested column + DType.List list = new DType.List(new DType.Null(true), false); + + // When / Then + assertThatThrownBy(() -> schemaOf("l", list)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("column: l.list.element"); } } @@ -347,11 +418,11 @@ void projection_unknownColumn_throws(@TempDir Path tmp) throws Exception { } @Test - void structColumn_throws(@TempDir Path tmp) throws Exception { - // Given — a top-level Struct column, out of scope for this exporter; the schema + void structWithUnsupportedField_throwsVortexException_namingTheField(@TempDir Path tmp) throws Exception { + // Given — a struct holding a Variant, which has no Parquet mapping here; the schema // alone triggers the rejection (at FileSchema-building time, before any chunk is // read), so the file needs no rows - DType.Struct fieldSchema = new DType.Struct(List.of(ColumnName.of("x")), List.of(DTypes.I32), false); + DType.Struct fieldSchema = new DType.Struct(List.of(ColumnName.of("x")), List.of(DTypes.VARIANT), false); DType.Struct schema = new DType.Struct(List.of(ColumnName.of("s")), List.of(fieldSchema), false); Path vortex = tmp.resolve("in.vortex"); try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE); @@ -363,7 +434,8 @@ void structColumn_throws(@TempDir Path tmp) throws Exception { // When / Then assertThatThrownBy(() -> ParquetExporter.exportParquet(vortex, parquet)) - .isInstanceOf(UnsupportedOperationException.class); + .isInstanceOf(VortexException.class) + .hasMessageContaining("column: s.x"); } } @@ -586,4 +658,165 @@ void decimalColumn_roundTripsThroughParquetAndBack(int precision, @TempDir Path } } } + + @Nested + class NestedColumns { + + private static final DType.Struct IMAGE = new DType.Struct( + List.of(ColumnName.of("bytes"), ColumnName.of("path")), + List.of(DTypes.BINARY, new DType.Utf8(true)), true); + private static final DType.Struct TURN = new DType.Struct( + List.of(ColumnName.of("question"), ColumnName.of("answer")), + List.of(DTypes.UTF8, new DType.Utf8(true)), false); + private static final DType.Struct DOC = new DType.Struct( + List.of(ColumnName.of("tags"), ColumnName.of("n")), + List.of(new DType.List(DTypes.UTF8, false), DTypes.I32), true); + private static final List NAMES = List.of(ColumnName.of("img"), ColumnName.of("emb"), + ColumnName.of("vec"), ColumnName.of("conv"), ColumnName.of("doc")); + private static final List TYPES = List.of(IMAGE, new DType.List(DTypes.F32, true), + new DType.FixedSizeList(DTypes.I32, 2, false), new DType.List(TURN, false), DOC); + + /// The column shapes of #721's Raincloud datasets (`mnist`'s image struct, + /// `cohere-wikipedia-simple-embed`'s embedding list, `coig`'s list of structs), plus the two + /// shapes Parquet forbids entries under: row 1's null `emb` list still spans one element, and + /// row 1's null `doc` struct holds a `tags` list spanning one, so both must be dropped. + private Path writeNested(Path tmp) throws Exception { + boolean[] row1Null = {true, false, true}; + return writeVortex(tmp, "nested.vortex", NAMES, TYPES, Map.of( + ColumnName.of("img"), new NullableData(new StructData(List.of( + new byte[][]{{1}, {}, {2, 3}}, + new NullableData(new String[]{"a.png", "", ""}, new boolean[]{true, false, false}))), row1Null), + ColumnName.of("emb"), new NullableData( + new ListData(new float[]{1f, 2f, 9f}, new long[]{0, 2, 3, 3}, 3), row1Null), + ColumnName.of("vec"), new FixedSizeListData(new int[]{1, 2, 3, 4, 5, 6}, 3), + ColumnName.of("conv"), new ListData(new StructData(List.of( + new String[]{"q1", "q2", "q3"}, + new NullableData(new String[]{"a1", "", "a3"}, new boolean[]{true, false, true}))), + new long[]{0, 2, 2, 3}, 3), + ColumnName.of("doc"), new NullableData(new StructData(List.of( + new ListData(new String[]{"a", "x", "b"}, new long[]{0, 1, 2, 3}, 3), + new int[]{1, 2, 3})), row1Null))); + } + + @Test + void structAndListColumns_exportAsParquetGroups(@TempDir Path tmp) throws Exception { + // Given + Path vortex = writeNested(tmp); + Path parquet = tmp.resolve("out.parquet"); + + // When + ParquetExporter.exportParquet(vortex, parquet); + + // Then + try (ParquetFileReader reader = ParquetFileReader.open(InputFile.of(parquet)); + RowReader rows = reader.buildRowReader().build()) { + rows.next(); + assertThat(rows.getStruct("img").getBinary("bytes")).isEqualTo(new byte[]{1}); + assertThat(rows.getStruct("img").getString("path")).isEqualTo("a.png"); + assertThat(rows.getList("emb").floats()).containsExactly(1f, 2f); + assertThat(rows.getList("vec").values()).containsExactly(1, 2); + List turns = rows.getList("conv").structs(); + assertThat(turns).hasSize(2); + assertThat(turns.get(0).getString("question")).isEqualTo("q1"); + assertThat(turns.get(0).getString("answer")).isEqualTo("a1"); + assertThat(turns.get(1).getString("question")).isEqualTo("q2"); + assertThat(turns.get(1).isNull("answer")).isTrue(); + assertThat(rows.getStruct("doc").getList("tags").strings()).containsExactly("a"); + assertThat(rows.getStruct("doc").getInt("n")).isEqualTo(1); + + rows.next(); + assertThat(rows.isNull("img")).isTrue(); + assertThat(rows.isNull("emb")).isTrue(); + assertThat(rows.getList("vec").values()).containsExactly(3, 4); + assertThat(rows.getList("conv").isEmpty()).isTrue(); + assertThat(rows.isNull("doc")).isTrue(); + + rows.next(); + assertThat(rows.getStruct("img").getBinary("bytes")).isEqualTo(new byte[]{2, 3}); + assertThat(rows.getStruct("img").isNull("path")).isTrue(); + assertThat(rows.getList("emb").isEmpty()).isTrue(); + assertThat(rows.getList("vec").values()).containsExactly(5, 6); + assertThat(rows.getList("conv").structs().getFirst().getString("answer")).isEqualTo("a3"); + assertThat(rows.getStruct("doc").getList("tags").strings()).containsExactly("b"); + assertThat(rows.getStruct("doc").getInt("n")).isEqualTo(3); + assertThat(rows.hasNext()).isFalse(); + } + } + + @Test + void structAndListColumns_roundTripTheirDtypesThroughParquet(@TempDir Path tmp) throws Exception { + // Given + Path vortex = writeNested(tmp); + Path parquet = tmp.resolve("out.parquet"); + Path reimported = tmp.resolve("reimported.vortex"); + + // When + ParquetExporter.exportParquet(vortex, parquet); + ParquetImporter.importParquet(parquet, reimported); + + // Then — every dtype survives, except that Parquet has no fixed-size list to keep `vec` one + try (VortexReader reader = VortexReader.open(reimported)) { + List result = ((DType.Struct) reader.dtype()).fieldTypes(); + assertThat(result).containsExactly(TYPES.get(0), TYPES.get(1), + new DType.List(DTypes.I32, false), TYPES.get(3), TYPES.get(4)); + } + } + + /// Pins a known limitation: the reader folds a struct's validity into its fields, so a + /// present struct whose every field is null cannot be told from a null struct, and exports + /// as one (the reading `CsvExporter` takes too). + @Test + void presentStructWithEveryFieldNull_exportsAsNullStruct(@TempDir Path tmp) throws Exception { + // Given — row 0 is a present struct holding {null, null} + DType.Struct type = new DType.Struct(List.of(ColumnName.of("a"), ColumnName.of("b")), + List.of(new DType.Utf8(true), DTypes.I32.asNullable()), true); + Path vortex = writeVortex(tmp, "allnull.vortex", List.of(ColumnName.of("s")), List.of(type), + Map.of(ColumnName.of("s"), new NullableData(new StructData(List.of( + new NullableData(new String[]{"", "x"}, new boolean[]{false, true}), + new NullableData(new int[]{0, 1}, new boolean[]{false, true}))), + new boolean[]{true, true}))); + Path parquet = tmp.resolve("out.parquet"); + + // When + ParquetExporter.exportParquet(vortex, parquet); + + // Then + try (ParquetFileReader reader = ParquetFileReader.open(InputFile.of(parquet)); + RowReader rows = reader.buildRowReader().build()) { + rows.next(); + assertThat(rows.isNull("s")).isTrue(); + rows.next(); + assertThat(rows.getStruct("s").getString("a")).isEqualTo("x"); + } + } + + @Test + void mapColumn_exportsAsParquetMap(@TempDir Path tmp) throws Exception { + // Given — a nullable map: {a: 1, b: null}, null, {} + DType.Map type = new DType.Map(DTypes.UTF8, DTypes.I64.asNullable(), false, true); + ListViewData entries = new ListViewData(new StructData(List.of(new String[]{"a", "b"}, + new NullableData(new long[]{1, 0}, new boolean[]{true, false}))), + new int[]{0, 2, 2}, new int[]{2, 0, 0}, 3); + Path vortex = writeVortex(tmp, "map.vortex", List.of(ColumnName.of("m")), List.of(type), + Map.of(ColumnName.of("m"), new NullableData(entries, new boolean[]{true, false, true}))); + Path parquet = tmp.resolve("out.parquet"); + + // When + ParquetExporter.exportParquet(vortex, parquet); + + // Then + try (ParquetFileReader reader = ParquetFileReader.open(InputFile.of(parquet)); + RowReader rows = reader.buildRowReader().build()) { + rows.next(); + assertThat(rows.getMap("m").size()).isEqualTo(2); + assertThat(rows.getMap("m").getValue("a")).isEqualTo(1L); + assertThat(rows.getMap("m").containsKey("b")).isTrue(); + assertThat(rows.getMap("m").getValue("b")).isNull(); + rows.next(); + assertThat(rows.isNull("m")).isTrue(); + rows.next(); + assertThat(rows.getMap("m").isEmpty()).isTrue(); + } + } + } }