From db8c46c2bd86c6a514c798509e5c8079ff86eb17 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 5 Nov 2026 11:25:25 -0600 Subject: [PATCH 1/6] GH-3934: Encrypt DataPageV2 repetition and definition levels together with the values The data page module of an encrypted DataPageV2 page is the whole page body. Arrow C++ and arrow-rs encrypt [repetition levels | definition levels | values] as one module, but parquet-java left the levels in plaintext and encrypted only the values, so the two could not read each other's files whenever a page had repetition or definition levels, and the levels leaked the null and list structure of encrypted columns. parquet-java now writes the single module layout, in ColumnChunkPageWriteStore and in ParquetRewriter, through the new ParquetFileWriter.writeEncryptedDataPageV2. The reader recognizes both layouts, so files written by earlier versions can still be read. An encrypted module starts with its own length, so a page body that starts with the body length minus four is a single module; otherwise the levels are in plaintext in front of the module of the values. Tests read files written by Arrow and by the previous parquet-java layout, check what parquet-java and ParquetRewriter write with a reference decoder written from the specification, and check that tampering with levels is detected. --- .../hadoop/ColumnChunkPageReadStore.java | 140 +++++- .../hadoop/ColumnChunkPageWriteStore.java | 39 +- .../hadoop/FullBodyEncryptedDataPageV2.java | 116 +++++ .../parquet/hadoop/ParquetFileReader.java | 79 +++- .../parquet/hadoop/ParquetFileWriter.java | 125 ++++- .../hadoop/rewrite/ParquetRewriter.java | 55 ++- .../parquet/crypto/Base64KmsClient.java | 45 ++ .../crypto/DataPageV2EncryptionTestData.java | 151 ++++++ .../TestDataPageV2EncryptionInterop.java | 431 ++++++++++++++++++ .../TestFullBodyEncryptedDataPageV2.java | 93 ++++ .../arrow-v2-aes-gcm-ctr-v1-snappy.parquet | Bin 0 -> 12400 bytes ...row-v2-aes-gcm-ctr-v1-uncompressed.parquet | Bin 0 -> 21202 bytes .../arrow-v2-aes-gcm-v1-snappy.parquet | Bin 0 -> 12800 bytes .../arrow-v2-aes-gcm-v1-uncompressed.parquet | Bin 0 -> 21602 bytes .../datapagev2/generate_arrow_fixtures.py | 106 +++++ ...va-legacy-v2-aes-gcm-ctr-v1-snappy.parquet | Bin 0 -> 7722 bytes ...acy-v2-aes-gcm-ctr-v1-uncompressed.parquet | Bin 0 -> 8684 bytes .../java-legacy-v2-aes-gcm-v1-snappy.parquet | Bin 0 -> 8042 bytes ...-legacy-v2-aes-gcm-v1-uncompressed.parquet | Bin 0 -> 9004 bytes 19 files changed, 1289 insertions(+), 91 deletions(-) create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/crypto/Base64KmsClient.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-ctr-v1-snappy.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-ctr-v1-uncompressed.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-v1-snappy.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-v1-uncompressed.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-ctr-v1-snappy.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-ctr-v1-uncompressed.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-v1-snappy.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-v1-uncompressed.parquet diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java index a21fec9ab8..0bfdc40ad5 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java @@ -218,6 +218,10 @@ public DataPage visit(DataPageV1 dataPageV1) { @Override public DataPage visit(DataPageV2 dataPageV2) { + if (dataPageV2 instanceof FullBodyEncryptedDataPageV2) { + return readSingleEncryptedModulePageV2( + (FullBodyEncryptedDataPageV2) dataPageV2, currentPageIndex); + } if (!dataPageV2.isCompressed() && offsetIndex == null && null == blockDecryptor) { return dataPageV2; } @@ -266,35 +270,121 @@ public DataPage visit(DataPageV2 dataPageV2) { throw new ParquetDecodingException("could not decompress page", e); } - final DataPageV2 decompressedPage; - if (offsetIndex == null) { - decompressedPage = DataPageV2.uncompressed( - dataPageV2.getRowCount(), - dataPageV2.getNullCount(), - dataPageV2.getValueCount(), - dataPageV2.getRepetitionLevels(), - dataPageV2.getDefinitionLevels(), - dataPageV2.getDataEncoding(), - pageBytes, - dataPageV2.getStatistics()); + return uncompressedPageV2( + dataPageV2, + dataPageV2.getRepetitionLevels(), + dataPageV2.getDefinitionLevels(), + pageBytes, + currentPageIndex); + } + }); + } + + private DataPageV2 uncompressedPageV2( + DataPageV2 source, + BytesInput repetitionLevels, + BytesInput definitionLevels, + BytesInput values, + int currentPageIndex) { + final DataPageV2 decompressedPage; + if (offsetIndex == null) { + decompressedPage = DataPageV2.uncompressed( + source.getRowCount(), + source.getNullCount(), + source.getValueCount(), + repetitionLevels, + definitionLevels, + source.getDataEncoding(), + values, + source.getStatistics()); + } else { + decompressedPage = DataPageV2.uncompressed( + source.getRowCount(), + source.getNullCount(), + source.getValueCount(), + offsetIndex.getFirstRowIndex(currentPageIndex), + repetitionLevels, + definitionLevels, + source.getDataEncoding(), + values, + source.getStatistics()); + } + if (source.getCrc().isPresent()) { + decompressedPage.setCrc(source.getCrc().getAsInt()); + } + return decompressedPage; + } + + /** + * Decrypts a DataPageV2 whose body, repetition and definition levels included, is one encrypted + * module, splits the levels from the values, and decompresses the values. + */ + private DataPageV2 readSingleEncryptedModulePageV2(FullBodyEncryptedDataPageV2 page, int currentPageIndex) { + final int levelsByteLength = page.getRepetitionLevelsByteLength() + page.getDefinitionLevelsByteLength(); + final int valuesUncompressedSize = Math.toIntExact(page.getUncompressedSize() - levelsByteLength); + BytesInput repetitionLevels; + BytesInput definitionLevels; + BytesInput values; + try { + BytesInput encryptedBody = page.getData(); + if (options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer()) { + ByteBuffer encrypted = encryptedBody.toByteBuffer(releaser); + if (!encrypted.isDirect()) { + throw new ParquetDecodingException("Expected a direct buffer"); + } + // The decrypted bytes are laid out as [repetition levels | definition levels | values] + ByteBuffer decrypted = blockDecryptor.decrypt(encrypted, dataPageAAD); + checkDecryptedPageSize(decrypted.limit(), levelsByteLength); + ByteBuffer repetitionLevelsBuffer = slice(decrypted, 0, page.getRepetitionLevelsByteLength()); + ByteBuffer definitionLevelsBuffer = slice( + decrypted, page.getRepetitionLevelsByteLength(), page.getDefinitionLevelsByteLength()); + ByteBuffer valuesBuffer = slice(decrypted, levelsByteLength, decrypted.limit() - levelsByteLength); + repetitionLevels = BytesInput.from(repetitionLevelsBuffer); + definitionLevels = BytesInput.from(definitionLevelsBuffer); + if (page.isCompressed()) { + long compressedSize = valuesBuffer.limit(); + ByteBuffer decompressedBuffer = options.getAllocator().allocate(valuesUncompressedSize); + releaser.releaseLater(decompressedBuffer); + long start = System.nanoTime(); + decompressor.decompress( + valuesBuffer, (int) compressedSize, decompressedBuffer, valuesUncompressedSize); + setDecompressMetrics(encryptedBody, start); + decompressedBuffer.flip(); + values = BytesInput.from(decompressedBuffer); } else { - decompressedPage = DataPageV2.uncompressed( - dataPageV2.getRowCount(), - dataPageV2.getNullCount(), - dataPageV2.getValueCount(), - offsetIndex.getFirstRowIndex(currentPageIndex), - dataPageV2.getRepetitionLevels(), - dataPageV2.getDefinitionLevels(), - dataPageV2.getDataEncoding(), - pageBytes, - dataPageV2.getStatistics()); + values = BytesInput.from(valuesBuffer); } - if (dataPageV2.getCrc().isPresent()) { - decompressedPage.setCrc(dataPageV2.getCrc().getAsInt()); + } else { + byte[] decrypted = blockDecryptor.decrypt(encryptedBody.toByteArray(), dataPageAAD); + checkDecryptedPageSize(decrypted.length, levelsByteLength); + repetitionLevels = BytesInput.from(decrypted, 0, page.getRepetitionLevelsByteLength()); + definitionLevels = BytesInput.from( + decrypted, page.getRepetitionLevelsByteLength(), page.getDefinitionLevelsByteLength()); + values = BytesInput.from(decrypted, levelsByteLength, decrypted.length - levelsByteLength); + if (page.isCompressed()) { + long start = System.nanoTime(); + values = BytesInput.copy(decompressor.decompress(values, valuesUncompressedSize)); + setDecompressMetrics(encryptedBody, start); } - return decompressedPage; } - }); + } catch (IOException e) { + throw new ParquetDecodingException("could not decompress page", e); + } + return uncompressedPageV2(page, repetitionLevels, definitionLevels, values, currentPageIndex); + } + + private void checkDecryptedPageSize(int decryptedSize, int levelsByteLength) { + if (decryptedSize < levelsByteLength) { + throw new ParquetDecodingException("Decrypted DataPageV2 body has " + decryptedSize + + " bytes, which is less than the " + levelsByteLength + + " bytes of repetition and definition levels in the page header"); + } + } + + private ByteBuffer slice(ByteBuffer buffer, int offset, int length) { + ByteBuffer duplicate = buffer.duplicate(); + duplicate.position(offset).limit(offset + length); + return duplicate.slice(); } private void setDecompressMetrics(BytesInput bytes, long start) { diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java index 9f5bc48f8d..5dd49a45d8 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java @@ -317,12 +317,27 @@ public void writePageV2( compressedData = compressor.compress(data); compressed = true; } + // The bytes that follow the page header. The "data page" module of Parquet Modular Encryption + // is the whole page body of a DataPageV2 page, so when the column is encrypted the repetition + // and definition levels are encrypted together with the values, as one module. This is the + // layout of the specification and the one written by Arrow C++ and arrow-rs; the levels + // must not be left in plaintext. + List pageBody = new ArrayList<>(3); if (null != pageBlockEncryptor) { AesCipher.quickUpdatePageAAD(dataPageAAD, pageOrdinal); - compressedData = BytesInput.from(pageBlockEncryptor.encrypt(compressedData.toByteArray(), dataPageAAD)); + byte[] plainPageBody = BytesInput.concat(repetitionLevels, definitionLevels, compressedData) + .toByteArray(); + pageBody.add(BytesInput.from(pageBlockEncryptor.encrypt(plainPageBody, dataPageAAD))); + } else { + pageBody.add(repetitionLevels); + pageBody.add(definitionLevels); + pageBody.add(compressedData); + } + long pageBodySize = 0; + for (BytesInput part : pageBody) { + pageBodySize += part.size(); } - int compressedSize = - toIntWithCheck(compressedData.size() + repetitionLevels.size() + definitionLevels.size()); + int compressedSize = toIntWithCheck(pageBodySize); tempOutputStream.reset(); if (null != headerBlockEncryptor) { AesCipher.quickUpdatePageAAD(dataPageHeaderAAD, pageOrdinal); @@ -330,14 +345,10 @@ public void writePageV2( if (pageWriteChecksumEnabled) { crc.reset(); try (ByteBufferReleaser pageReleaser = new ByteBufferReleaser(allocator)) { - if (repetitionLevels.size() > 0) { - crc.update(repetitionLevels.toByteBuffer(pageReleaser)); - } - if (definitionLevels.size() > 0) { - crc.update(definitionLevels.toByteBuffer(pageReleaser)); - } - if (compressedData.size() > 0) { - crc.update(compressedData.toByteBuffer(pageReleaser)); + for (BytesInput part : pageBody) { + if (part.size() > 0) { + crc.update(part.toByteBuffer(pageReleaser)); + } } } parquetMetadataConverter.writeDataPageV2Header( @@ -382,8 +393,10 @@ public void writePageV2( // by concatenating before collecting instead of collecting twice, // we only allocate one buffer to copy into instead of multiple. - buf.collect(BytesInput.concat( - BytesInput.from(tempOutputStream), repetitionLevels, definitionLevels, compressedData)); + List pageBytes = new ArrayList<>(pageBody.size() + 1); + pageBytes.add(BytesInput.from(tempOutputStream)); + pageBytes.addAll(pageBody); + buf.collect(BytesInput.concat(pageBytes)); dataEncodings.add(dataEncoding); } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java new file mode 100644 index 0000000000..6659c052b2 --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop; + +import java.io.IOException; +import org.apache.parquet.bytes.ByteBufferInputStream; +import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.page.DataPageV2; +import org.apache.parquet.column.statistics.Statistics; + +/** + * A still encrypted {@link DataPageV2} whose entire page body is a single encrypted module. + * + *

Parquet Modular Encryption encrypts the "data page" module, which for a DataPageV2 page is + * the concatenation {@code [repetition levels | definition levels | (compressed) values]}. This is + * what Arrow C++ and arrow-rs write, and what parquet-java writes since the layout was aligned. + * Older parquet-java versions left the levels in plaintext and encrypted only the values; those + * pages are still read as plain {@link DataPageV2} pages whose levels are already separated out. + * + *

The page body can only be decrypted, and thereby split into levels and values, once the page + * AAD is known. That happens in {@code ColumnChunkPageReader}. Until then the whole encrypted + * module is held as the {@link #getData() data} of this page, and the repetition and definition + * levels are empty. The sizes of the levels, which the page header carries, are kept separately. + */ +final class FullBodyEncryptedDataPageV2 extends DataPageV2 { + + private final int repetitionLevelsByteLength; + private final int definitionLevelsByteLength; + + /** + * @param encryptedBody the encrypted page body, in the serialized form of an encrypted module: + * {@code length (4 bytes) | nonce | ciphertext [| tag]} + * @param uncompressedSize the uncompressed page size of the page header, levels included + */ + FullBodyEncryptedDataPageV2( + int rowCount, + int nullCount, + int valueCount, + BytesInput encryptedBody, + Encoding dataEncoding, + int repetitionLevelsByteLength, + int definitionLevelsByteLength, + int uncompressedSize, + Statistics statistics, + boolean isCompressed) { + super( + rowCount, + nullCount, + valueCount, + BytesInput.empty(), + BytesInput.empty(), + dataEncoding, + encryptedBody, + uncompressedSize, + statistics, + isCompressed); + this.repetitionLevelsByteLength = repetitionLevelsByteLength; + this.definitionLevelsByteLength = definitionLevelsByteLength; + } + + int getRepetitionLevelsByteLength() { + return repetitionLevelsByteLength; + } + + int getDefinitionLevelsByteLength() { + return definitionLevelsByteLength; + } + + /** + * Tells which of the two DataPageV2 layouts an encrypted page body has. This must only be called + * for a page that has repetition or definition levels: without levels both layouts are + * byte-for-byte identical. + * + *

An encrypted module starts with its own length, a 4 byte little endian integer holding the + * size of the rest of the module. If the page body is a single module, that length is the size + * of the page body minus 4. If the levels are stored in plaintext, which is how earlier + * parquet-java versions wrote the page, the body starts with the RLE encoded levels instead, and + * the length of the encrypted values is found after the levels. Matching the first four bytes + * with the page size is therefore sufficient to recognize the single module layout. A legacy page + * would only be mistaken for it if the first four bytes of its levels happened to hold exactly + * that value; in that case the page fails to decrypt, it is not silently misread. + * + * @param pageBody the page body, as sized by the {@code compressed_page_size} of the page header + * @return true if the whole page body is one encrypted module, false if the levels are stored in + * plaintext in front of the encrypted values + * @throws IOException if the page body cannot be read + */ + static boolean isSingleEncryptedModule(BytesInput pageBody) throws IOException { + long size = pageBody.size(); + if (size < Integer.BYTES) { + return false; + } + try (ByteBufferInputStream in = pageBody.toInputStream()) { + return BytesUtils.readIntLittleEndian(in) == size - Integer.BYTES; + } + } +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 5a611b41d1..b46464d034 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -2215,14 +2215,37 @@ public ColumnChunkPageReader readAllPages( break; case DATA_PAGE_V2: DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); - int dataSize = compressedPageSize - - dataHeaderV2.getRepetition_levels_byte_length() - - dataHeaderV2.getDefinition_levels_byte_length(); - final BytesInput repetitionLevels = - this.readAsBytesInput(dataHeaderV2.getRepetition_levels_byte_length()); - final BytesInput definitionLevels = - this.readAsBytesInput(dataHeaderV2.getDefinition_levels_byte_length()); - final BytesInput values = this.readAsBytesInput(dataSize); + final int repetitionLevelsByteLength = dataHeaderV2.getRepetition_levels_byte_length(); + final int definitionLevelsByteLength = dataHeaderV2.getDefinition_levels_byte_length(); + final BytesInput repetitionLevels; + final BytesInput definitionLevels; + final BytesInput values; + boolean singleEncryptedModule = false; + if (null != pageBlockDecryptor && repetitionLevelsByteLength + definitionLevelsByteLength > 0) { + // The page body of an encrypted DataPageV2 is either one encrypted module that also + // covers the levels (the layout of the Parquet encryption spec, as written by Arrow + // and by current parquet-java), or, in files written by earlier parquet-java versions, + // plaintext levels followed by an encrypted module of the values. + BytesInput pageBody = this.readAsBytesInput(compressedPageSize); + singleEncryptedModule = FullBodyEncryptedDataPageV2.isSingleEncryptedModule(pageBody); + if (singleEncryptedModule) { + repetitionLevels = BytesInput.empty(); + definitionLevels = BytesInput.empty(); + values = pageBody; + } else { + ByteBufferInputStream pageBodyStream = pageBody.toInputStream(); + repetitionLevels = + BytesInput.from(pageBodyStream.sliceBuffers(repetitionLevelsByteLength)); + definitionLevels = + BytesInput.from(pageBodyStream.sliceBuffers(definitionLevelsByteLength)); + values = BytesInput.from(pageBodyStream.remainingBuffers()); + } + } else { + int dataSize = compressedPageSize - repetitionLevelsByteLength - definitionLevelsByteLength; + repetitionLevels = this.readAsBytesInput(repetitionLevelsByteLength); + definitionLevels = this.readAsBytesInput(definitionLevelsByteLength); + values = this.readAsBytesInput(dataSize); + } if (options.usePageChecksumVerification() && pageHeader.isSetCrc()) { verifyCrc( pageHeader.getCrc(), @@ -2231,18 +2254,34 @@ public ColumnChunkPageReader readAllPages( definitionLevels, values); } - DataPageV2 dataPageV2 = new DataPageV2( - dataHeaderV2.getNum_rows(), - dataHeaderV2.getNum_nulls(), - dataHeaderV2.getNum_values(), - repetitionLevels, - definitionLevels, - converter.getEncoding(dataHeaderV2.getEncoding()), - values, - uncompressedPageSize, - converter.fromParquetStatistics( - getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), - dataHeaderV2.isIs_compressed()); + DataPageV2 dataPageV2; + if (singleEncryptedModule) { + dataPageV2 = new FullBodyEncryptedDataPageV2( + dataHeaderV2.getNum_rows(), + dataHeaderV2.getNum_nulls(), + dataHeaderV2.getNum_values(), + values, + converter.getEncoding(dataHeaderV2.getEncoding()), + repetitionLevelsByteLength, + definitionLevelsByteLength, + uncompressedPageSize, + converter.fromParquetStatistics( + getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), + dataHeaderV2.isIs_compressed()); + } else { + dataPageV2 = new DataPageV2( + dataHeaderV2.getNum_rows(), + dataHeaderV2.getNum_nulls(), + dataHeaderV2.getNum_values(), + repetitionLevels, + definitionLevels, + converter.getEncoding(dataHeaderV2.getEncoding()), + values, + uncompressedPageSize, + converter.fromParquetStatistics( + getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), + dataHeaderV2.isIs_compressed()); + } // Copy crc to new page, used for testing if (pageHeader.isSetCrc()) { dataPageV2.setCrc(pageHeader.getCrc()); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index a453d4e756..4c02bc5725 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -28,6 +28,8 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -1353,7 +1355,9 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page + * Writes a single v2 data page. The repetition and definition levels are written as given, in + * front of the data bytes. To write the page of an encrypted column, whose levels are part of the + * encrypted page body, use {@link #writeEncryptedDataPageV2}. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1385,16 +1389,107 @@ public void writeDataPageV2( byte[] pageHeaderAAD, SizeStatistics sizeStatistics) throws IOException { + writeDataPageV2Body( + rowCount, + nullCount, + valueCount, + toIntWithCheck(repetitionLevels.size(), "page repetition levels"), + toIntWithCheck(definitionLevels.size(), "page definition levels"), + dataEncoding, + Arrays.asList(repetitionLevels, definitionLevels, bytes), + compressed, + uncompressedDataSize, + statistics, + metadataBlockEncryptor, + pageHeaderAAD, + sizeStatistics); + } + + /** + * Writes a single v2 data page of an encrypted column, for a caller that has already encrypted + * the page body. + * + *

With Parquet Modular Encryption the data page module of a v2 page is the whole page body, + * so the encrypted page body holds the repetition levels, the definition levels and the values, + * in that order. The levels must not be written in plaintext in front of the encrypted values, + * which is what {@link #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, + * BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[], SizeStatistics)} does with + * the bytes it is given. + * + * @param rowCount count of rows + * @param nullCount count of nulls + * @param valueCount count of values + * @param repetitionLevelsByteLength length of the repetition levels, before encryption + * @param definitionLevelsByteLength length of the definition levels, before encryption + * @param dataEncoding encoding for data + * @param encryptedPageBody the encrypted concatenation of repetition levels, definition levels and + * (possibly compressed) data, as serialized by the page block encryptor + * @param compressed whether the data bytes were compressed before they were encrypted + * @param uncompressedDataSize the size of the uncompressed data, without the levels + * @param statistics the statistics of the page + * @param metadataBlockEncryptor encryptor for the page header + * @param pageHeaderAAD pageHeader AAD + * @param sizeStatistics size statistics for the page + * @throws IOException if any I/O error occurs during writing the file + */ + public void writeEncryptedDataPageV2( + int rowCount, + int nullCount, + int valueCount, + int repetitionLevelsByteLength, + int definitionLevelsByteLength, + Encoding dataEncoding, + BytesInput encryptedPageBody, + boolean compressed, + int uncompressedDataSize, + Statistics statistics, + BlockCipher.Encryptor metadataBlockEncryptor, + byte[] pageHeaderAAD, + SizeStatistics sizeStatistics) + throws IOException { + writeDataPageV2Body( + rowCount, + nullCount, + valueCount, + repetitionLevelsByteLength, + definitionLevelsByteLength, + dataEncoding, + Collections.singletonList(encryptedPageBody), + compressed, + uncompressedDataSize, + statistics, + metadataBlockEncryptor, + pageHeaderAAD, + sizeStatistics); + } + + /** + * @param pageBody the bytes that follow the page header, in the order they are written + */ + private void writeDataPageV2Body( + int rowCount, + int nullCount, + int valueCount, + int rlByteLength, + int dlByteLength, + Encoding dataEncoding, + List pageBody, + boolean compressed, + int uncompressedDataSize, + Statistics statistics, + BlockCipher.Encryptor metadataBlockEncryptor, + byte[] pageHeaderAAD, + SizeStatistics sizeStatistics) + throws IOException { withAbortOnFailure(() -> { state = state.write(); - int rlByteLength = toIntWithCheck(repetitionLevels.size(), "page repetition levels"); - int dlByteLength = toIntWithCheck(definitionLevels.size(), "page definition levels"); - - int compressedSize = - toIntWithCheck(bytes.size() + repetitionLevels.size() + definitionLevels.size(), "page"); + long pageBodySize = 0; + for (BytesInput part : pageBody) { + pageBodySize += part.size(); + } + int compressedSize = toIntWithCheck(pageBodySize, "page"); - int uncompressedSize = - toIntWithCheck(uncompressedDataSize + repetitionLevels.size() + definitionLevels.size(), "page"); + int uncompressedSize = toIntWithCheck((long) uncompressedDataSize + rlByteLength + dlByteLength, "page"); long beforeHeader = out.getPos(); if (currentChunkFirstDataPage < 0) { @@ -1403,14 +1498,10 @@ public void writeDataPageV2( if (pageWriteChecksumEnabled) { crc.reset(); - if (repetitionLevels.size() > 0) { - crcUpdate(repetitionLevels); - } - if (definitionLevels.size() > 0) { - crcUpdate(definitionLevels); - } - if (bytes.size() > 0) { - crcUpdate(bytes); + for (BytesInput part : pageBody) { + if (part.size() > 0) { + crcUpdate(part); + } } metadataConverter.writeDataPageV2Header( uncompressedSize, @@ -1451,7 +1542,7 @@ public void writeDataPageV2( currentEncodings.add(dataEncoding); encodingStatsBuilder.addDataEncoding(dataEncoding); - BytesInput.concat(repetitionLevels, definitionLevels, bytes).writeAllTo(out); + BytesInput.concat(pageBody).writeAllTo(out); offsetIndexBuilder.add( toIntWithCheck(out.getPos() - beforeHeader, "page"), diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java index fce8866850..15c1fb35fc 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java @@ -738,6 +738,8 @@ private void processChunk( BytesInput dlLevels = readBlockAllocate(dlLength, reader); int payLoadLength = pageHeader.getCompressed_page_size() - rlLength - dlLength; int rawDataLength = pageHeader.getUncompressed_page_size() - rlLength - dlLength; + // The page body of an encrypted DataPageV2 is encrypted as a whole, levels included, so the + // payload is not encrypted here but together with the levels below. pageLoad = processPageLoad( reader, headerV2.is_compressed, @@ -745,9 +747,9 @@ private void processChunk( decompressor, payLoadLength, rawDataLength, - encryptColumn, - dataEncryptor, - dataPageAAD); + false, + null, + null); statistics = convertStatistics( originalCreatedBy, normalizeNameInType(chunk.getPrimitiveType()), @@ -765,19 +767,40 @@ private void processChunk( } readValues += headerV2.getNum_values(); readRows += headerV2.getNum_rows(); - writer.writeDataPageV2( - headerV2.getNum_rows(), - headerV2.getNum_nulls(), - headerV2.getNum_values(), - rlLevels, - dlLevels, - converter.getEncoding(headerV2.getEncoding()), - BytesInput.from(pageLoad), - headerV2.is_compressed, - rawDataLength, - statistics, - metaEncryptor, - dataPageHeaderAAD); + if (encryptColumn) { + BytesInput encryptedPageBody = BytesInput.from(dataEncryptor.encrypt( + BytesInput.concat(rlLevels, dlLevels, BytesInput.from(pageLoad)) + .toByteArray(), + dataPageAAD)); + writer.writeEncryptedDataPageV2( + headerV2.getNum_rows(), + headerV2.getNum_nulls(), + headerV2.getNum_values(), + rlLength, + dlLength, + converter.getEncoding(headerV2.getEncoding()), + encryptedPageBody, + headerV2.is_compressed, + rawDataLength, + statistics, + metaEncryptor, + dataPageHeaderAAD, + null); + } else { + writer.writeDataPageV2( + headerV2.getNum_rows(), + headerV2.getNum_nulls(), + headerV2.getNum_values(), + rlLevels, + dlLevels, + converter.getEncoding(headerV2.getEncoding()), + BytesInput.from(pageLoad), + headerV2.is_compressed, + rawDataLength, + statistics, + metaEncryptor, + dataPageHeaderAAD); + } pageOrdinal++; break; default: diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/Base64KmsClient.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/Base64KmsClient.java new file mode 100644 index 0000000000..e02e8355af --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/Base64KmsClient.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.crypto; + +import java.util.Base64; +import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.crypto.keytools.KmsClient; + +/** + * A KMS client that "wraps" a key by base64 encoding it. It is not secure, it only exists to read + * the fixtures written by {@code generate_arrow_fixtures.py}, whose key wrapping it mirrors. + */ +public class Base64KmsClient implements KmsClient { + + @Override + public void initialize(Configuration configuration, String kmsInstanceID, String kmsInstanceURL, String accessToken) + throws KeyAccessDeniedException {} + + @Override + public String wrapKey(byte[] keyBytes, String masterKeyIdentifier) throws KeyAccessDeniedException { + return Base64.getEncoder().encodeToString(keyBytes); + } + + @Override + public byte[] unwrapKey(String wrappedKey, String masterKeyIdentifier) throws KeyAccessDeniedException { + return Base64.getDecoder().decode(wrappedKey); + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java new file mode 100644 index 0000000000..b6b3dacf5a --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.crypto; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.bytes.DirectByteBufferAllocator; +import org.apache.parquet.bytes.HeapByteBufferAllocator; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetInputFormat; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.example.GroupReadSupport; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; + +/** + * Deterministic data, schema and keys shared by the DataPageV2 encryption tests and by the scripts + * that generated the fixture files under {@code src/test/resources/encryption/datapagev2}. + * + *

The schema deliberately mixes a required column (no repetition / definition levels), optional + * columns (definition levels only) and a nested list (repetition and definition levels), because + * the layout of an encrypted DataPageV2 only differs between implementations when a page has + * non-empty levels. + * + *

The row generator must stay in sync with {@code generate_arrow_fixtures.py} in the fixtures + * directory. + */ +public final class DataPageV2EncryptionTestData { + + public static final int NUM_ROWS = 500; + + /** The footer key; with uniform encryption it is also used for all of the columns. */ + public static final byte[] FOOTER_KEY = "0123456789abcdef".getBytes(StandardCharsets.UTF_8); + + public static final byte[] FOOTER_KEY_METADATA = "footkey".getBytes(StandardCharsets.UTF_8); + + public static final MessageType SCHEMA = MessageTypeParser.parseMessageType("message schema {" + + " required int64 seq;" + + " optional int32 id;" + + " optional binary name (STRING);" + + " optional group tags (LIST) {" + + " repeated group list {" + + " optional binary element (STRING);" + + " }" + + " }" + + "}"); + + private DataPageV2EncryptionTestData() {} + + /** Builds row {@code i}. Must match {@code row(i)} in {@code generate_arrow_fixtures.py}. */ + public static Group row(int i) { + Group group = new SimpleGroupFactory(SCHEMA).newGroup(); + group.add("seq", (long) i); + if (i % 7 != 0) { + group.add("id", i); + } + if (i % 5 != 0) { + group.add("name", "name-" + i); + } + if (i % 4 != 0) { + Group tags = group.addGroup("tags"); + int count = i % 4; + for (int j = 0; j < count; j++) { + Group element = tags.addGroup("list"); + if ((i + j) % 3 != 0) { + element.add("element", "tag-" + i + "-" + j); + } + } + } + return group; + } + + public static List expectedRows() { + List rows = new ArrayList<>(NUM_ROWS); + for (int i = 0; i < NUM_ROWS; i++) { + rows.add(row(i).toString()); + } + return rows; + } + + /** Key of the columns that are encrypted with their own key, in the tests that do that. */ + public static final byte[] COLUMN_KEY = "fedcba9876543210".getBytes(StandardCharsets.UTF_8); + + public static final byte[] COLUMN_KEY_METADATA = "colkey".getBytes(StandardCharsets.UTF_8); + + /** Decryption properties for the files that were written with {@link #FOOTER_KEY} and {@link #COLUMN_KEY}. */ + public static FileDecryptionProperties decryptionProperties() { + return FileDecryptionProperties.builder() + .withKeyRetriever(keyMetadata -> { + String id = new String(keyMetadata, StandardCharsets.UTF_8); + switch (id) { + case "footkey": + return FOOTER_KEY; + case "colkey": + return COLUMN_KEY; + default: + throw new KeyAccessDeniedException("Unknown key " + id); + } + }) + .build(); + } + + /** + * Reads all of the rows of a file, rendered as strings. + * + * @param conf may hold the configuration of a decryption properties factory + * @param decryption the decryption properties, or null to take them from the configuration + * @param directMemory whether to read the file into direct buffers and decrypt it off-heap + */ + public static List readRows( + Path file, Configuration conf, FileDecryptionProperties decryption, boolean directMemory) + throws IOException { + conf.setBoolean(ParquetInputFormat.OFF_HEAP_DECRYPT_BUFFER_ENABLED, directMemory); + ParquetReader.Builder builder = ParquetReader.builder(new GroupReadSupport(), file) + .withConf(conf) + .withAllocator(directMemory ? new DirectByteBufferAllocator() : new HeapByteBufferAllocator()); + if (decryption != null) { + builder.withDecryption(decryption); + } + List rows = new ArrayList<>(); + try (ParquetReader reader = builder.build()) { + Group group; + while ((group = reader.read()) != null) { + rows.add(group.toString()); + } + } + return rows; + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java new file mode 100644 index 0000000000..bf843381af --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java @@ -0,0 +1,431 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.crypto; + +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.COLUMN_KEY; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.COLUMN_KEY_METADATA; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.FOOTER_KEY; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.FOOTER_KEY_METADATA; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.NUM_ROWS; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.SCHEMA; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.decryptionProperties; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.expectedRows; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.readRows; +import static org.apache.parquet.crypto.DataPageV2EncryptionTestData.row; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.ParquetReadOptions; +import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.crypto.ModuleCipherFactory.ModuleType; +import org.apache.parquet.crypto.keytools.KeyToolkit; +import org.apache.parquet.crypto.keytools.PropertiesDrivenCryptoFactory; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.format.BlockCipher; +import org.apache.parquet.format.FileCryptoMetaData; +import org.apache.parquet.format.PageHeader; +import org.apache.parquet.format.PageType; +import org.apache.parquet.format.Util; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ColumnPath; +import org.apache.parquet.hadoop.metadata.CompressionCodecName; +import org.apache.parquet.hadoop.rewrite.ParquetRewriter; +import org.apache.parquet.hadoop.rewrite.RewriteOptions; +import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.io.ParquetDecodingException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Interoperability of the encryption of DataPageV2 pages that have repetition or definition levels. + * + *

The page body of such a page, the levels included, is a single encrypted "data page" module. + * Arrow C++ and arrow-rs write it that way, and so does parquet-java. Earlier parquet-java versions + * left the levels in plaintext and encrypted only the values (apache/parquet-java#3934), so files + * in both layouts exist. These tests make sure that + * + *

    + *
  • parquet-java writes only the single module layout, which an independent decoder written + * from the specification confirms, also when it rewrites files; + *
  • parquet-java reads the files that Arrow wrote, which the {@code arrow-v2-*} fixtures are; + *
  • parquet-java still reads the files that it wrote in the old layout, which the {@code + * java-legacy-v2-*} fixtures are. They were written by parquet-java 1.19.0-SNAPSHOT at commit + * e265ac7f5, just before the layout changed, with the footer key as the key of all columns, + * the same rows as {@link DataPageV2EncryptionTestData#row(int)}, 100 rows per page and + * {@code ExampleParquetWriter} with {@code WriterVersion.PARQUET_2_0}. + *
+ */ +public class TestDataPageV2EncryptionInterop { + + private static final String FIXTURES = "/encryption/datapagev2/"; + + @TempDir + private java.nio.file.Path tempDir; + + static Stream fixturesAndMemoryModes(String prefix) { + List arguments = new ArrayList<>(); + for (String cipher : new String[] {"aes-gcm-v1", "aes-gcm-ctr-v1"}) { + for (String codec : new String[] {"uncompressed", "snappy"}) { + for (boolean directMemory : new boolean[] {false, true}) { + arguments.add(Arguments.of(prefix + cipher + "-" + codec + ".parquet", directMemory)); + } + } + } + return arguments.stream(); + } + + static Stream arrowFixtures() { + return fixturesAndMemoryModes("arrow-v2-"); + } + + static Stream legacyFixtures() { + return fixturesAndMemoryModes("java-legacy-v2-"); + } + + static Stream ciphersAndCodecs() { + List arguments = new ArrayList<>(); + for (ParquetCipher cipher : ParquetCipher.values()) { + for (CompressionCodecName codec : + new CompressionCodecName[] {CompressionCodecName.UNCOMPRESSED, CompressionCodecName.SNAPPY}) { + arguments.add(Arguments.of(cipher, codec)); + } + } + return arguments.stream(); + } + + @ParameterizedTest(name = "{0}, directMemory={1}") + @MethodSource("arrowFixtures") + public void readsFilesWrittenByArrow(String fixture, boolean directMemory) throws Exception { + // The keys of the Arrow files are wrapped by the KMS client that generate_arrow_fixtures.py mirrors + Configuration conf = new Configuration(); + conf.set( + EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME, + PropertiesDrivenCryptoFactory.class.getName()); + conf.set(KeyToolkit.KMS_CLIENT_CLASS_PROPERTY_NAME, Base64KmsClient.class.getName()); + + assertThat(readRows(fixture(fixture), conf, null, directMemory)).containsExactlyElementsOf(expectedRows()); + } + + @ParameterizedTest(name = "{0}, directMemory={1}") + @MethodSource("legacyFixtures") + public void readsFilesWrittenByEarlierParquetJavaVersions(String fixture, boolean directMemory) throws Exception { + assertThat(readRows(fixture(fixture), new Configuration(), decryptionProperties(), directMemory)) + .containsExactlyElementsOf(expectedRows()); + } + + @ParameterizedTest(name = "{0}, directMemory={1}") + @MethodSource("legacyFixtures") + public void legacyFixturesHaveThePlaintextLevelsLayout(String fixture, boolean directMemory) throws Exception { + // The test above only proves the legacy code path is covered if the fixtures really are legacy files + List pages = decodeDataPagesV2(fixture(fixture), FOOTER_KEY, false); + + assertThat(pages).isNotEmpty(); + assertThat(pages).allSatisfy(page -> assertThat(page.singleModule).isFalse()); + } + + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("ciphersAndCodecs") + public void writesTheLevelsInTheEncryptedPageBody(ParquetCipher cipher, CompressionCodecName codec) + throws Exception { + Path file = writeWithParquetJava(cipher, codec); + + List pages = decodeDataPagesV2(file, FOOTER_KEY, false); + + assertThat(pages).isNotEmpty(); + for (DecodedPage page : pages) { + assertThat(page.singleModule) + .as("the levels of %s must be inside the encrypted page body", page) + .isTrue(); + assertThat(page.decryptedLength).isGreaterThanOrEqualTo(page.levelsLength()); + if (codec == CompressionCodecName.UNCOMPRESSED) { + assertThat(page.decryptedLength).isEqualTo(page.uncompressedPageSize); + } + } + } + + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("ciphersAndCodecs") + public void readsWhatItWrites(ParquetCipher cipher, CompressionCodecName codec) throws Exception { + Path file = writeWithParquetJava(cipher, codec); + + assertThat(readRows(file, new Configuration(), decryptionProperties(), false)) + .containsExactlyElementsOf(expectedRows()); + assertThat(readRows(file, new Configuration(), decryptionProperties(), true)) + .containsExactlyElementsOf(expectedRows()); + } + + @Test + public void detectsTamperingWithTheLevels() throws Exception { + Path file = writeWithParquetJava(ParquetCipher.AES_GCM_V1, CompressionCodecName.UNCOMPRESSED); + DecodedPage page = decodeDataPagesV2(file, FOOTER_KEY, false).get(0); + + // Flip a bit of the first byte of the ciphertext, which is the first byte of the levels. When the + // levels were stored in plaintext, changing them went unnoticed. + File source = new File(file.toUri()); + try (java.io.RandomAccessFile raf = new java.io.RandomAccessFile(source, "rw")) { + long position = page.bodyOffset + Integer.BYTES + AesCipher.NONCE_LENGTH; + raf.seek(position); + int original = raf.read(); + raf.seek(position); + raf.write(original ^ 0x01); + } + // Drop the local filesystem CRC side-file so the tampered bytes reach the Parquet reader. + Files.deleteIfExists(new File(source.getParent(), "." + source.getName() + ".crc").toPath()); + + assertThatThrownBy(() -> readRows(file, new Configuration(), decryptionProperties(), false)) + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("Can not read value at 0 in block -1") + .hasRootCauseInstanceOf(TagVerificationException.class) + .hasRootCauseMessage("GCM tag check failed"); + } + + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("ciphersAndCodecs") + public void rewriterWritesTheLevelsInTheEncryptedPageBody(ParquetCipher cipher, CompressionCodecName codec) + throws Exception { + Path plaintext = writeWithParquetJava(null, codec); + Path rewritten = new Path(tempDir.resolve(UUID.randomUUID().toString()).toUri()); + + String[] encryptColumns = {"id", "name", "tags.list.element"}; + Map columns = new HashMap<>(); + for (String column : encryptColumns) { + ColumnPath path = ColumnPath.fromDotString(column); + columns.put( + path, + ColumnEncryptionProperties.builder(path) + .withKey(COLUMN_KEY) + .withKeyMetaData(COLUMN_KEY_METADATA) + .build()); + } + FileEncryptionProperties encryption = FileEncryptionProperties.builder(FOOTER_KEY) + .withFooterKeyMetadata(FOOTER_KEY_METADATA) + .withAlgorithm(cipher) + .withEncryptedColumns(columns) + .build(); + RewriteOptions options = new RewriteOptions.Builder(new Configuration(), plaintext, rewritten) + .encrypt(Arrays.asList(encryptColumns)) + .encryptionProperties(encryption) + .build(); + try (ParquetRewriter rewriter = new ParquetRewriter(options)) { + rewriter.processBlocks(); + } + + List pages = decodeDataPagesV2(rewritten, COLUMN_KEY, true); + assertThat(pages).isNotEmpty(); + assertThat(pages).allSatisfy(page -> assertThat(page.singleModule).isTrue()); + assertThat(readRows(rewritten, new Configuration(), decryptionProperties(), false)) + .containsExactlyElementsOf(expectedRows()); + assertThat(readRows(rewritten, new Configuration(), decryptionProperties(), true)) + .containsExactlyElementsOf(expectedRows()); + } + + /** Writes a DataPageV2 file with 100 rows per page, encrypted with the footer key if there is a cipher. */ + private Path writeWithParquetJava(ParquetCipher cipher, CompressionCodecName codec) throws IOException { + Path file = new Path(tempDir.resolve(UUID.randomUUID().toString()).toUri()); + ExampleParquetWriter.Builder builder = ExampleParquetWriter.builder(file) + .withType(SCHEMA) + .withWriterVersion(WriterVersion.PARQUET_2_0) + .withCompressionCodec(codec) + .withPageRowCountLimit(100); + if (cipher != null) { + builder.withEncryption(FileEncryptionProperties.builder(FOOTER_KEY) + .withFooterKeyMetadata(FOOTER_KEY_METADATA) + .withAlgorithm(cipher) + .build()); + } + try (ParquetWriter writer = builder.build()) { + for (int i = 0; i < NUM_ROWS; i++) { + writer.write(row(i)); + } + } + return file; + } + + private static Path fixture(String name) throws URISyntaxException { + return new Path(TestDataPageV2EncryptionInterop.class + .getResource(FIXTURES + name) + .toURI()); + } + + /** What the reference decoder found out about one DataPageV2 page that has levels. */ + private static final class DecodedPage { + private final String column; + private final int repetitionLevelsLength; + private final int definitionLevelsLength; + private final long bodyOffset; + private final boolean singleModule; + private final int decryptedLength; + private final int uncompressedPageSize; + + private DecodedPage( + String column, + int repetitionLevelsLength, + int definitionLevelsLength, + long bodyOffset, + boolean singleModule, + int decryptedLength, + int uncompressedPageSize) { + this.column = column; + this.repetitionLevelsLength = repetitionLevelsLength; + this.definitionLevelsLength = definitionLevelsLength; + this.bodyOffset = bodyOffset; + this.singleModule = singleModule; + this.decryptedLength = decryptedLength; + this.uncompressedPageSize = uncompressedPageSize; + } + + private int levelsLength() { + return repetitionLevelsLength + definitionLevelsLength; + } + + @Override + public String toString() { + return "DataPageV2 of column " + column + " at offset " + bodyOffset + " with " + levelsLength() + + " bytes of levels"; + } + } + + /** + * A decoder of the encrypted DataPageV2 pages of a file, written from the Parquet encryption + * specification and sharing nothing with the reader of parquet-java except the cipher + * implementations: it parses and decrypts the page headers itself, takes the data page module + * from the page body, and decrypts it with the AAD of the specification. + * + *

As the specification has the page body start with the length of the encrypted module, the body + * is a single module when the first four bytes hold the body length minus four. Otherwise the + * levels are in plaintext and the module of the values follows them. + * + * @param file file with an encrypted footer, whose encrypted columns all use {@code columnKey} + * @param onlyEncryptedColumns true to skip the columns that are not encrypted, for a file where + * only some of the columns are; false if all of the columns are encrypted + * @return the pages that have repetition or definition levels, in file order + */ + private static List decodeDataPagesV2(Path file, byte[] columnKey, boolean onlyEncryptedColumns) + throws IOException { + byte[] bytes = Files.readAllBytes(new File(file.toUri()).toPath()); + int footerLength = BytesUtils.readIntLittleEndian(bytes, bytes.length - 8); + FileCryptoMetaData cryptoMetaData = Util.readFileCryptoMetaData( + new ByteArrayInputStream(bytes, bytes.length - 8 - footerLength, footerLength)); + org.apache.parquet.format.EncryptionAlgorithm algorithm = cryptoMetaData.getEncryption_algorithm(); + boolean ctr = algorithm.isSetAES_GCM_CTR_V1(); + byte[] fileAAD = (ctr + ? algorithm.getAES_GCM_CTR_V1().getAad_file_unique() + : algorithm.getAES_GCM_V1().getAad_file_unique()); + + BlockCipher.Decryptor headerDecryptor = ModuleCipherFactory.getDecryptor(AesMode.GCM, columnKey); + BlockCipher.Decryptor pageDecryptor = + ModuleCipherFactory.getDecryptor(ctr ? AesMode.CTR : AesMode.GCM, columnKey); + + List decoded = new ArrayList<>(); + try (ParquetFileReader reader = ParquetFileReader.open( + HadoopInputFile.fromPath(file, new Configuration()), + ParquetReadOptions.builder() + .withDecryption(decryptionProperties()) + .build())) { + for (BlockMetaData block : reader.getFooter().getBlocks()) { + for (int columnOrdinal = 0; columnOrdinal < block.getColumns().size(); columnOrdinal++) { + ColumnChunkMetaData chunk = block.getColumns().get(columnOrdinal); + if (onlyEncryptedColumns && !chunk.isEncrypted()) { + continue; + } + ByteArrayInputStream in = new ByteArrayInputStream(bytes); + in.skip(chunk.getStartingPos()); + boolean dictionaryPageRead = !chunk.hasDictionaryPage(); + int pageOrdinal = 0; + long valuesRead = 0; + while (valuesRead < chunk.getValueCount()) { + byte[] headerAAD = dictionaryPageRead + ? AesCipher.createModuleAAD( + fileAAD, + ModuleType.DataPageHeader, + block.getOrdinal(), + columnOrdinal, + pageOrdinal) + : AesCipher.createModuleAAD( + fileAAD, + ModuleType.DictionaryPageHeader, + block.getOrdinal(), + columnOrdinal, + -1); + PageHeader header = Util.readPageHeader(in, headerDecryptor, headerAAD); + long bodyOffset = bytes.length - in.available(); + byte[] body = new byte[header.getCompressed_page_size()]; + in.read(body); + if (!dictionaryPageRead) { + dictionaryPageRead = true; + continue; + } + if (header.getType() == PageType.DATA_PAGE_V2) { + int repetitionLevels = + header.getData_page_header_v2().getRepetition_levels_byte_length(); + int definitionLevels = + header.getData_page_header_v2().getDefinition_levels_byte_length(); + int levels = repetitionLevels + definitionLevels; + boolean singleModule = + BytesUtils.readIntLittleEndian(body, 0) == body.length - Integer.BYTES; + byte[] module = singleModule ? body : Arrays.copyOfRange(body, levels, body.length); + // Either way, what is decrypted starts with its own length + assertThat(BytesUtils.readIntLittleEndian(module, 0)) + .isEqualTo(module.length - Integer.BYTES); + byte[] pageAAD = AesCipher.createModuleAAD( + fileAAD, ModuleType.DataPage, block.getOrdinal(), columnOrdinal, pageOrdinal); + byte[] decrypted = pageDecryptor.decrypt(module, pageAAD); + if (levels > 0) { + decoded.add(new DecodedPage( + chunk.getPath().toDotString(), + repetitionLevels, + definitionLevels, + bodyOffset, + singleModule, + decrypted.length, + header.getUncompressed_page_size())); + } + valuesRead += header.getData_page_header_v2().getNum_values(); + } + pageOrdinal++; + } + } + } + } + return decoded; + } +} diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java new file mode 100644 index 0000000000..2fb2f2d8a5 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.hadoop; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.apache.parquet.bytes.BytesInput; +import org.junit.jupiter.api.Test; + +public class TestFullBodyEncryptedDataPageV2 { + + /** The serialized form of an encrypted module: its length, then that many bytes. */ + private static byte[] module(int contentLength) { + ByteBuffer module = ByteBuffer.allocate(Integer.BYTES + contentLength).order(ByteOrder.LITTLE_ENDIAN); + module.putInt(contentLength); + for (int i = 0; i < contentLength; i++) { + module.put((byte) (0x80 + i)); + } + return module.array(); + } + + private static byte[] concat(byte[] first, byte[] second) { + byte[] result = new byte[first.length + second.length]; + System.arraycopy(first, 0, result, 0, first.length); + System.arraycopy(second, 0, result, first.length, second.length); + return result; + } + + @Test + public void bodyThatIsOneModuleIsRecognized() throws IOException { + byte[] body = module(50); + + assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(body))) + .isTrue(); + } + + @Test + public void bodyWithPlaintextLevelsInFrontOfAModuleIsRecognized() throws IOException { + // RLE encoded levels: a bit-packed run of one group, 0xff, followed by an RLE run of 8 values of 1 + byte[] levels = {0x03, (byte) 0xff, 0x10, 0x01}; + byte[] body = concat(levels, module(50)); + + assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(body))) + .isFalse(); + } + + @Test + public void bodyThatIsOneModuleIsRecognizedWhenItIsNotInOneBuffer() throws IOException { + byte[] body = module(300); + ByteBuffer first = ByteBuffer.wrap(body, 0, 2).slice(); + ByteBuffer second = ByteBuffer.wrap(body, 2, body.length - 2).slice(); + + assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(first, second))) + .isTrue(); + } + + @Test + public void bodyThatIsTooShortToStartWithALengthIsNotOneModule() throws IOException { + assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.empty())) + .isFalse(); + assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(new byte[] {1, 2, 3}))) + .isFalse(); + } + + @Test + public void bodyIsNotConsumedByTheCheck() throws IOException { + BytesInput body = BytesInput.from(module(40)); + + FullBodyEncryptedDataPageV2.isSingleEncryptedModule(body); + + assertThat(body.toByteArray()).isEqualTo(module(40)); + } +} diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-ctr-v1-snappy.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-ctr-v1-snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0e56362f2e9048faea19e0448e4b6004e0bf4eec GIT binary patch literal 12400 zcmY-0V~{3IxF+DXZQItgZQHhO^KIL^0kgY5-SQ%s!cl< znu8GQoxmGb3Iz_S%pD<2RfLuPZw^JSgjezIM$i3CA_oyycX*BSG%jw|vj`Vzh>Go7 zu+#0l5*P^RB^gBJ)XRWbIrA)L-J>mkWud(4Ylrj{cJH?)kf!nOq)fe-wuC;Xy6<-B z=!`KbhbXn`6Cnj|8yJ^waaxp3q7%76(<_D>;(EpNgAe;Ids8MxmrQc&)H}{Ep||&@ zzbu9pGk;K{x5>zPlK8yq)3}Zh*B_SZ8I|#pd^`-n5-%VazVt=o*qS*cU~ob94@Nf$ z$Zrw$wjrMqfx}*-csO#|NihYy2?SLfJUnbUN5 zlZYA{ty7<(UMMaYI@cHplOXa7E6%BJu~BDqh~70sQ5v*B3_&5=fvqh1JU{SQG@8|&6O8zlk2_UiB z^YCQM&}mW21@cr%dJR-l{S%7rnlFa%zD-&q;d zKCgk6J}7cqEG-&~N%i))jt&U_^*`1^Ml4-YsbV_cDmAFSz?LsX_9wfkd>|q}w2$2J zM*`7Xvn^Cs7F4)R6U^O)5ec3a8>cH-VT7Z)=l5K2=eovdhrgQTpT|%ZX&u&KLT28A z9UZ`06GwnyYSL*eR;H97Utdpd46s#JocKY3n8|roHbq9@2mOt<;5VpafMWNd53u&FFOL+LmVBshZhSWt+U-c6U-aj+J4F!e5Rz{R$1old%_#)%* zdC}o;bq^EC+5WPKM~=kIIKr_lYrH-@Kyt*ybwiRLey;eQ*E7s=UN*?bB3Re*8|n<7 z{>2ZB^zWa@BcCbVUGTrWk3;+Nce5HBwUeyVy9x=hz(99JS>gAWwPmt=1(R$ zuGk1yIOxL1tvr3!K{$yJ+ykZ(Fape=N~lAjpyH9XB~gbh&c>P%?|4>U`#Ez9$jKBy zOSK#`9KTXqD^iO4l_~h1wyO$~db6DZ_Ztu}Wi%paqI+@g{_0-K@rW9ts(k=MJYL9O zUxw2AP{{XLfwtEm{Uk>ZYd=vKt{MR+_)x$#pQ)sKgvsbFJd8T`+}iABL32vX3dA=8k)r^Y=D zN6^5EzgvGB+;X|w=!7syEw0p+FuPq~(-16xVHFCzb(M5%2)bM_>WvyvGG?>n$ZxF1 zjgj_y5dQgW2n5I1#hj$3a}f(4;k6{`)EPGYH8PRiqifw61D*6lBk|uG#3jQofoR|w z^L}tChO1BKZy;p(pjrcK+Oby}gOlk+C5dGib=%A;}fi1-} zp6ZrzoMSD1NX@hYdhM`EE+a>A{Q4E^adUS5$9%uMfP~?AW3iELPhWpD{7Lk@%c}LY zWx4yZQs!f#<1^;ePPdm*5uV$>C}?& z&N){ee|+OFflQ6#`~H2Vs_-Mp`6DgJAS8ab@=jbdG@8yO74IgkyZ@{-xJ`7xNoUdu zO$B@yk?aI_yyTFGj6KWuH*PFtcj?39Pf~T97LcHcrc#) zgobAgi>E(uK0*v)_X#8fq^!JLzNns|9D$+SDWD9tu0rh!t86|Y=I>icGOp~}Q-#9z7#^Jpx3o(N%yIv`81Z9*V>! zFTLwBX>Ciinse1e=%=g;D#QKa-vd;S;eq6i$D4TVUFbFSvtOOW_tA-=n#`;Z?!p=9;t|?>>Uh%@xn#m`-$?rJ8w4b8Gl0ux!`ux# zMqjW9th^nThs1iU6CR12<9uq(koNun@Tlv*b*^Eg43iIfuR(0G5}HU+l;A1plb!pX z(forzxCECO&7cGKRewTpz)(sSu$M`%%XA^e0D?YzO(FN4pJ-o(4yH=0)&@1nGmXNW zF8lqMJm*8VXCN>hI<);J5`^17TgT-fJbT|sCP6u@Qb;gm@t-~UW)9}so08<6{Z9x` z;s;FDYCJ*U!p-ueItrwq>7zVbsKM;_)^xkxacKGE>^q9cT&L_f=tX$tZn0|;?nRu< zRx4rO`@R`a%*=1EVh{sLq>B)$c(4MXps$@gmhKNyZ`2vT=K6f6v)=)Q0hlP5Q!&^YRY1P$dwVa>3i~qWwkcSEE(!2W_OrVEX^AF{|+c$+?BV5lqy}D z#ZkqwD~cm2?4!8@S2h>P3fDq6_HO=7i;u-j1-`@{|$4qliUR{?$ zUGwp0GjU_SklxDLz|Gubz!pm4;!^&zw3DHM=#VzcrRZqqaqE;@W5as45Eo`EA zGDC!kYY_%=J8PD8se3LHY|+{re=*Q$#qaDPGN!v%zt z#{@Q1pjT+oJ4bL&1l<@BF2KT^i&#H4Zpb;z9ehUpT&}!m{*JoR^bp&8=M?u~46+eU zwy*6oeIu|(;z33>%Or*U$favfgIrP3G;$5xE>QnNqy zmB}*Dql@^_Ykembb9qICRF5=ZkJ-;QRgVt;GzmBchP_uO1fGC}KU`kWTuoSmh5~%r zL%zoTszV8*OhiGky>f==R~5W~1qlwE&c5YxwQu&P*i4{C?Vm`vtSclA@5y>|O^#pvN=*}2Uvqxzy?G&S;(H8DhK-DFsh%xcy|B7y zx>tzb(xcN{N%|sMo#^Jpl)#}jyR90gAvkrSMn7@Gi6V=Q622LmjF=s#x1>@NEpw7#DVhbxih>-kggFYbv^>uht6IO^gqr8>r87A)G7 zrj}f>@)PhOe z8n3@R9820E%trsFdc{!Gqr9v@1aVz2DYu3UJAvUWrr(E9`*IrH|CP6~HiW`UpHK8> zOdK$8%I^Z;q9K}(lgvr7tGpHHanImNI;=-`^G)xHHQrlt0m5!lT%|%6do;nC>n!LZ z0p9UO98A8vjx88?-)JS-rJfvERq7h`XcPhufm*s2mHDWr{VoO+e0(NByQtPSHA-L# zJkSnokrl&H%<7o3M01 zlYbEu-FrFCs1DSF}jRKNd)D3nH+1ShM+`Vw>>>nu<2eEu$I<)r70NonAAU6?bdjCCcIypCI@u>@pbt@X-!s&ApD=kBT4<=7w0Ui`v5CcEUs=8RiF%EQ`NYsVh! z2zH!1=d;~b|{Ak+U~;*eCzz|7DP=KY=6JF_&-MiVNlm4W)o}!srd6N2#P=- z0Caw);n74MWd)4lQKP4bZ%c3hV#>v>^G+PfjPwL`)T3#>2y$rKxFa>{M`Vb?*4__y z?%M}eLhheXAX8+~C`e6GYMdY93wA$eNTpen4I{|koeQ|D$Y^4s~oLc<}@o?btJp&Bk{h`27_?fIs3Wg2g%gVNJeHpr#4zOZD|d`bl0 z`B1<$QRHUbV3Q_I<*T3Mel*!1^t7>(L1T774eyxH-@54nF55Y&ULC^7C~={Vb@;+E$73N(4a5G^x}#;|x{5iOi&@s~%!}!d9JDSLPzVy+-b1;k_aD21z4T z6qaAUfI@zjBVK}+@eBGiUw4Yn6M=Q0z~l4*08WcN)t|h!DzY#=7b^VBE=s*F$~WA z|Ad`GfCpUeEd%^4)*dKxxHr#4<*8D3JhcIY7{JI8zB+UV3uL!KR!h>o6QDy+vSj?D zS%1)sogCwX_aI5n=BW-qfNsT;yXE!baBtL@{(dY4b1IMj`cu{qYhD89G$YE^hqbQ>w$B=8CFulm$& zlQW6VMBT>>pJPs|v>vrl4$X%B=LD0`aeUw1u)CdKX0Fy(ZwuQjev!c`8grN-0^Dek z=5me+ogwpu`mh|V>ac>c`IpOOAs!7UBzc-ouui_2T3x|0<}C`#(aXTTr@4+E<*c%! zSy$Un=Z;MMu>vJ#ormD_Hp*c*e1z~g-0fPwr)vza1#$(S5`IoBab)$KHOZ&*SL~jx zV0eWZGqpDjnr+t<{?$g9Y|@V4hZzMa?Rxk?OzJz|TS-E4}+JIY7T_)8^8n}liBvhKB^ zGYqdy4Q7V4nE&34=h)i4=~^E_%35@U=N^42l&LO99r!=m!AixbR_5tJm!r-f_r9DlPKZhvDwAYa1w!nRG={RVZ!$CD~A4WyH z299)@86iH4e3p5+1^gQJ58yQXoFQdf)9S4OJT(!7z{sf_tca)I)6jp&!46ET$8H==(x@Rde*C^lGe*KRCk+yVk7^?=9MK&Um@V~aLv4N7X8(FQ@r|C*w-Vam!{ve z!)!GzS(bLE{WiWzLppBOC1u{jETO*kc0;`%%SW!R+9bTk^8|fkOYb-&4R+0aoWDov zWSSBefo#&B`2*!?^1=eEhwRzP%MR3(hD;0Qx;;f*!0Lr;VR&f7a;bZ2q%uY@{>cpK zlEyKfPsjoF1~?Mljz|0ORY}?OYc?`7?ZrY4NFRLZSL*Tb=isD?Ov=jTU9=?=NhurW zjfSeDA~iue^ghBA0&ck`Lj}u0L&V!y;#@s#(Z>D;3*+V+%Jw)8FSD7Yi3D3NXZ%@4?4lM`V6V2*d1sU- z$JpRf<+#QF-*fuvXfiNMPh|O!rC+mnin$Va4Dn4K*?i}7T#1;Mc4C#Id5{KY*L+;P zflbrV;o#aGDlC117sbStR#4%q>kO~sbZ;g500zc_v!joKVVb!dhK+aCsRFbb^8l?S zXzjk-U=PpI&zCVTkV^pLkbE6A^`MxvZFTrQIynI-@LUZIxipgCjT#MX82<}Qa`ybr zJL*kQ1B4nxsk)m8nh5h9aY!jzopHSp=D&3l0-Tk@|*Lk z{W$2@9TZSe!yn5q5VC!#z!0=CheVgwEDD32o^UPOSt`umwb2P6!5}GgIiA^?JZoio zxIWu-D?)CKNol^096yMd<~0xD3dfJAvKp9@%3I`BRfVRsR8LYgV5&l^TVQc;HJL%7 zs>fex6LM-*V2RFP-E!JAy?Q-|nwB$6lJ{MM?}HnLxf)|d&LM*~oU zham=B`R^Ro7ox8F;Uzjpq(cn$cco#ZXXs<@SnVMUNyBLfLf5ssrt7p7Q{2&cQ&xw9 zu=%M>gH%to6lgz-Ii|z5kIp|0y|7rT+XI@z8vGSo@z~2DF?3(l zv#UDH?Frb%xj=E_3+U%{G1e55ZeL^?4aymvp;nvkR)zv!Q3gnOv=p`uF!I=3zoeQ< za4HK`UoV>;@CgfoR*&(q9JmXf*TEwDj-W!)z z8Lrj&l5xWtR$ylmR|_V=rT8El|8y^$ps(Dz zE`DCFjr}kK%8OUQ#nc3k--~74A~NE+9Xq2fYfEAgh7_(Dv~^B~usY}ZG=S=tPPp93 z@?l63KA`Bo!>0FhWLGi+uqZWQJ<8Bfr@-~m)O$gOoW$qu(2)K+MhXMIXfE$y>}HXn^{!tV6X{F>sVjaQc?TBXAKpDd^wjs zCed(OhK&JDI)VQU#&4C_c9q~}6G&AZ8PkiE)X{msq9<X8 zhp&{9*N@^i;ix`tJL&g%5h*$C2^+dU_pPUggmT5Gr?KsvoUNNDXL`|$4hTuG^q=?> zrUWt-K4Efj%bM3VXvMss2ZB>b!mPwXuL(;bmg6>;f+eS^V4UgS={8EWWAorD2r&-u zL;&N}Yk6bX2BWB$^tc?E&z4VmgcZ^dpMOTl@K-viciRpG^bLy{;-aj9ry?LGGl)P`lL~&>|?kNrloY0 z#?0So%f>g)$+>~^XdPy4_otQfoo)j?%#bBc{-HN&zfxRm7rxmIpprSq-frA+Jb452 zqNku*i8`4S*D&q+s`CQ}fVBzpb4SP}+%lAq@t!7T%@n{pE*Dr4k-*PAVZSh$g#)mA z#pFX9K6BC>Byp52r?DTxNUfQ{Gg;1#jL)KXAzA6J?*}-Tf9(y<&26o7}BlpjYJlo}OY;p2#v7y=h?xej9_#{Of zVS))1|7EFZzKvPAU!$OQUoEH8XPn_9-rq$j*1-j2Gv{?k`E=C}9id{Q8vYOsTkFKh zZ{yAlY98{lGb&vFsk@_A(54-+?iBlTBZp5l_KyJ?yNnWqGTtFfFZX!zh4x} zh$`r!83)XviJJT-dl1PCVXO*qgN!nHjKNOvTb*NuI)s4A&>qgbWCJDG7~TvyCC=sJ z9IQ1~ubfFtNTa+iQjpDfwMnk_etvs_%miApl2WC@5XOaLK*ouCyXfft4#`pafjNc1 zx^NL=+A2_9sh87#_ziPJwTQ(q=qF_VF@lC!IzG;SRn zuo~+V6heY?Ii*W$omV@}$CH(uA^n93yo0abgI2f|lFu$Zp|~1}UTDNl5sS7iX8_;q zl&ccV?cY_nknYW9Y$o-m=KnT{b2I2sSK^7vr3yrZzGmExa{SY>Y9w+kC|P; z3KOlKv+r8)f~cT|@VlGtCG+IW6otuzC~iV&_w1Q=PVMj5;>@smS}Pn_KIGV?z_E0i zpKZ!rqeX0c?O86|Dm*^S-|}`)GPRm>+IwB9-!1^y^8Tm&1EwWN9KiXnLQKjf^XM3Q zHbseIobQU0C+ure6{}raFh&LlPw1gF^F7NP7zew-VMVnx%FplhjyKGbgKr`~A705_ zj9Nb8K{m?O|H9FCQ-EgK;Z}}qMDNHjOwN)(4y47JJ)uRup^^!_O?6C~zS#v;boLI@ zz@OnSIZc>tk*gOWFJXfO9S^qADiLKCaOZ?*^brn)rZ2KsQ9V6M=(V#7d=V!`A_|)nsO%G`;PP(9iw4{(I{68sPoru8mTc8B%5NhJX zlc5v95On`c1d&gH&uTBV{pIFv@(Rq+B$O;W@T}GbvA4VarzY$%wjwP=OtW!}s|-8o zMBl{-%KKO!%m+-%W8{6P@8RgTdJpd)R*o>MMKk_iT!$@;yIwv z1b(c>9ETj}br4!N>Xq=LUcsQvVQy3f!Voz6HPk1t0Z~d2*ye=X?X)XJHJsVLDY~8p z4rI6)7CHquMZ(Lp0R<3jBTiY#ylR4(!H{#W=(5mXlKt**jM$!Xi%nhJhoG$0+#eVJ z#3WJPb+Ln4(u1&0C`|KMr@&^>HpJG()w2#9|(q899TykQoX{=h`Ntm`A)U zpLhc*{S+*gr~-n5)i-J5**b*qP5#rqCecC;IE_-~uRrx7?Syy)rq(%$8PHVjP9+D8 z*n?2B&zdl+1a?Mv_aa`%6D%v#BrYr(PAYzaJYRz>(>_=%iBwbOmD=94mtc%^89l0p zYNfm5&0d2Sm3xzyDgU)c1UEN3Kw1nV>G*3+jmL+ynJlr?{%g<26U^G$F%=1uGjZ4y zPqDbJL6$RoJT)BSn;p}$LsJVQPU=lNG!(tXLFx+IRLIio=0>RRw=x+2t3v>7vS^x` z_zsobZXknj<-o)+Od~mVEo@B7e<90rONDBIF+4H_VwCU(C(TKAZi*Fb zwf50F(deM@iJ|>Km~z^?|Hxo26K4Rm?wY=&qP^;sq;OOM)8jb;Izp!C7z^{U@|s8q zbf3NNOyW@~y}WcOhJt=B+oC&|z4QREvl7H~W!pZjmq0l8hGWSl#n>W7@%+5^pQ3VY ziqb10&kw*~o`F}7(r=Tsu?59=xj5kx2*S@OG&wTAa7q+r(PUf6LJNqtK`X=j&W89o zXK3Pbm|eCG(#3!yLBvL#d2ywDuN9ysrdCtRk*4B6(!kbF^&p>3^%13xtNRTsf|i3s z2SCKv_J{*(-*b)JSCnqr0bJBmrZ1#a=%;i`nJLKXXG z%KAh@8vq0Zg+=bUnasi`u0yE$kX}Q~jg1Uv z!BzqF=@bFiPZBw+ACHa->YIZaG$azf5CsSbYaZB##Kz1^&dAlw*~-XP)yvV0gquWB zMoyKPgpS0@{-@dA$X3PG!P&^-r^nUV&FrVyMcl#Rr$@%j>;E^|*ttmAySN(Jo0v(7 z{!|bZ6BkmGRsE^)KYG-ZW&c-~osrA`tMY#jG5@Lgb1FZFcsLt5I+~e^ipl&8H)mz^ zU{W_yma=tHa#zq$aZ(Y}))M05`x$5I;AU)VrtxpImAwTCx4Dt6i6^yyWP5lpdF2;ca49BhR_W*;U7FKnfEJu!%wH~RBv(CFRsDO?R!K}OX0 z>7_L@GbVc}+An`c2$FI?^2SB;W@Q>^4s2`3xTT8yA?^db!0MJkjgcH`1EIWdfzK&W z?sPJPaa?9E$#s~Vr-0>n)<3eK-%;BX2fewTvDzcOxmC2&^toM>BmQEq9ObplG`sLG zf0j!)@BbUlHxAMt+9xgg1|8opXm~Ld02fXAq>=MVVsuBT`vtRWdUSyjmU_!;mp^<1 zr-En&n2d7(fkgKX7;toND6s1!R1XOgkKZa?v9pci{T;ow@W*fl7tku-Q)G-0+P)KO zs-&qz3OW$GN4s+s739Vwn?6hLOCxJ((GHAN3!R3f@nIVC8)?6)vRLicrTKHD7)c}d zqR$Gxzg-N?*tl?YQMi08awC(+Ub|1RyrxWnwtvU;KVl zF5RB~XMmv5^95LZi$ce1LySr6{@}dyw_qWxl~=???2A$^ACe_WRIQ9^#;*sL99w5N zOc!Cvq2Ea?@>GuX{izc;efEtw8kfT+wthaeNL6_6K+nLmMOC9VIeAK(X`qJnL2$+J z2lvHVz$Asf0?o(~5W~*x1CLgG15ll;ZKmD7E)c{t5a`;-yN6K+7-++98heAhR+`Nu zx#?X8i~|Y|^h^7_uIXaEn19*tpr>BK;SqEFjnB@bEB`A1r_Ko(>pnCvlsJdqV|_*h zS1Z5wcAo%O!kuf?Ee(Z?qZ<1+?NX8Au{!t?Az0<*10pBhlROpB@Ao28{Oz6~XXOR~ zlrq6$HD6Cl!B*IS+mwm}0?(-Ct^=#q-;v-p(F9YL$AyH}Eh{8)piwXB(7S?`l@e4S zt87)pvnwqD%u@m^w-$X2@^}l-Na3nBTKV=#ypIO$md?cL-b=daV#PQS{vGJrG)c?f zO?r&Vz+qB@fDDn7A}YRh5rCOVxHy^U{MKY-Pi#75i+(juWv(cE>IZf-IHU zs#cG~3!P&XELWOxQ+2^&165w;wW`a7La(Z7G{-cLpo$c=+3b z^n|yWXt~s|HJm6IR9RwEjIj8k6l19AVRb;K5RU_s0cwD&IDN~y$wC{`*b`aDUXJr* z%1R)OHno=Y4P;7!ep3}0rBLbZEAn}L r;fHO4qt^Fnj9-m+zGXS5INxKLzTx&xE~~ymU>t%00V)3cYw&*n(g9`3 literal 0 HcmV?d00001 diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-ctr-v1-uncompressed.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-ctr-v1-uncompressed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..43ecb0d14e6f51220f6278931cb6366d63861a92 GIT binary patch literal 21202 zcmWjJ1A-_z76j0?ZQHhO+qP}ndTrabZQHhOW9F}+aw^G96axBE<9 zR0fk3jS?8d+X<$~vn;zQbkSf94#L%c7POJ_YpMnnckXng#+8ys{(Wz;?A?Azu~FrT zDp#vW29ckfRqxX@dpJgcM&)K`J&U_{g)BExs+p zS<3P?o6x_LuN{k2`eSA*M?vunOQACo3Kl*CBCbpgtGWf{>+F1Yiul$&w6Dh|wrz{1 zZdx;G7M@4nvU;W2^-PQMcV8l1I;yFHVWyLVqs%f3_L!SsxJzN*w~5~c{**vYDtQa0 zyD@yFv7+*s8EQ<3&Mgn5Yckg*_x=)PaB3=ff_9DDzZ2spznFCuUTTx6N70}OIR0!5 z>D`VEY>$uIT#6dJm-~CWcrZ`Yy2YTw=Qma6Z?9HMcpwdGPnPmJ@NcD489qc-pisyJ zw8m|QAK)g_6d;rOyV0Fj<1r8DtEg)?i=3y93{Bl+^P*)|^~q9?5e_Ric|N7=jYe$o zgJz6!ik699<)OZ%VrFZA9 zbDG;j-$NTXQ$>CTqQ%6Ysx~Ky`~Ib1!o2*8cA9_0SF}!59=L1# z5KWKV^`y8X_W)920;n^ar~5%ZL7{JdrVl|3%o=;(4Dck#tad3wb86 z^%Q2E3nn$HYnd!|d>ZF!1JE|w+(k}Q7Nc_D&YTp(_6}X2chUgS^+ym$x$}VD$Y|s# z^$451rrRZ+F!Uak>+Sr|EW!ZU-p{hi+0fbH=mReY>3*$dD}z?&+520ub*p{#*I-{K z;{r=>x`a{&Qb~t4CijC}0aut0-0P{;*}9wOYAjnXkiSg)CpG{lgHi=8#LBpJA@(6&{y2JBd$@WZ4gR{!O0}LmSHX( zjkdgGHgwPTsN>kHoP)i17ZHk__8?kJ3(Kb(2NOiZDpH1AUD%Y;yr z{dx<{LTYujV~9yu;FV(ku`Q6m2i@3W9s||Czk^Oeaa}%!f9IBhfUb3_e>3)2<2N5E zfULqEeQI+T?8Gzc=ZS>GMx*N9MVO1*hjZjN_%Jrl$?3zo$y6je`>n*kzygzRg#jW) z7o7Vb`m!vD;1M_SzK*rV>k3xpORn^bd2An#a&H5Sx*t{S^_QW8XGO7rW5qHT2(KRo zc)jY8-(<=6R%zo~-}Eju9n0XEyRF*?hM*}BUwwq;KI^`n-P&05x;Sv}d1`{HmFZz1 zBcDqpYNXwb6@oj7D9k8;(AODvv(Xde@}k0nl`)UcU1A*N(R@Wqo2tAH*@|dT)6VAD z|1$rWyBI9IfsiP(gi$Vk7*l^j*c(H>AkK6gy1z(I3|r&iFO^YvPs_7|qU2y(O2|t# z-tob+MG`bJti}DEw$#}ngFsN*P3@9MWXF|%r}gUj8%Q?Z?BwvY!pmG7B7DmLjs^4A zMma2xgfG`Ui0BmT8{}HExTd!wPHOOXtJk-?kaT2vz1Pe90r{T54T}O*#ZnwrVbCVx zVHc`geQ2)jPnF~(SdY4$;`D35NgC&mbSPnEVf9%sk3wI{J4vRHAMs5nOgjfjWv*9$U>bDTh-eFJ zO3kjYx7mI9g}#5Zm91sq5({!)rv2}jpKgRwvO3(LskCTcx@8dIQ`!UQA7w6c{RyE; z+XPoylFENBTVTvVi^>rzA&lxvd7fu`L8JaPoje>=C0Xe?(HKDAqnga_GS+AlMGZB# z#5HNlTjI6IL=hXdXusPT9@UAF_C8;iS6K7)dVaaTVmb&_^~C9jV=^S8E>HY$gh?S^-c)XXGj@AhRFN%oktk$|Z$5ent5 zWL*eCebA9(L#Z99>)fSA6dztz{$l+k)-^|3f?hIV4qJUsn&g{XhzvZ6Sye@zjFZAFoRGI!;1(ePK6~*^bhS!?*h?0;+ z+3Xpmwwx*8bJe%;-nmhabl!D+4Bc>FcS8!}ZU>R}DeX z_6C5~480mU5lCQ3K$cF*~7Sx_J)2qM|lOui7nXZ7u$da zG0C$oEjbB=+@2%ts}`bZGHF;-yc7x*uSs}@GW1kleuI{V{g+M? zHwg2?T936u*eDk;S(K_vLTk}U+ZZR-*Rs_&Csi!qKc$81CL*_B*SPI#%AaG`bwdnU zeN;fHQowBNN%$!2C)BIcLO%p4k{6Q9$9L5bmd$~k5&j6jR$C819DdaPgS?;!1^bU_ z5u1Xt!NKh(m(U(r;IL#L1CbW}cG@4<^(JCW5y@XHoT*KI!Y(L^I@%u*%>OU;DkL2T zJ0eHopEqTGpycF#qR%`tCz5ZqtZ~wThN^BEM>+W8PybBkEQB}zbBT`X+ju7#!EQ=IX^&G8Vq;Zy+4pC zH8}gPa%irnA1f$Hoqv2F0!mI?)3|6 zjOwJ5R_tlHWq&np2W0QRt9R3ZphO;61gt(3(o z2LbwmHwgU5ukc;a3OB9y@qMxuwR;=wxEo*HhNm*)cLg6Z z-R&NrjV*%^4hj36Z8DSigHsofz$AOwbC4#;C7%KSqdDJ_l1`=k%F%qDX5u{U46Hd@ z1}y@4gGRzySpy@Pnab2|oD45PwtO6e>&Iro75N&wqtF*m)Fei+e6hoP3ys*eAl`>* z^LIw}@IUyr1hP>#Cx~rl=M7+|fn-Xr&<$%J+E-#!YMDKTW>Lk<28d~-1=(->Nf*?0 zd@--2Kf_6|;N|5VI-R@y!9#vvInvQez>yRkl6E>#B zll2piAIb-<5zx(A<=Nb5EncYpL+Rzx=;$|q!by<;TA&b&yoILj>ESjm?wsVX?<%Rq z-@MJS0-FldUEGa2jL7fzvOV3;v$0g1jr6{BBG<%g-$31xDvsDsI@^pt{Tyg z%r$15ik~rW^wfuhS?Q`IYhPu+H9tF~lMGAF(z_TKZCaIQ%8+x%vj$DuZkEx7`@ zT90AHgNK0w_(FLB6OFX#@|?4%lPkTq%S5odEqfiuaw`5qJ+pt5$Q|WKW@t9Zx(o&? zn-C7?xas+md|4dF6zxk18lOb6)dKxs3LjxOW@bqbVd4JpWd-fPUI%m8urMr@M1ac| z+xo&A9Hm%S=>3jT^TmgK3$UYaOTz7SB=vI$%Zdc`&fTegd9~#)>5iDVqh5}sB@QEp zzlfFcj(=Dg+r*AW001h)uIXw^ovqDc+^#Q`_d=v+t>w!Oc`)tF_uQ7Hn4+cT9@5{?BDe=AnSi;UUMoqvWHNuh9UN9_+^J%%Qz-Mf~?; z87Lr#6jeS553oO>y=Rz|oL4vCnwD$8r{FY^-H*Or;ZfcI!72|DYblQ52Gx(T=xzTG zZYG$KBJ~i34XdUue)vrMKO3z7T+$``BKen)iTd6s>yVN)_rdG1{^RraoIgp6^{8(sA(JMf zJ$z@^P`@el34aj50X@|{5l(zJZt`Z*64+(2(?1hxJfz@d3cEAaj4(KIi0Sy30jIo$ zQ}`jO5)8;6JX{=jo!)SPoa}J2V^emX+1|MK2FMXUn^O?UZa0gluaodescS&$)1 zK6w#Ja-u}Vs^s2-60o`8#J7LR-mL-Ib9l&8Un8YhLD z7*C*uNd9phokm?~YaO|a8i=v?sraj_+TK%HxCr$~Um;GbxGnL}uEXK`1#BoUo&Y-! z0r=doM*8kPf5ncIdRlHX7>6|5s}pLu9kYuepA@RJ%m+Uy8{)nwsw~ink zgnN^ODlX|=_j6Ep3(K_(>P+sm-SCKL7R5YHr@ojb!;nU(UO_v6TBB5^ADdUQPX>}h z9x-5LV)|1$&47wh4tL+f9=YEBebImSs$+)tD_b9!Xr$2Y+lXr-!^V_(EJOj|+sLiw zFUql_WkZHr+TqDRZihOry5my$r}PTKqh*7lEN#D39AMes)(UE8MSVYH{<3f`nk=O1 z{n(x`waIJbC0aEx{*T&BL&ywxcdt#EF~=E6c(%i9QS^*i{1~dQdTX$^`v6wWu0>P$ zLNPWIuoRdLK-qFxOGiQ;Sn|Y;^9NPk13IAyEj;fi(YhK*%;DK@B$Y$Bz{Qd|w)anm zJ;d7J$o!v6odR%*xjxlO;T@1iHOX@U=!3#$N33Tl{StWSxpp_3ZZI5|-yVYp8%-v8 zx|hoTU3M_vJ{4$Dyb`BB>^$EeQ1T;^>R)K)H}ntooo!Z|Wv24B7W1x=T=D-DIXg9> z$-LbiFwS>p4hT#gac?81q))x%KM5)A}|GxG)z-WWUH2G zTG3|X-B5}ggUzPkw3PrsWot=*M?&&;JMWp6GoBk4H~ZMCltd&UX$auL;OgkVUe6NOcOPZCk_3^UT6WCO)@HZ|SA5~XIOa{sZ1eLE5F znliBK?o5Hf(6P79VtnJX_50FMBml(sPzm20pvB#Xy;^ddoPwL`W`-be9^NmQr_ry( z^TxS|c*1eGNB2-0uM3kPBoX|LbxPtTbuCz*a{hLE(o|`8d$#WBly$oOmn7oYc;&K6 zl0+|}*wLKQp9uauk5HIa$8E8^%;Cu zJ?{>I@In}GWJ^PD)x(W>G5eTnRX{JjfUwj5le9qhCJ&!9mU4{G)TnnXWGVcTR2d0D zS%_{A3kQv)HV!Iw*NuD0s=xt&7vdkvRO<*XEad)UL?=llgA6!PsFfc9-E91$KNepV zo{IAA>A<7txk2TR%S}o@QW=}+toqcPxPO6kZl-L6FZppcdV={bD+artAk|#7d2Wze zR@z(Fs!2-&^v`V7nJqoR$`KG470zjrUyWiH%Dz1if-iI2>Tq5GhLI~mzF+wpl7a^N z%!*)PStBt3E}4vs4YAZ4u5B-kGmjRfs?WzlY9)U)%3EBc1UF9{XF3Urjn@|(?@xO8 zp75|+vf<*0T`i&4fbyUy-7WF9F=CCH9>MA=oUkzNey4I@@$QA0@n0v14!$YV54_Bpi5}lSXhJd_N^#xN+a^Fz>!-HAs2FHI9F!D?y%O_XQVoRwhN zCAV8EajvCMrX8fts7G!&N35O#3fU0DnpTMly-S~1Kagodo9rrtZovE^HHoGp>b0gk z0`(etxUQ|bup+m4(Y^|bWr6?F>@yqPUng_xOm6jOBPbyc2J|ZM&OY3s(08>4ASn;uc4loW* z@&q)nePO~!QMOE(xOODL5vCs_NH0HQ$6>pi=|;9~)0Q?=JS#fToXX25L-!*hhn9tp z7t6KqDMTHTM=6v4;q+}{m;MZ92~9aiU#|P?bWZ@ zV(UX$$*NE0aw71^pr7YFdXs25vygaJ9DdJD=TSZCvb<}_KGV>I%i%HF;JyTfW!2H3 zSO1;_U)Mst_TzBM(c!K+k z2uOXT$jPtr+n*T8f$G6g2d>Nd9o}6;V6z{Or1H$7;+Nh1A44X~>I~g@Mji$WpPxaP zO+E~SBzT;R*Z*4f;}dXN-Tplf5}LYQEy{%#tDxv=ClY>Uw-tsSKJzLvHU(0sk)tB4Xm68ns&lu()he3KXeIpd#~c|ZLr_@36$0d9_6eU6|4sI{Hf<;VV^S5 z5Pa{(U5@A5xJ)bcC~aU6dPQE(a`0$>`L(bFS9T_d>?iNkMbX25iGw$yGB5k} zLnFdL@UyZEX9?xf-*YijhY|TLD%;jvj+B-T9!=PlZUbMV{-QJTAi23EIm6mhl_AG` z*I*t#95C6U11awf8T`K_k;z6^ZajP`bV77uahkC6+fjKDA;#e9D&H)a zqoMN0S*TB3KaYnD=U4pt05gS2#ev}CEni`OykNtlnYIjekQ*u3nh1;Xsrj^@|8p60 zhRO~gt=zEct(9?$ncZLUfG(!-NK92xQw(@ryacH0J1_Uz+?QEnC}c#3*NPpp-2-`- zd;fYw`JlcCVX=0FzOu}BHzCGdrFglD&fidb??&kHoY&buh+@THC0e!5XW$SewPf?a z0t7;+k>6DV9+l2!vc~bW<3fHy<+a*O>k*h2NO>Os^Hg14USkSUkI{)&s#q461+`L8 zj{_zHU8es{O_2HxfLRjU4$=c}3pCgh)NWhx_aU)8AyMK&s70~LX?77%ekXk(6oQ|QA6CE`b!7Y)dU(eWE^6kR`bXj(8vvH zJI%33F8lv9HPmN#kj6qAyM5EqWuVO9Jzr{ZY9k@H1qf`fpo>X62~h$xwBHjbd|ChgXk6l5Nx%LU~d8H_|vU!GivO{V)#|*4qn+3C!74I3iJq zCLT!=DFb#@Q7_05_2u zDC*59BeQAbBu@Ww1D#U4-~ah#9og54$mim+BM>fLl+rrWT_KHOg?(o8^-o)Xnv8>< zc_9t{kz>w?tf5zYC!?jr1>JmdK-!ySwTYimFRsr(*A^1d77ZXoNTgAv+wlEMtbv-^ ze-Z(@FwFv(nn{nnAXEi$IreH{;%6?N!QkV&{8-#st7*YY0*WrwI%`lnLzz24knz)K zd3(l{eIfozoj1M)e^~N~Y#Vh=1arRE9Tr5VaZET1;dH!Lh&f_=t+}?1_ur<6GLIX5 z&KFjI%MiRwcHg%@(M{izL3QoUKxY*Ebj}(ZRb*d`bd`HO zI^1z_cu@n=QN!30=BnngtNw;AEEP<*4nfMG(rdJu{tm$ftWdB3;W5cMq|cJvVtd0+ ze_FTNHIosa_gGK3WU}k&>Ky6E1dFZx@T{|0_`Ee8BPS7IgTaGnwc z(75RWr>bGVJ7xX@mTIzywkv<{6Co4Pnf+@C=V(BF@C4D{h2P|r?)PH4zUv1dnvUyZ=(;=7fq zL=*Y6LA?PWp_q94+9*m^s*q4$^93&)I}>{ z#M(iyO@^5P43ZexmSczhrsjw+ZG;4u1ytzZUf;i%Vvx$<=>!9@_;dq#KQtKqZ6IVR zP$)uTENer7W`+&<*J(TCNRq3`jq7L3j8|B&n2$S5AmkPr#zMu9KT)KE^LQuLLZ+%@ zF)hN@ymP;lBX>b$0y@;(Xe(15uXsoUKI3{u@vRIXTmcd{8HGAd|Nh-IQ@)j zF;>%a+~M>7WN3On^Rks$OQk}h3!=Gs7ZmiD3fm!#6zEnyFq^kP6_5m!9v_AS$=$<% zQe7UJg6YHo?PAg>4!OO7-V+c><@LPMfuef^*$Rm9rZ9Ud_0y*|qlNYx3Nt70?~m!Z zF1zfr@TaA4>FdC#ks5%O88;hpVj`wcThPZgpf3)x^a{}j$ujQjkW?|53_i)rupAw8 z#8YaF3n!D9uvsx3=Jt*5^SM0E-21Tdac3K9aRW}h^h6_Ii2Xw&N?;i-e3b8sJ<*@3 z6E>!Qy00e{AO~8g50I4K*HRW3x_XqK7BSr$>_|nCBnML^SbSmHSs=(=QgAQ!0e=(Y>E3i>FF>MFs41dmov(@<0= z-B}!gQqkGzeA|`&5!hRsBv1b|(ddJir+tErFF}r3b}^&!ku7bpLR;K=<;MK2x8u+U z0$A3Q1>U^9=v96Os6p48rZxzY(dmwWS2wz~CO9|oSK1Do2ZZDF!FZg6Sguh?-*KBA zSYqi_i(XbMWu;+mSc>^Mth)Zm1g~EoFnwk4+44gV^GC4-sEG-kuz?@~u@2G1T`i0H zH#X0VjUsPt4yrtiS_J1ak56d)ko3%vJ^y_p%s2z+5tGWurbt@~>(l&!V%6Fy*(FV< zR%6@Ya{BF{@H%&>gghbE`YzxQOpd7?Hm>dFFi?~=d3+DIoi#+R)?J;KEWF>JEx2xKO%ace}(BC?F~v#B+%^ z0wX_mjaht>7#Pjpgk3=?-tL(c_zGt$WRHFK0xzqZ-iZWz-8xK^M3_u%kzhG9Qcqg+ z0*b46RA6|21M?k*u!Wq$?D?>-8}iB61gKc_6m^4xhAmSw-F({{eHtv zygicCWq``*kzvoJLJT(P2krnUO7dRY3&?JbL1JMuzzxL~!={XxN~SeC-@`0`E8`s^ z)3ka6J5Ak;A>h---HQ8jf0(K0#k!#cxM(o#_#%pioC{l+nu zJlNxGQR|7ZXQ^Js2+)T`GIQh>=f_nXyd{n*j~)X}9?QI$Q2NS(UJL-fD$IIGLQ&LM%S2+I`q%uU!0!w6?N{ND9^~>HWyrAad?X^sD{0v|{FnD&mZBdW zOpcJs@hz~#Fr)Az2(*<&vljn4-&uID2M1XTIPksvtKITRjmUMJc=rVNPUlPKFJz&N zYBX}WdudIZ30hT816&P?1KpKQwQP66d>b;w{zlV9G%7>R;yyjEpw<*Lh%k>{F{@BRIEK9Q7p21v6b&S zId!5#JS&Jmj&7w;0!cEkw(TZ^2zoFL3#(PFKnh2$P7c>$E@x_nffX_^ea%^g9zHlH zP01pYCh^b}+^nP^XdV1dScOf6i~v327R{^2Yij^T+JFe6>ceoOwPvT6Rr=*g4a#LU zK#_DE+E!!=yug#oI9!Um@3wnM@kS$B?-6yQa37qP*;98v>V@NU>&V^+jUH?B`&nHZ zVoF>HRD%GgBsWoTj@{-?!8g<&ug-}9zb1o+OU&m;vov+Z!^iIu*3Z^=7 zdBYW2IO!j_ztB!kOi85>ENb`g-jBtae#4+2I;c{*4dtNO*MKwJL_; ztY8ERe8=xD4y*XCrhf?dTa?%t$eLVlu&AIYQ1Auaz6hD8 z1yestn7JB!1y$ZzIh5q$NcO4!4`h`Xc-els$yHa8%D9o(uyYxL>4^&Bzs9)0{nNr! z+zrxxNp21or2|ou8Q8^~8ZE40av173BN3phIf$v2M&Aoqad-XhWBF%;45Fh;REm#8 z=-u-&p;-58NmOZ>x$a28H13yTqZ4X%<*Y!tQ|75&>iG*WNnqBcvc=@LHe)%P3(#A* z?TB%AM&8q1m{OY!iq*F`b4)4Fui>|I#$Pp z#`hSY6Y33s!`-AV6vpdPW&WPfVqwEn+gF1&wO-t1=s6F-ya0B0J3+w1`DMi#KNp=u ztA#ddgl#)f5d~els};7e52SwU8c+M8mvj0jn`(jT_V(==t*)+ybH(9A51ja~|?le2Yha09+hNzt8xI#~2pm@mxz$wD)1%w=~Q!0dLhz4gK| zmA0cd>&^u_Hb}L<98wzIDydx&+~KOG8cSoI)rReQtm*|DHGy2;@QIFpw<}p;sR#tK zFQEf+&{*$ZHFHr+7+ZXxxLAfT0w5)&TFtj5)18UJEPIOhATlKJoMx|^DJ?Tdmjsk@ zwCLv&Hq-6pt%^cXlBos%((VhG4r>{N5TYL%G|d<^5J%t5s_u#g*FkItac-l}UP&IANiSTeq$l2j&r4?Ue&>+m8$be@pPs<=Z&%*rqkkD`G5wW(U&{3zD#fG_=gmP)o zFXtp^QZT)nlc<3nhhsRjFv0)tUuE_@u`bMq6$L-k=0o6hV7@n+qpO{#zyCFce#$)? zvN@R}D1%tzHt&#&pWS}w@Guy#SRQ$>gQu`1ay^3qgNO(ed24r?DMMe8!VnjTy73eO zgmE~}V&*Y_G%PyBzy2rUj|l>3SF z3XZn05N!Q@{0>iw{nGouBz4*?`>zqivj_sw?*A`!iQnx4BEarHmFS?rkW1ksw;EIt zws+;YFMIb8(VijfETEjIhk;LX$*qL(hbtBYm3ck0mUEa+WiskNSnx0S8ttl>5`l@u zisik6Y4^ZJO@vQPf7VC3o$G^pSeCN(NT)kFmqC_=fo{WGQO{&h01=hyBB7R2hM6 z57M4gA&#mferV$3yU})R@v-U}?E+b7z`*drrn>U|MWN$nrOr-M=zdvay?6Nmbi4vp zZaAjh{q+NB+82WB_|Nx%rEyw>otOV0S^ibn9D%%RAjbcq%F8o-{PZCSv@@mKGckxP za6uC>Yt33gfou3F36DcPGoJV_MPXZp;h)yTzv-I{VAZ9m;&Pc>0Uw-$f=-_XhzV*p z%`Wy@5Bnzt{`2V(-A-Pi413Dx6fmc3anRDZz}1>%-)Q>n1%n5nwC+Idg{R$;RVI@! zOfVbA)WK(4m;0#eyXJ4(5|waRcO&1!T`f^YSn`k4bn7E+ZEUjUVfg?rS3^8}DuY>R z5i7pA$e~TGl8!ddkZSqZ6-v8<%0;S%!(`quWRjC$h%2$Ly{l}Ms-T3H|D!_0T1skd zXkg+xvR*dz@-rB1k2a0*r4OOK4W`oz39%`LY$vRKL>E=Q6v@$v_lqvVJvt|50?Hx? zfvS&4r;g@yF~)Bp2@B7ISz&q$wY)zkE3q{balcU{Ur%knQYzv)w7J(c0`nPiN<#8u zRJ^TrU{koaa}G$&Cj>qo7Xx7b69uPNzq7XtsaDbLW9zV#K=IFZbBQR-w$a?Q2QC~T zpuV{q1k#$wu(Rii_uTH6`_4}kV89A<%9*uVcOWZ9lZGJ5J0jh|H+ zS3Uy-0I==CJfXoWphQ3bRi8&pCv+N8n5o&A{SYufU53-l1MKTZMChnUr0)fyS8y9Af?fb>ku?wLbvKRVQ{$- zWq+pYvCL5@U!9~4xQCj|fM3Vrai@of!6BVB$MX|$BI+8Y1Td?x#^=^aN=~q+gP4DI zf{9{Re=Lz(Ghg;HQYk0MJ~eNq!+>?ujcOsWlU|lITNQ2s87!>fM}Rs-#?iUUc#_g` z<5E-rn`jR@=G;s`HGWUprO~{rk>Nzrrevc~CexB&)byS(Vc6To%NY)l-YdaR$yz=J zCmjGn8iG9$<$g8b%hM+;xtrINfVpI67V%U~9_23R3_BSQX$VE4RlAtIXQYu$C!mD( zCz7fy8=j1d8fvi?LNyw^#E7lyV(ze8{;<%F$EsC7>KvATiC2aZ3N#fq6Ml*qlIk%0&AsZ*G#Whhq@cbFPUUMJj4Vo4?jaZ5lMFWF3ue zMx_}Vx_9aQCH|SBwgNa@(gcz`Bj1kwE$dG!rJrma3A|y?;ZdXO3H>}U0X%_5LA&vn1qG^=5q|=Tc8^QXtPN5`@Bhf$kfe57V5L_&>}#N+e-(f4jC{7 zPDRRCXanqUn)>@v-70=6G(h+y})Mi0Ogml4VMm#_Ksm(X+r>5PzbXQXKfca*S^wCx@n&sJ|n zIH(q;C`ZWzhgd=p_6pqi%`T$6c8SA0$%?#Dvp zUux8mzAmmC#f)ccjJ|XNE0uyE@NF&29yT>q300;>*=PNGZHdI-;ZZK+8ym3lZFWh^ zXN<|B8=n!TauVmn6zXI#YRXMf%at@u`KNA7A}iP{enC-C*ohItBg0y()%^;hIBe?ea` zFS^Dsyq#_+eYRcKu*VBTxEM7-3YWak-AJgeufq_zg(r3fx3%!(b+OYK=8>|I=kZng zl$Y43aEvrxP5|WTmP7^Utm>>G>FNr7#ra{VB()Cc5_xFI2 z!+Gy2czA1>H>lG>Li7oX!Hd-t!VbRT3Y7#FE-cmBC4t7QS?AXVqgQH6jmU@8Kyh0U z99EY5u6|B}E0cbCctr_cN53FZ+RMYqISbOWm+%SaDk&xapeC$r!pGb z7=-MaKbnd!1ypX($N6a=DR&eAUTq{EF*@5u$PQz$>|ovr)IK;Zm5OUt8z+P|E=!%@ zEFsKf=>~_kJPV#i9}Pv8n2Sq>>z6TUNUIsOSM3)z#-_GZdB ztjS?gNHk0s@$Q|U0(~50wdbYZREA`FS2%P2$u9ycH^piQR;tB&^gJVwMa}KzT;1^L z$~x+!d&F8gT<`VM4C(jD9sx)za+g$+J}vE12mdKOZ>y(d%Bno;g~$yi0-k$)s=FVG z?hgU8qgryI7F&y@Frp@2D(0$Bxb2Xvl=#o5+yYKk;Bqywf3`;VgcLDe9Jow*U z(gwvG&SfMov(khS>fP*)XuP*Ono^QN`v?lip!WnHUc3=Lo*YwLxV1JE#jpD2Drkot zd+%{Db_*5SHFFs-A*}gKtN_$E_7|c$&eLS9t}a%1fim&I2eRE-h#Vw2%pj^>fYK1X z4bQNP#kPTl+0C0xj1pfNc^kEbyqoF+PP3e-6qmc-=sfFD-|=;Gg%VmJd~IgP_FZGO z>Nh*EU2_Z17*?(vt&Z!EtxL7~H-M0UP`NpM9npDxTx`sF0sCZ3Q`LmKl&rmocjtYw ze-7h?{Bs3?2JXoF%&6}AV+juK?$WnZ-b20kAWMs~-Da`dH4eN**4}D*+gE$WOX{_? z99O31&KYCxm3syJ2=Wda0<~}Ie8UT3*JUwk7%ygG*p0ch`jiFYme23qK+2_UR%Luh z`Y1%LQ`T*{@eb2TS;qP?)D5d&>OQ*BzCK3c)0|I{&~V8bWzmL4f1ra)_N8rFBh zIorgt6|OA|$(~BRHJ{NZeWxsw>ecBPN67Xd69=mof>8=F4z4` zdZ}4frrfW=V75G+budd?AV-2K{wU}DEk_IhWGUZ#bJwxCtth3PRUsuNs0m8jYuWWx zh1g+qtYN);HPm+S4C|jK>dIcoO0$v8sc~^BsV8*s`OAL%HLL5wW&?2)D`Rq(lgz-i zsmYecu!zdfx_jF>YAFEpQrO}u?%0aowx3mP={XGA^pY?lXCW+PPhs#r7_f#!Zv0YT zaR0r08lDpueuZkKA zhC`FY1sw>iFYtY!I#h?*I{E)!>y@b`LhPnSLcAG)r3&GlUQL^5Yj#&-!?}mjdJCb& zcWJW8Fv*&UY!>%kUo&jSo(Qt(Be(rK!*ba@h!z!{x0f3Etql6ZufK9{Y+L)ON5--f z5@rKeYRZ0CyFAb$STf!E7@G+XCX5(2rTa!nb9CIVG>OuT zljaZH`Z09JMn8r>7NX>KYRV4Rd;L#Ymn0GetQ#s{M@ooqy^W0S574Bg53>1ND=g>O1h>wb(i<%zg_|UIt}IB zqvOiQRIWEB6qsG~+g?UD7bny3R?OKjYGZi$nhf}#C64?#n;b7@8TYR}fmL4ee^g(aK61egf2qD(VI5?` ztZP!t?%Lj6_fwGsHq5GlZ+^<_waqO(Um6l%#+6Tggp-QNY~Y|Jwl$ySHCELC*I1_9 z=vZnmE>TTr&BTo-!_=YGUF%ZSSW`=>CpQpINvu@<4EVcr28FX9#zH{M~2kzuR26fD228;wdEh|IbIAMTm5E z1{Tuvwg*BE&irdPm{2Q6?<}XZ3q$)5CH#hzJ-o0>Cl^IxD@7teu2FDuQ*8~c&r=6! z95sVZa!sWM`U0?s%cts%ljMP5NL7UD#@mF{I!G9#&+RTMcAw*5%dvWeQB&Ox6Yn2` z+qHK0dtE4sE;rr>q@xz-jEaZJxBavpmF)1&`xIQWjXRC`d~8ZNYY>$^f}NbVXaK%l zycip;RP|YgWF>hStZMv`!#m*XeYWU=v19%bF{ z)T6&#KToP{J0$z%#1ym7(Pc@WQ$-2ff$d?T1`JGL^_b#G24J&w))w^x`QT$Zr4A}h zZgfc+z{XbUQ1$3hG;6jd-ePy{URb)ky^oD`;u0ORL^0|X*ru+T zvZgGKOBkfEvCt7=eZ=1TUjShep6xfFR#5-|05Hx&p1Hv{p7w`>=4M9^Y&zySC!{6Y z?lp(-g4Ywhz-3{VK$MYHz#bQuwb5s^A?B18XUgp}LN6N=NM=&cfy5}``3i++`zW8lOS;EATd ztj#_KtZo^Gk9NZbXl`#}wMN#+JLYcrbcw71nbgc*q*)t@(12vA{@{O;>_l}$53s_LUypZf+t3%J3uuwaQVF)ztT2-L?Zn zGaXLO2By14>^N6o^pLh5V|usk@1bucy@22lV200bh$k*TIaKK5Yy zZq}7YFZf)1c6?izP){<|!2&EovO$uLp!#2;*;LQW0pJE{K1ws*wZGf7#4AC~LXs1O zn~R>q`SwVWUl#$gg^-W$Va*x?000a!#ii-t;ce>8LWJc(T3-VI?#cIJWnQN7@4z0- zl?>YE)}mI?`Z@_l;C#@yy`iV+xYR%S{CHETcb4RteVM@{y_WpUNRGgbJO&C{BAv4L zk8=|7jIwTAfs|JJ-p^OF1lhw@aq{$<>2y*>_m-^=&3n~W*YS6_^b}0t@NnF~fFO?v z&~sYvy0L-%bwcZY_ky<`*ZJt{zIPXGdW=(|aG70nZ1 zs8LA%>wWB@4$_sZ$DiQb4HILLsCR7cIp{l43ii&p5ov(x`aQ;DF1Pfa77@pgg^M`C~V{R$i9t@SBsT*Sn2 zO-+xL@b5`z3wsCsROo&K8dOiHwbrfzHYU1AV@zkq4Zd7#b@W3e8dU*_Cc>cHvecM@ zBbguyk)6apHwIe92}LrMHvP`^$rfqvrRt2Zf2nr<#M-&u_}U0b*3nE^8D(=)(h=E8 zD}=GIOuEHAR`<5kXTFTN(O`0bdbVLyBgT4Z-}8$hWDxNPFq48@DHi*_65P{Ln91ioC6@S5}+OW$62Y7`lM;`>KwHaw&lv z#?y5_T+qbgAc&PHB3kqn;>)`s!(?R0A%5%FIXH!4FfeKY&{^u~OEdRUU+Jeh_R{dHrlTH&dfmd7$p<+Np4}t-c-kQi)}c#pb#k@WZq~xIGOo_Q2+n{R~N5Z`Q{jxWXP!q zIDbxI(-b=cYSv9-E<&nUCWN0or+^L5*xLy7z@Mj}Ls{1)2iiahn<8awG|GFnml*lQ zix57h4tN(I!1-EV|Bw7UxFcH zk^T_<$%Buw|HVbtrrhdW=m{T3wJ4o~2~l=jlY+4^iF9cYv@1BBrkjlBIt%d0!Qj;_ z>Z&ArnNfx5YDYD-KKXC(ywMB}OM4H4kN?Zljn6D#_E${;w4#9=mK4}$nfG+U ziUL#d0{i^eLVv~H4~p69G6KHzbhxfJ$wu2BVAZq_usS{U>&5BatC|7gHUI?1+A}zd zb@g`0=4ZM?lqSYbr#?60QtNViOY)z7ByEq~wU0l$+#muV41D8~*Q8jayFa5)0u5ZY zi3}mxr-**TaZa*dlv+tzYV@DyE*p z^|4EC5q6At8~@Qv_@*2(%ygk#O$s-z(z?e0gu^v2WxQzjV+hQ?Xi-aMG5!MdmL^s0 zE-V0?ReC9w<0a=z-s}xS?wL89bMV+7A$moL2(De`LAJ3hV)=`Nl!597*7UeYxnxZtKO3H-fpF!cLQzJLJIAMxmg3-3mXel|AKeBJL>9 z!N#6JmKs@%M(yXxA7ay6!=?G73lH@>hsM2KM-D`cHH5WM0002jUJEw)RZ{45X5LK` zw@|pf<-^vji9}gIDaxG_+5d%N-r4bV!Rh4mP^}|ikeg1jLatOI5*>wfQet!FMHmGYg{+b7F#-Sp1_;U&S3=S$HT@z}0;8!}%yAZ&9fuL>`C?o&3epTi z3-ZScd(KV0Qd&4h1Xlo3XaT-@h2CS>>5^BKkaQiHNiWLY&J1E?wFP0US}Wbggc`#N zUyC9j&M_UPJ3w6X^$rEu|_lA?eAj)XB8)I&;S0Q?3uz9 znJ}k)9v2kI9spi>?I~#m|8G(~EKY~~iL56C#>jK`I8W7W`gt+J9V6(iOwW`bf_ zzHb?n%`MdvL8;<PV4&t?-VxuJ4+L&m?IcGzZ zOp~j1L9^l8v<$Pu&$nM?D8ov-5(Ysif4U|NGkD}qnF1HS(0mX#)sXy}0(+HLAEz}F zczogltcBR(8%?|1$w ze|du}KANhRoSEq;fi^c9nz41r0!f>+hO1$~&w7!xn00019G1J*qy+sU* z{h=cfVuVlf(-(fc&-UvX1ELZUn8Z?65^wsn-!&k%i0vFKCN37a&BR4Q*>`~VP|C`I&^Y%Wh^3T zb4G7(bY*f&WqBexbaHiNEFx=db4hMssd)s57p4b=IL6GZvc9$Gyu zPl1%if)i0t9brq|&j-I#3r7OQCY?wTy%aYikcTA4EAjII+af*o!GTj!CWY%9HU<<$ zO#~ctddjg5PwL*b>|22b=dLg2z3lr^X{D#uY8DQ}gOI|+l|G&eZ6u19^zPMk4BgD| z>gq~2`Uy8_=Mc)lc<(0CB8{(32AN5)u@%{mW}AT$B30R}45i@J=)|Ny9sFTT_zY4r zB6fpD&!#O(YYmK&&zU%;ovTO?to0+w`*@WCyzcw?zG}SWtr(%R4L7X|f8sLGkTkD) z&h%+S2}*>X;Z;MLoZ=wRHuFzrM7)$4Dbr$C1T8my(^>eVu%`NNCE74sz=BZh@*1Wc zOdfTUl%5|0MYVG_x)0~x$LHmR;;@(a4e>h%4v8-J`6ic8h=PtFA3RAgBYxSmlJ0+| z->3wthX_TicrbrLZ_36PR5@K4uGUCB-RAUz84*oSEE5ak3hv(v#M`Q5LVsbsa}|Z+ zb^Lp`xwA|HQWJ0YA8^$@2VEex&~dm&89qZT?(*24G;#?XiPdoO|QVV9^>?(15?|GF5PH5MKGP6LLawrKUaIv`)C+CXyIa0ywGgKFchpuB2N*P)XDn z007_}M%D2&e*%q?rCkM)}YR zhsi#=jCw@%lmg%mfyGiGT8RbKKjKH1a}(2Z?QL1NLlTdb6OWi4@#u-QXm1uGFUz8 zND$nSks;p`VJF{!Mx3YsqhN_+x(QJ4}d26Sm?|i z6if%B-iW)@{1qQD!BH_%?1Tm$1Op-b6;+yz48tdx_V%!6=nS!s^OSaOXvVr`SLGrM zkyUZ}s^Z=e}7A8J71VZZKqv*!3M2*T#(oR5ZyXT)(QyO zg`4E)jlrZpJ3+4SQR=KFXuV{O+GK5-zrKet!7PT0zd$;Q@(<7tFyKlbks=^*cCBN( zRO?-kRU#=<`q?sq4H2_@J=X%`3?K(B-N_#J{|gDAgW3mDS6O2aZ;w9D1#U>yC4`;_ z#)iF!l7MQX!IX{{Jr!tiMH4PXgYTFv8=#N-^r-H^=kn2hW9bQmPL#s++H=sVD|E$A z^*|4waD#V?8Z7SBWj^?hsxYt71@nXBaxHx4Z%AO2pNn+fus3VJnjfs)^Km0l@X4e# zrJKoqB5>Abwb^@Rt^O!S`=P_0QaZKqHnUHZmU^~h%4Lw8--1c+EZi4WB+5vXo<(MH z5RIW8s>qtjDt=&d)Sy6kkWSfp1W!xHGmLD0Vs+VXqEl%Ix|A8>ksr}SC2OV1fW0y@ z>`Idss=fwuAH-=+;{mB^5lS5cMX7Hg^~IlhpFy#BFD!F{3zl}`GWpD+FVH@zn8W32 z3Z{D!y3I@0_i%9vOg_KPE#%X009i8+cw5NnN92d%z@(o4%Z(ZA9l2=t8stm8SBwc)zHY1+%)nc0N|VxE2O2wfcEa1 zom$qrcH@9Z2QRL)WtW;5bX$%%yu+4waqU8U;qQK;H3opXC9Ga;GCRbfb2j`C_&|#dIIhC_ zjjDIijN#szHl6LzKc=^B)Y%ORDY8EFF*mcl-;Y&Q3Y)rrXcB(jn=M-E)$F%jmp)zE z)eaJdI(}nfwZ;aVMJeZ2kX>`EuenvDa@Pj#94t7cxl0y;JA6D$t>h~XGt$ZW0Z zgaT6PD@OTEycuYm(#^o##$j&l(F$KjPjLh(w;q(%3O0l&MZn1Z*bYYq1=X<4t+#(_ zq^MhIlXwM+H*QZ2&}_kN>>6S;_Sx$pB%`Mm3SV%FkxhANTbm`q)Jy{L6(j2M1@@au1dwj7g*tLwU{4Gti|GbJTBs)QY^B!XE}Z*$A9bBE8Wb*6 zvYGJ>D(_}JQ?IY+G72eB#Vm^nJjfryjL+<_4NRR`XwQfkrVq!Y2s`W5Tv1fVXc=RM zYD7SpSI~t8Fbxh5tf%%NEoBt}(8HN~AHgwKI<;HT{*?Zg5`@?3PSySzOF|B?$00h@ zm)w?e4Z`;au!&F0egj!9p5t1Vt!0#0od!j8^O(Fnv{;y$BX!zDKL71F(m@8c%g7&Tp6I5K{JH?a z8j?eSw_o=RzD8*)uc)T_6d~PEq4ugtmNyzul}iLg&ZU{(&tMiP39=rtJq$wk#0L_c zNOU_gra)-f1koOguhELj6apb^_MLZRqfF38gbUxyVAAb8C%bxJU$Jg_RH1RvV{A~c zLT9TwSAO60_IaTx5LTQr;SjWh*KN8;_15Lj_wcD#ih!5n&p++Jf5=D~2NDrBq)#H= zvO3caFaLog=)53vQ;^Xp;8f6n`G=Q!g)597tQWdOmG9IqwX7xe-2y>7QEW>u5C$2S zkDeBl`XDs&f2ZH0r!dfQUS=Oee1GgrE#H+YExQzNS$s*KRQZsu(;WYkQU%vk>rjOR z9PC%mAI#3C_a5XA{A}QU_hE=s2xX3ez5O?h+A?f1>%vSBpmB{P-Tx577d~wB0M|s> z9=4~y`EcRp#|^_yCXwyGoXxe*&U@w#_IG4uT2uL%TfA0Yp~H)(64cYjryZRkL#Tgg z62cXht%|toW4%a7X&qOlWl8n3*M4QFcxM*Q{Z@)JSDldgoxkJX{7N(}O+6?8%QhaU zQ8`h!EjrsonCyQcSv<8;`e4h~-Jnrjzly*)e|>is^u&bIc2i@4DAami8CV^Kuu1RV zx3fUNUkZ}Lz~#6({@L!hoy9Q+b}*yDX4M1(Tg5Of$`$9V1_ZuQr8HwycNIJb`WiSy zNO{y>r37y{{Vyb`!S2#t5>XAt$XA1}X>IhV4_U(6FEU>1Y-Eg`D{;D7TqG7oo}bp} zf3$LTV90_IlpT6ovH3RgVsh=|&~*L$a7*A#XQoLa7T*sKycz;v+=)19H&NqcK)YlC zXj)lJ)EZCd@|1D?tJpe)l25(vX%H|a1wG#?ILAYPBgW3zQX)Ea!60Fj|3RUoDrOE})a}=G?`gyQ#31(?SnD@bIgj zvwr&s_oivyvmmbDx>=oXF@JJ6X%XCvX9k9^m6#+!IM7@XB=Ruf%Za|4W@vf|X*tTO z+TG&NM>by97f@~C05l?PIDZNkgLK07WJXT#4~R4Hd%5b;0AnlG*kdpMN|&gPLT+EC3nvb zDi*FXqRPZhK520(a%tBRw!B>9#EY4;y{Ll_(!{}ycJnE>(V(w6=uFQzk(R;72GUiI z(YV!4i7KTl_)P7SV1!Ix}D>i_yv?>5&W7lY5w0u4Z_cP>W^K_FocPr~&nx?%N z^q25#YS0|rrbde_p2Rg6byp)nB1qawc07BYPg1N#<{Dx`-b&h_oMzY$(|0?^<{H&h zgF`v=M&mdSX&4mj2PSn{#>s(nS(3mDTiP2F-xt}_I)j9~UJ9W;E60c#=Uv;{<~vbn z5KVR>)8J>Hoyf^F)LfnXx#hhAlA4PmB_A9c(mhOBXa9}p-ze-<=~}0p@Qw5aGHP`% ziQZ93pA;fHS}c-DGJP>qsA}qO;@Gte6v12U@V1wiNtQ+BKLJ;8V;=md4CrUfV3tH~ zPY=ov@HV6gpH)=abHA?e{LpPsnneZzTcZrQTqzH)dRe_6r0mt;cNNb8excX@jod&I z{6Uf`WA8mxtZU5pqjJ@kEF!AXx(v29OMBNMreeQ4vc@A-qR`8}rv9HSij2Z%a>1Dv zPDZ>@74O};a!V4{@{?GRBO(Ek2;jzHusRR&IbgVSvM`$SuO#tSz$oH@rEZa zwbmhqpcCB#p4SY3ue5f$O;nhpobZczz^^P(%kGI^LrEF~JD#M1y%TIJhmHX691xHW z5S@bl{<6z<&4x9aj-wZz7<-{uLnu+EiQQl{>866;wEK*d05!3I-dUiuuWbu@>Fj)E z!)7&xY12PxR0GAJT^LqUZY6}C57kmH9paj~K_5z~0=n_$x}jJQ)U@yir18olqWY<= zfeczj3?rctAZjres4(#rlc|aj*Q84I+ zD3>?WraxABdsrxE^1X)5!^IbSfv>NBMS(tkbe4(#Pq!bRRdy%ei^G=QD|8qQOSlEk zc!b)XfI5v-%4i_w8dmMLHbA0!gHBCHSrBm1*t&I?;@J&&0>oNOR+3_?gj?jIB`^tc zPSSQkNk~s%OaG}P<)hYoeG4v$kK=E6a}v4{&k~={QlQJC{m(N*5xJ3I0un?tOAnHbL4)7doc_Bd;z$y;5rO+zI|Jk>&8)@)zgo+5hTheJYg3oF(=&rabP zs#(GC%79ug-(HfP>+Rfo*eGnqS{%1%Lga!?D<-s&;8JDzCl!&c;T~m-Y7d~@&FJ9S zii>46=7W>2!pkmpk&!79r5Q{1v2 z#m7CB|LJbC4(}U=ur=1HcIH4+;w3cbcG1t@s53%zurZJEj^_R&9XofD=$?~Fn?e7v z+6xO;=HGz|H1;m!bQ(w4b(EbT(Wh;X9Hdaf{?uDnCv2MVao9fnBr$atR-Kgt zF0bR-W=-bk)wtJ;ta(U{U|a~dW!BAV^Eg+_sL!=!z?akWwTgnao$z;|5z0iJQH~wRZF#cU!RKx?e_uT5b_iZku+!eEHjQGfA1{V7Ki$*&3mL+J1F4}x zJ}CDAtv|isKUtVW*AdqwVEk&Xl8{tO@EQ6hpXu<>$z^EoAObl>{-jw&m<)FM0LG2J z9|j{LO49;f`{^-=jpXnrhiQ&Yt{=*o3*?NFysQSvKMig{+P`ONQ^Nn&tq~}?Y}OYL<|Wk6WAP!9JyqId?76n9&Dhc?$)2gk^_b*-R;M0$ht&3Q&Ne;jvIcIzqmd% zgr_qEQb4VZU>EH8w)f@&Jmx^~Lfx_%imM%7tTnqndPoyX4U$qw;29v?h zhRiI0tmQwgU5;AlB9lM|<61e90?$Bgo|wv@n+b6sk5Wb1T3AHoDDTxHce{H!c#kwE z?=R`FqD=bRukDv@QTQ)rJ)v;%brzK*EVekNdHv;+6RK3CG2oQ35o`0R6S~Kd zHiY5@K@BxupTJB8K4op>J?I?*>E$4Hh~8>*122SBc(ww~!-aRVq~;Oj`OAt8in2}} zll8S0YkIsANf^%UX|!yD->>K`4}WI=L2Q?S z8M`Sbm9zS8ysxfMdg#gISwWFH0fGAm4z}Xa;ggJ z%U>&9=5Tsd_$GhiTyWtB2Eg84T{;b5W5tWYB`MAGEw~27M*jG+qQSlgXQ)__vOE%` zI`4+C%=mjIY7XeNN6z;emYT*lt{5em6WdSm?PBd*Mcrwc>1$diqhIy>pigPbDUT^VL*-|KtTRw&YSq_MP6!?S2q2&$iIcJq_w_hXF?Vg zi^xjdZ@syyKLWqXrb_@y-x)JVKiEjB+koUNZaGOfwW52Ge0@Mx#!kSo-+f@gis0)| z#KZm=M*8TGKzwI8Mw{XBlUy<{Z&%6oKaz8gUvQj>T;VQVX)4 z!#>PEedh7RUK9#hFzSZa^Do~28}^pA#^&Mi*+}h6l1k-`>8d>5#d;AT;C-ZoxwcYu zs3#J0B3{wS8?A$Xo_CjIu8_*WD>omnJzO2#Rg9{ZTdYfD#wtcNUsY5Hre!tQ^o7AU z5~j&-&@#Qn;gL>!G>G<5E%HU>bwuo=NC4BhAo)4o0u1)I{NkA5W^KabYj!_(P_1Ba zG;A%(CyG74!skELf17V`iQ*@S$@~OmuVprfbO#5#?rsTLlOxHU*@POofp4#74a^K- z6`$&JnVEv>Eva$+kSU^s6*1A$MGI5nVS^D4mfNMKS+i};4}FAH|1DNwvbSUo^f(U` zJSy63Ji61HTPx7i5{KhesTW3m3j#p^ZPem1Zq&1^ zaN02@(LsLQnFHG`lOcnIB!*6U-n`kAhuabtxTb+ltDD=C1saVDmn<_VV(dBcE^u@H z)a0C}T?ss+HB#9oQ3H19qBz>+(5P~Y^6I2SiDu-LR&h)k7zW%bYLe&kDq)Jv3|5nJliIYxSIt zP;p9l(e+DWb-;K-2{cK`lewE`v%8g zh~#m&)=4t(>P&jZ0Hl94ETMa@lI-)8TPYo|yAAiuKcHkSM|XUf^0L{sbu}|Sqz?Vx zlDvgn5<~D#UlgLT6ut{u`1-?S-KqaR;zk%9JCN?!-Di0G!5#oZNlZ2YBSPS#w!UJX z5*B2$wX2Dt18e(_)rd*fq;ho7;Bx~$jXcw6)w`I+m@AZcf>Lz~Zs=wthbxkD16d&W z8D{*6^@5LCfD<+!@*7Z#xvDh+K(mB;YIN-&Zt}3F+PAHblG0Q15B0 ziym(|#zuQUb=b7(WuI3KS8p?E6%GT<5Z>mW=?Nsc&x9gY*uQD$N4+eO!v0!JvmYtE z>BQ0P{4?w&LNBLZCJRc$r{AN!%^m!>J_oSVJZ^I08?HNF@rSmxN|4WiZKjgZ=u@yV zyT&B8h?ow;XU=KKsJaKzp!dm@(%j7t#(&924TXcJU^jxavyuG}-#w|9Nn=3=sGKTZ zu|iF>-%!UK8Ro=a3L$BD_XO`=2G#eiRtBKK2pXotS2F8hOovTS9%YWv? z0oyF&Y$?w1cw3~V)%~=X1MQ(y!oX3$z!iZGfposEhw1DyaK4qsy%GxS(oLnL;&U$_05fBmSR#I&b zJ;u4o+IQhx)3v8yEuDjPc*?B*2H7gB>{FdzKa4S?fXX_{zQ9_%TKpF^_>ZJsq%UfW z&J9YQjYq2tm%7OfN<6Euq4DBkSLAU82Z-%`=c>>RI;Qeid6dq|o|*cDY`L8`qaIWr zg4)Sq)D+9T3mS6dBKer-0OSd3+^N&`#E}_`k)Al&B5#fY>csk@U6_Oi^A_MH*{wF0 zcI-r@+I{v&X|BDBb=I*$de7JXa;Xctv-wuzON2N~hI?g%4?KJyw5nVVAq#n`_!L;F zeJL~u#@^1FQOWc@p#eW=KGUe>iY&XEfJ{G$UlK9l>T_h@IPrNfY)mrjW9*t-WH;+V zVE|bi3?iNGpy=Upw3p7`zWtfkwUc7l8u`0BXe0c+?VAq*@n`P^9!~J8JnDdGd|Dxe zT5g6u>A&=BHJ052^?Wu6)btA|B5tEV@*ET?v2NvG6tRpceaUkeia37b z;3I_GFX`DQfEYb}F^iWCrBw-2`l$7X5Y`j+_2Y_dL=Ic`{y?jMnNmzl+y`MoV@ zX?(dX>HU+2&)$2TxAJ79`H<1@mS zC}4L@&MnG8k@(I@eVt6FSUA^cEVzl;*!;zywv8R`3jBH)qVm!r7}iSeAo#Uwgo%Ec zx?cH3HbJ}Fy%-La-`6a3JzU?4B|;~86@k#(iArgj2ejKw*gQpBbb*Ce`h{!_5Vpvx zTn!ZPerF`y`exQ1BRNq{@K#3Spp#KVTd%V}oOltCZzvLHRxl*-KuckRE$E*?+KX3P z`LUOJy54x}<9q=NG=p9>DJj*8zgqebP(i#;jChqg5H7R;d!CjtDB>_xlaSi#5Y9MY z*-$&P-zCn%SV&*9G~_RMYTs-8Z0`z$)potK(3a!(3acO=1 zfVIPPT*1rA2R&nT-D)IKu~f$gsAM5DD&=SGqf$^=00f@e=1Hlo3>^UPWTrGR|8r|v z({H1)NJgS4R|0!9ZOfa#**XWB@%O4C%Z$j+FNfE`6aoLQeg43?Sk|Re=m##ge3YmK zDjg=lp%@wtoL5M8gp#}O9QD8bdRfrOr6j$}4Q*s0?G}`*QlSXKQVq3`*0@!Vej~Yt zx<46f^pvEVOz*Sq^+yg+^V)1od0c|l|*5Gt;GAjXPmc1x=4KPsT> zH7P^O&u6yfX3CsywxCLjywo)DrDs0C6zoZ!tB4^e_g7j*26nQ;H0~2JS%RkEk{4JS zt~ycg;$wFl#!2UOKYiscNWKqfKuoa?&^L@v6c$Y$n$JV-W`B_3GS%&c0h3z1vV6DU zly9kI;I#t(gTxk6vBtS#bN4v;@+n}P*HnLLWGz;zTwRdAQ`%#T3g1Ym)a9vepJ6GU zQD_v#UY3X~%)-y>cUJB&QRFWW;%}Q#*>oFTDM7ODI~#PR(rK~!g|vkg$GUg%2X5xF>)2 zw5S9cItxi1Gum>dLXiCjW9+BP{Vmbc#(1WdjaS6}e(*=N9#co5tO6IAAs#0=AR z_CSf`@bkVf(h*DysdDDe9&=b9xSzS|^5Ea_D6jYkT-eUO905oOBWg0O!+DA_4Vs3K z!t`KJeqvkU^8T-7oeyhJUm1-@kmFZ(h1SYzB&m_Zj)a{0sBYffMXyali}I1yBLU(u zDE+DuSg>*lG$Op1sIHGL>P`Aq0Jl+#A~!!lne6tT&A~h`SWo&^^1Mmo&*D@^OfNcu zhw&Zi9-z+Z?>{_Bw-h_pi0qxiVGZeb0ATeWy|!WarRN>J47V&N|7%#Ty`(?XG?ZRQ zTAC!07b(&qvnaolUE(}Eg(jwfDSqtX#V{hkQV;S2cNJX$)>@Tra=|55#+i;=;T3{H z`h_X#!`iwy6;$AO*Typ`)NVcSpW#?TQ@d9JS_!;zSxSR@+cs?Y!bbobTya2DY` zHyZ_y@b{MSn&Em58L49a8%yZsYxpUGx&5h7fuz5%XP~Bb{OgeeMZ-B+Ql(tgow4wD z*gqdCb>5PFc(&v`el;|1Hg|@tpUO4ibjnwCP17$eaM)@<0C$SOcTgE^Mj+;B0!|q! z<2B*}+A)%gZ;%XKpZcu%V7rHe=;$(c6I%=W%h1)u6zx&gNy^ZFGG{}YPoyZLR!6Y& zi@tS$Y<`5iX&ZiT4Omy7da@OcoK0dQ=^mFPV)wjC9-+60>M3^Zls8$>FD|4=#W1p5 zG1v@#_GnnK>W2z3jho4V+DdoXRhm2>z>_0#0~h-_ZS@R2w}edAGZlL}2CKXjL7d3$ z2D7}+K=z;6rrm>t8nDtkc?Uyw^??yi=$IBi+_w1oh>-pKDB}{F<;%kW`ia-Lun@Ti zEA5NRf0;pn_R^Y2+P>&LS@_GlpZ{BjGPe!&P6;0?{NyUp%DpM|dw5=*aMB0E|6r^? z>^zq%#;^VjkRFqF%!k?$`}2IU-Q9jc09l+d4|$h4)e=cV33A{Sq8+hBWvHI>-p|4>c~siDzSavpTkP6cTB53HAXtVYJa{` zL!5?bUib5f(gWuCXAU^_b#`ZlJWX7@qZ4|EXFoIZR;15n%H+VF|HAtu z`bhQkIio=Fiu%F+SVja<|H&H)QHhR%^?T7Y(Vw=gc}WJ`bh|*115l8Ptlqv zt}l#&@XLRS^MVYn3Ur>|++boL5p{UweB)vQSEHsA6HSn`9X_q&r8cWD!vfE&%dahyP%V-~-P#@Vt~Jum1)e^HFZ z8GYU;Ow}@80Y5bfDIry)sy&6Gi>xYLki1`CC9TFqXm15g{@Qx2^#(lFIH1y=^c_}f zDzIPC&{2%PJ}pu$?H7uRw^(<2ue(0 zq?O?##DN`{*sv5Y|8=#pr)trJc)jiCe{B)=PSM}TohH%r<VOA&d8uOb#-U#%tRM zgIL$@FE%_x_J?PrkF~apqy=Os-v)V+=^{Z6!Dxrt&II)xq!%eZGRMN(KzJ@=;TY(5 z30PQx;p^KdIV9KaajtN?N~qlWa1b+{ZC$za&ohkIq-B2QhlcR%s0BEOldACq!>m@fnrSQ^>(Nw&qFx)mcq<(KLFK4VEvh_r5_CfBreNPk;}HGd4_mURDuIU3WI{yBG;} zPV!NtndS?8k#-`ta8W_pPM)1tYY^D?rWDz1r&*HD3cuA3bvw`fd^4_5HA9;FE~@`2 zerm$8801~}JTO^lCEzsQ?%=u8u~~RzK-;)#K+~u?C~Uh0Mm#7kQjDzfPe@7aUA~`| zSyS@@%|HmHpp?Dh!l`0_5|l>YS`CkjN`A>aopzm}hU_LOWYy9DT}JMgV@@;q`Yk+R z8S{*(Vn+oAAorYIcNBpIik;zodH}J%9Q;mYyo#ey3Ur)YD z=FHeNKJLBZg@f}&y^&*6Uz3c#iMob}`=V`AAdMHC7uDmY_!oNu!P}p?wlRdtEA@s3 zV-G`&yoQ3Z8X)N{)g=Aq#+&~LwWkt}NOTse6_OKq&H+ZHrEEep--pp5A2QT46eO1( zagB`ez&TE9Bm%V!(uC?~iksc0xWrTM!EX}&b$v95AF{!IaP8?a?mC?&PeFrB(Lw(b zVgae(p3_dwYF9o7Q9Fn&>O;4-9dxM8Y`D!kRa%w@W>dw5V$Pp91ZkaVRQWLi$XrZI z$%ssr{baskK9_$aX2QgEO;hRlZ#BEgP7Pw?e{Lu#zA;ui4cVC)koMznFIvXX@wTWK|K zUa1~$S1%!ct1OR|B59mc?W#S=g;B4Y(VPLJ|VQ)gjMWi4ttHMY`Lu6t5+iYuKt?X>) zXkhl+dGf19jqoFr|XoDFP^Oe95qD~O1S398Dd{MPt?dQ_EU z{=Y681E>E}<$tG`{?`1xl;0_Cjt2JjCdMM7(!awwh0Ga+?A1if6cm`ulq^_86{+P7 z&3Ji##~ItX7+RaC|2Nvg){KbD)WF)w#0TIL7yv+zQwHGPTt!?C9|A96=Y4b0ZYRr?ROM}LnI5k7$!R>=9_WDvpG4m|s?jAmt{HT5OqI6x&|87vNWxZqJ~ ztGK;IfiFJ52;qg#6^gfZqnhuKa3A2abS=G)#WVo{5pS_OVdQ%wHIQT!o-jX$d7o{eRSL$sq^K00kOCF5RA#C{>$+HnFn&e6#?bhaD6L*T*#)HE8RfQZ|*P)5T(JgenLt7*n}m zhCB>*QRoYfeg+)igu*ipbENM`Q&54-D>jbzi-;H3>l_k98yDX6?*ck#;bu$4=oG@+ z>Vn5)Cj~y>rK$4p!K7Fmj!&dn5^dn|@^AQ^%#{?_rp&i&iDd8-Lm|1iK_+{PT_u)kwy68HcLIY)ho-AQ4`f z_Dj&R78nDc`zs)bdtxEz?X=~K+}D`R(UEAO70OxU*IxqdK0Qf;N#L1hw19bg+YNW# zl=TEhbJ*r~EVBjSLNmI1YVT75gH6b5e=3ObJ@(ZDoXIJVfm0SSkYG8(e@2rmGsgNh znK=Eo$F2bp#tNj*kvi7G%>DVvTT_X4SAg$)ch8Q0&>w7cjw^_}PEI|XdCUEgQM6#MaetH;=NHE$%FG{1> zp2XSA`&<5X-sf9&9L`Q*cAnqqH^Ms04VPd1SIwq((-6#Zt990^oDft8=PuGPJ-z!O zn4~RT=i|t--bFk^1ACa?()ZHRrEI4$k8ZG}lh6kY9%TE)o)7%0WvSNBR%7ALQ`h(q ZdA|>>M=&P?*Z>E71Ofn{@cZw_{{Y>qYk~j( literal 0 HcmV?d00001 diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-v1-uncompressed.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/arrow-v2-aes-gcm-v1-uncompressed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..696840fdada5fd2962a901685b7acb2cc30c7a75 GIT binary patch literal 21602 zcmWjKgLWWT5CqWJwr$(CZQHhO+nCt4ZQB#ucCtI)Q0G=ZDhMixng9R*oNJiXsgs5? z!~7k(aK@R+Bt9l!&6*pxa(d&|Cb$Z+i)&xe1gMR$ueDiqqD3IUy&+*XR<9P(2<5M0 zz%@SD&2uirCOhuea@+?0dtCnrys2s!;5yB$ALR>nn4auw8*uW?!vhQeAhIty2gsH8 z485QyPz7*rd`6lgA33^?Zuo{0^D~R8sdvNW!eXe$5$ja@i63!%cT^0cN`@3%s$59 zQH>^Sx_nG;1<4d=vwyr5@EwSyWcv`ALM%l0yTf&|@}~W-Pqxz>L->>U{7dYVerYl= zr#$?{#S3n$Wc}}+E>m6fS+l>TD}8s}rK+$(a_5FZzYJtPXs9Uc0}(4$z|qLdfca10 z#Fm@AmQ3j~^)=M4$&TXoSuuycBP1rp(+E|@x#ry$f&2+Gnu)G=pc*8ECW?e%FWQh! zK&)N6$2GfJxBq2UCEmq_*9;-9Q>c;%=gT;#51!~zQ>|!fK&UkW1kQ)5|KVa0CmvVh zS|%StJ{MJ-W;)j;6{_G))+Y=oJlL%u0ASd@7eyBRwAxI#G@fq%3C_4-m>1& z)>Y#Lw3?pQ=*+zk@23oB_>5nv5%W##D1)+H@Jk)IO{3@voTq;eda}w@ZaPrk&m)Q| z`I(Xft+F>M6#+=x%8FE6x6&dhm4|XE35)UVh$Q`(5(9#RC%mhJb*!e$XJzH%(2gBo zSdoEnP()}vqUn_NY+`*-yE*NT%wTTO(6ztLvCL34|4Sf2N*RFSX(bptg_SZG+CUfy zEq`?=AY~7Jlo#}^F@r1lP0YCP72G|w+1mU_iPb$DXn=S?dtS3*kbvO;JYR*GIoiXD zZkqFn=F+G=`i)@RAa+7~+MT|DE4(On%deKL0Qe`VWHsPQqGjBN@(-cpLc$L^I9i%nZEFLa?pt&`k#B_7}Cmr4{B} zS)d@sZ-%&xr<`4Ay2L$Z20_r!su5#*`a6~;BJbme_JB!s~v?7pJU3B_}E6l+&X^%xo|t~g}Hg*ITlB! zs90dfI^w?IkP{OFJ@QRfnRLRvrj!7Xo>WHe$g4>y1{#kyV8sqP&`GCRd^TIzbJ$p7?Wh@NCdH zY@QjPtq1!U{8W-j%ilQ!RjT$mP~<{!TFnYTusBeE1S6|99_ww@!l+V&Mi=j400PLc zGq3P@1({~Pa{UHdlcxJ|&=(?rImQ9gZABEp`oq4#twH<4vzCDcgwryiqPeix9PoK4 z>R&Sa&^#V_=@JuP(KmL{>S#FLkdB556|9P};)1%_thCBGCR31n^~_(8^P{*B8Osq4 zRSN)vfjK<1gF5H>Q8%`6HuJ$cpQ8=eE3^R@%d&0*y1s#JDerr?zetkyNj!VOBzj{wa9j^ z2RBOLfDSCsC}gLh_e{g$Xq_u{!Vp&oDNNeah8l(|mOsGY;5q;n&qXsf@-1(Gy`ktZ zC6HIh1k(HNa1n=nu|Fyg#t`(0@w-jCjC;M53bmdDiu-IwfgM*Nw!mTuQyM^3 z&>i@3%T)16&=4(xK#ojrj4J6977^9iiaW)G)<(Qw(#tsOi8zgj|IB8@F5iv1fVPuw zpB9J)7DIiO*i?ipfl#fMv0u)5R2vahI?Q@L9n_TO(;5ew=|lA#If>RmW+vvA%e*(+ zC=LL9dpsgu=usAz)xf&lWr|MfW>-gK-A(lD`5JSB{1HXRe?6Sy>+ZKk!_|sNWur^p zkOm|e1%L#<$64$r>fC4jo5}9{vgbL7WiOR`tsUDJ5IN2-LwZJH?U7`OtYYm9bLlmm zbotOG*|AzTO=tv42rqQU0?1mho#_6?EJoYLdd7AI+r0=o)dfWFjqUIr+QTiya_hwv z2!T8Ziyc0VWq08;7ciQ0A!0yZkut@3#Dmu$+HHB=!Str9y?Zl`B`vyUj<3QS$m2|c zZGGZxRNJ=U8Ie(eTeWN6gD%VHESlzpAqHq)-qR=kY*CwqO&~_~4Gk!ZQ0a(W(de>u zuxr200!o+F8ewb1e~AW zYxm2ls6LQ((Na+=V~Vtk-VFmY(8hJqZgwVOVOmS+U`ET-+5}Xr@;ssZKn6t*Oi-8DhF+J4Tu z(qdOPp3xG*#6M>+65MZCxn>2U_BLRKMIyvTDK&|b8Sx zG4eKbMH))kS?QNYC(obMdpY(P^qyVB+wQUmhwq66Tc1pbXwb=j?*zyF*z(iPVmXuStQZ}6>q-tacXn)po*2^Nb=C74b8 z@rJh67*+Gww?a=qfv#nQtD$0&dKCKp#@ci-p_>>kUJ$tAh6{!oCeQ;w(K_Oc=3H~? zEY8jRh|B(wMWwr9`TRSt$~e&hBQ%PDTJP!>K;q?lgYT@ooR_gJb~qh;z^2a+jx;c$ zh-_KjPGD^x%%NmYc6#P*o|g;DZbr-`J}W3}D-HLxbAsJxOSm%+n+xYH3=&p%m{hLN zh`G${wYlIx;Vv#2Wiw7(m8l15SfE+@=_+LX;%h`3FFWCKS0Mgo*Y|=tbLj#T0@_yo%JY_&!!5M@ zv4y0|l|zUxq;2lcJNt@sBPld+v+f|*i6Hus9^cIAB&bJievWblpfc^!|Cq$>n?qS* za8!%#rp?ZI9y{g%@7H`3GKWK19JiyclHoJBZjt*_8}`TJSA}0|ST52aw2m0qokrzH zj^N0BXJkFyg|T0JsWf3j&e{}_!-w_~rC=X=UyNdJA7Os59u5+1hjPXP6ERQ~Ryq^GJ zwfI8(lhr|^+JCl)PxAcTjr4HDkEy%yat#*#I+VC{v3v&8iC`Rcg`a0YHQ*u6I3Oge z+H`J^=v6l90G0T@Fw(^wNLzy#E9MuM_Dh$0TCRu$)A-~ zXr2E<5EYJ~68?D~>&$u%o;tE&WH^#mo9Mzw-UM850h!``Tcezh3H;TK8tyNk-VHV) zB(1)A;Dm2{gF1#?9Jx(mkR6S^%F>dnO*(Xw0Bi5uW3G|DygAf)!1V4q5 z);4zv#2gw@cxX#y(p9^IdF_5(ha9ag)%Y^!9#A`otw3DcVThD*IlUfZ%1!d-lybUL zh}h(3Cjo=oDgDXZp05;XL&8--(>cpuYU+)+#(!{|=@@Ewg3y}Cv!4Ng^DJv{0tN0#{lt91YE;fX(UDI4qgnm~a=Fy5EY z{g+Q}&X_)+N$MsrRewZ{(-pA~SFYw4222$360zN7lfJQvR`b3(VNxX%c?EO2QR*y| z@I7eGsB!?brQ7@fR=Lh`sDx?QRpcZTYxS40wAY8;IjAjpyorzh#j9&@21#8yI2)vW z!`MvfbT!q0g0XW9z#VvzFKwe|S4E!0x)Lv+*M>yRM>$o5LyeStfYunhUfgKF_8DG} zRoQ8~7w#DOtgB3VX~vFm1n_nrzDP9&P=sr&!G`qtZY>Z87W|R`)(7T25HQyOYN<{q<2mKM`vQGN> zMXfk#$jz7gs$*yetn$8NaPci5)ntmXH@ssX4e_NN1gzZL@D-JrGeWOE%UA>=>a$q! zhE;x^`kR5x0qwZBvw86<7PjRM!&vM~CYPN#K9&ESN(W%W|6M}_5%a}0OYF;h83CasDm`X{v`*-m^aE(o~Mz4ERKd?lmw77(EfT4 zpNi973$}r*bYOUgw@VZHR7j{ol{|ygy{Sa_qIp`M+L!Q18Smx&)?^W})zQ9`p6*_+ zZ{1gW>WL@~Sa^Pr&|aKvAjjhUFt-#Jb20LyXvf2_5_rQh5=tK1<9GR~R+~!&Kuc3! zyf0OhE8JSZ>%cfK?X8Y4^Ta5gZg3FA_Z<{)d!5q%ooA4q?Z9uMS+hU>oU_7lr&N^Y z>rA{Z_DbN4NJu!+=KTdVyN1xWQmizbX_Y*e?pSA_5eJtg-ur3qiV>H5eaG=aze*5B zRR3Ln*rh=|nip$8G`VB`j{dWGBB{XX)WwJIscH2pmV1$v`qHrSK|Ez;ue~d??O}gq zoqB%54AG#C4xPu}0mVO2!KP?Z;$Dn;CY*H&m3dHw>ZYqjM>N)(CFl5_@*QfCICSXy ze4FGN#A;Qw`*C}$x6ZYaRqSmSAvQ?7M&7%KJUHWXX%;jJlFi%dNc;@gn02{1d zq*4Ej(fQrfn}Rfy0`oBA)S&;94e?!h%@2cQ%`#y8uBnWrf*Pukhl;Pgvd<4n96eEG z8q#!5B?y{A-E;^r0?az<{fM?g?>X@k-5K`=gLKWEAwhXsT`9-b?m>n}OS?vf5VbkiL9!4kuPBrKa` zedvbqcXaoCN~#DksCZy=f>wL|!M)wiTmZ&zb$&o~ai40c=mJ(l^$+%bQozJtIg(LJ z3S(P$#>Ok4b@IR{E8vdP^&J-lAO&ve*_5YUe&y@nE#1b=z2M(!8{~N&>KrZuRF~d= z|G)^sLw^w5YySFJ)kjw8g)4MR`nZa}YNY z=TgUq4RtCSMOy!e+;2|HDAz?P;Cr$nT*a8g+p6I*N?IT+l8ySBcrep}#{p&o4nk|2 z!ii{X+@~K~@X0}%BIT`mpXFwNeyiY${7Y8~qA~0sQ3@?>cpwjc0zis{4Bdb!7%`6xBay7L zt{@sgVYR<&v`3OD>_?Y9tqR}SE5`6d_y{OZXF?J%yuBO`r2^!7vnt{^v=T-YiB2_x z{euV&kkA1)35^=m9Wl4sMv8qJx1vBou?nfQ@luSJI$>2AB?@*v<{Fmuu6DzbnNL}b zkSsz!dwg!RX!lz!+hUcPI@ZIImI{41!!+xe6pPR9uSQwJ;MkWhy|%gz3clfHKzw8%C+@5B#+DD#oHVtM}5L&+ZxGN$q zr@u>g&Xy}8wN)!F7t#TmA>@FY3wXkLW|Cf_O?NK;wemB8(rxi`0?wE)oDVKqzs8Dd zMX_SjH~_>GUM@EiC#z<1S3x(8X7-05#S5C?S$Rweytvx5{wi2@`!r<_Qb&Y}W zPgXYq4Q(&u$jb36yqX)h7?<~i@|1+!99!AlTvevodGl55tF1+mzZ;UdJnHmQlflYH z99m{5x&gJXK`AZcg=0t7ZbQ>6iXtLc-*)!UU0o&+QS`iMQ~}T~<3ZoI#M|`Q@7D(v zGx;YRMZNQB3mJ#tILj#seMiSKe`fangT&3gl=bX;oq~la6iQv=JA5vL06nWdpQ9uK^rQ?XlB zXz|eHZK4dNJWTLb=d8a~WL>t=ssPgB8b=IL1t`B!a!jK_5!4h9Xx7_14#)W_z{bV39b`kjhhr8`$h1j)c5S>jleriL<}$WnSn!o>>Nc*#p)OvdH*g<2?Eb;=phO^Nib^o@ z`K`Gh={rZ7f*Yzs(Awr#k@mOGn4|SPum>y5Hin>Jc32xCh+ohtZoopa*!3-oaJ}r7 zN^XwjQTHB%O8rUdDKDpzM6)2x&IT~|u8vJ4cXK52P*@*(+__(ZyS_r2(F*(=D9v!k z@KHDAVoJM^{Xbp2tlt!-?5tI4d|6X9=@0nKai7Tqh1feN%n{yDIN*Dmd)|z`iP`6)2ruZ$k6Ct^IDDr2n@!*-z-mS~fkZN*TRj zlfg#Hm*Swxn(re336Y&yqfS+M3hS){Fj@m{Yigy5X;M^c!y9~?CrGPn;}H_|^qklz zXxrbm2zHNFFtNH)vM#7b2+2cLRyZT7pnt%^7y{Y2dM2psJ9dAy| zUW=?#vQ1dXzWfb_rt&zhA?SC2W5||?@{sW|+IXHWlGZ!Ra@sj|w(lY9RS63=-XV$! zC5SqGl??VzWenF`-j3gtAgkR+dI6V*$tkx_IfDW>6>?ieBJ^rL4LW^LlQ41%bC?0^ zw{zO{ZY{bl=$mRx^OKNG^jHI)BFtb247gfA3Z6=?yeMtx)guLCH!%KcBx!|LB5{}N zJt%qF^!p?6nLzIO^28GQ0ol#(jVi2f4IJ>)6x&T7?`XWCGx;Vh9cGSlpZP=FTZe6C z->CL`m?_ry42o}87`^@(weUO4JtdLWjb@m*G_Q{@<^E>I^w7T`96y2)9X!m zh;iQeU`)FEgcke8)lKM8W=kEMpjQBFuGQ;d(bX!J0S98=37pbYv~IJmBs|NO0M|@$ z*2D;FPIoYG=y=BB&KHy22Q0DpH^LiO$eH3SW`dF=J6DunAlbFr0v3S_7~?B!Ffcio z?w%Z^YQLoCooaJxOxGeZ5RI&;&J}DW`yL?6e%#da4lN^BfTzg0&+d)880TZ5eu)5B zFbQaUq)t1U8Fu7G7sk5GJ|Q=)TGa3TlRaMk>rcx5d>eejEFcH1=AYIS$kl3tn2F7p z*U)RhyN7h?>w1mUp7o^#wwZtc{l_k|_2gZTValsp^d?N?s3`HxkZyhZoqC9r;nMO* zPw#Mww^+&(5QG9GG-jwHp8imn@w;BsX6AnFux0VDd`Dl%SWk<$ja@bJDpUjC9tO1y zN7^Ph#C?j@uPCOzDWF&UM$jx%v7wof9fQN-VnG|1fJ*!>Z0IomtI_p?`h|) z>~AcDB$=yo^QZZ|R`w1SlZQMM{%4NBkKtcQB0=c5fex_g&SqRyg6oUU{#To5?};u@ zY3Flm)8pkG?Ebb_LsC$s`ifWzVBRik_pO4WeJR0vfCo(zKg_^{0-`9002rQR(GKC_ zf9a5)wa7+I@d_|gO)q>9?7mFc7lfnYlMkGv)|voDQ$a9|--Gj&(E2^)P{Oa%)5%9$~OmK>G>JSMTy6f`4O{ z!Ort*@ugem`^&dA=_6Y(A`cC;{K?afX+Ou#M95R-(WKg%BFd;iCOR5fhfZ ziSyM{v-ZiXV_d4vl9_~$m=yt{ewd%<0eLj}4|w$uZOFHa z*nP4xnycqbF&1`jc&otpd<2lh$*cKqa%l4$*Nwv6xq!MvxV+th{`5fv-k6N~bQIfZ zgMgODFg^5aMHSFA9=3kb2vqmhK_5{*<5x?ilWyH9?h~J@r2}*5|MHY7F-}NIL?*1% z_cUL&T$F~K&o1R{((0yMDbBJq2FUwZK;(B~ZjFMo)GFkl8_Hn^G1HSVqtq_A*SYCL ziUXbmP8Y;xFEoV-1T5%Z;yRM5IebLuRi6U>HWk&_uHw z@+Y12oU?PdMvWq-K!Q4HrXuCaG63D`qIN*GqJ+o^1a|s79{0$S`^h zU|OFpJwZkRV6wZO$sk4&)k-=)n5Wc!7x7$xpNyk7OZLs91nu|Xf+Ixz1ppRqs5?=O z0(57H=JLm5rSO7e&50!;0Hi4to!n_Az=57MgPrddu6XB3aIk%~J>kfm=lOhzs-0!~ zbqYczJAE8NxXFtNsc5epK%5#lY6KD07g~cGg(AM8eATC!K>Afby=?0Bcu_?!$LNL3 z={L3U!mfOocesh?FkR6G;dcTcqOgE-P>02dT6$~=iCN4W@astK#poIO{(PsqcyYWFF@%p_=&dQ>A%ldyR{t7^D9LKNOBiN*qnX-q@`ly_h2}o28+>q&hr$yC zF-V-zDJ>1CP1eZMvA##(=}m4;a`K$r9UI6M-$^5G6ylxqk{8Ed1cZcz&+=0e#UY)}mb1t2 zhI42oMf3pP^!y3~WqkHE$623B4amlUs%~;Hwx%Cd0ZssT#uPBUO&pZ{<-U`T=uGG6 z(_gC-caR2NmW0>T`ygy`7QIRi7BG2K5MGph}H-fk*5D7u_zf-|ofJkq0X!yZl)|hjuIOA;! zAxEGhAQln{1xf{EZ`Z~fc_AiSqE$1i8n`k-fX5J?{%qfuj-WcFsZp+i-fti*atd>k zsWIC)D>i<-%AZ9$_Jn7dsA}8=XR-ID*w1|=2^H)=fiGw?w^PWhK;t1cz?^0Sr>j@c zRyeBh3)Eu>%5x-{oL;7!5J3*>JU-5k1%%q74w=HnQta%*CARreH59m`$onMm0Au`I z{_a(O6_0OSKHI7XsDD0?O9P)Xk^m!G@ksFUyrYuUsUB#;(!pZmA`FEJx7_+X_0kLy z23NgEWe10;xNodfJw4w`8xf4Q|0*jD!VIyz76;K*;}a!eENv4HQ9^dR1b=b>*fEQ! zu>`3`#ZdF)E3Hrv;W7qk=5s1;Jz{ZO5V4#UZEOW!Ahhm9crz;Y`p#aYr=HZLWcV7H z0RA~DrtOC}6r@e#&@?;#c{f46Z=9AoKbsTC*`Yeeg`_z8)`cs*Wha@9SIxT_$E}V7 zMR{HxtQSV$V!MKuSRQ?edpTFneCmH$mx$VF4aCUy?0IyyW?mKEeYp$@PBl(3+l`Eb zcLh$=@{w^?I45_jQs2g%C5G=N+&?Nri_^`n-!Vk5ETTjb7XA>mVWVOYxaLuXLVg@Q zqBeAnWX8IkO0I)8-1O||^CyaVYyrd6-fqD;{M2_6nS1XK_DEf?db$N%7lUbdSD zEsmXcuksmw8AMChuPP;ihsqg&W*g>l_rIay)h(--nQbWgLxs4<98IObiFy{liaGWTnm@giD`BIHOM)H$`p&= zQxp^s?uyP*g!?1(;4l_ z0?DS$;AeqoVR3A;)il}UcH$`|i60dZ!_y6-lNd!FWuf4(hCkZV$ngPXJoV1-xh3@I z`3;}0JA`MdG(<{_-YJ5igtxVZckuAP)Bpj-#L2p3>+@w`1ozX%cLpWL;8MGh5o$5Ka(HRiZ5mPpso2`SyPeCWrU-A}7Lpr357RA;J> zdlaYKnI93`)D&SApjNwQ2;Zc`GBHsOXoUloc#UTz4RVuX%nKn3EQ}7)WULXTVVE{C zs9{XPXqxmG`LhVFym${Hb^Q!Vwg#xYdl=Ve%6;N-;5MKM@xw%%<5`Yqz;=NKI2{2v z2o1os@TkVCkf5rY1*1vJM&mq=R6OsZ78KC*<+ z!tu|~fNg~D{IXQdWr|N775~IDXXp3JlPImauFPaU1eUOweI?@4#k?yJ&RcP~*~Dml zhA=V?2*r?sLRrT4~=;>lT>G6)IHs)LWokhkY_BA#c6fs1ZXELP<;#{Mwh&Pc_4 z-Qa(~)`;Le&u+AaTq?bj^WTVSoMWOvr|j*l_nI<2ci+tY6%7c#@Rurn0C(_PeS;NO zkH-*ieNk^FSDi_d?ZZYEK^Lfxy>C;7Dg^$xto8o+$u7E_iR$B$F;EmdqgxG>w?MIw zi|k8xa!W)qwB_XT^_fYJ%P>q}9zuUT+uREEu9b&6KM>kZPfu;RV-kx&(e_w_0*sfA z%^KURZ+&m&U@<2NmBe818MDP7hZARF;J(ykEb=d5F1DC6D(|m2A73%OVb&Dw>=_Nb zT^Arz%P2!KHCY;P!!FQtyHQ2ndTnp+{lk;9KMj$3^1{?b$rM3N2Vk03Nbg9C{5P0= zN;D1@Aj2jGVWHMKoFWmnOoQU~OhepGT(oK(UW6)*(70E)L@XragsPj2hB**~4y1LC zc%wM_^Eq}D@?V2=2f&5Z%Pl6@4fB;u?8P*60!u&j2R8mLqWxKuYd6&4kH9hRZzR~Y zT*G0*m)*EUFjCIS6#Q1kXfjn0P$=txh?wgV@?x%$2Ny>Vb$FKh*C3a7b@j? zbW;oG;m9#E-~0}bbjxJ(LfBZMj2Fc_CRned;MxoYWDI!}G5yfKA@~r>eQF^dVvI|j zOxi#I=ST@Ilz)R@cz?|wIYGIhWEezB#{-wPnm|A5r5~MGO7)@!nQDL>a~K7bEA)5P z7*Kl^D9A*nIkbkQ{+xx_Xw1fmn7ZuC9S`)p2QiPA6+m_)NoVEh6d`ufPvH-K8(UJK zFoxo2XnAv#Pc2X>53a5jB`A&}9oLiwEjZEt zCLh~gEd}}#IUcSB!2cxf&Tc0UT}B-Ovvq7v?GsYshzdwSnCP;slOL~*^J>fDcw6zd zg^45pD#<@Y105}t3*PE*KtBY=?)KyK8w zkhw4`@x}XGlFD-#;R<50xa%UePbZo4VZw9KpH_azn-1BXJN}xo*@tEXEFhu9&Tuo* zqS5B|HF_0S>^~a`nq_gS0w%WRt509+RWt;i2#%--$}fDTe;LV}_NCxmWZ)6)Dr{=} z$nOjG4g{!2D*S|buG3|$>z|JCuMir?(@%uEt+Twc?aPYD>B=n?MOymW>b{1rw+Ctg za^$wsHjGG5831#gcMq*}N-u~hkM~_N>kwz}zi~=;dCC)oOTwe-c|ZZ&v(6$N5b;5e z5+^nW?~u|rpHtsYYK%NN<$iM191Q@*y zDfM;?`oa;v+%|KD+^U5PKsizq)Wh6>3A$j2(WkOZ{K222I@bC~TL?`>%rG$hS}!Vox0-D^3$ zHfc|B<@$~1MkKMndJX10m^fga@MeR-K|pZl?J+AY;0_P~!1v9YNW?^ULnyT)HY587 zS|69oho57V{h%}-TxK_n$-7OM)@Nn-f(4@CfN?$=vejX0U1GQB*qg%em9*wxt;BQ$ zsWJ6liZemJcDew8Zaf;mQHYZ0nGGa+OX`(WV-m`lpi)WJ@Ev`jm&_w2`sb;^l@&=| z9El#cK7VwmN4RAA#4t3pXa^j!>W3R=LBKh$7DpP{i`BBFC|`m^3e2?74w?R;`HNNL z&jjE<*mu4X67-@FVX?YU`@sXTW39Q1D9~WAxnmK?bR$~voc1SlFHcGWdw6Kojk!p-&U*J0o4qoi3GIarF3sfh+=sg{7y2tv!({OO|7 zKzU^CBviyF2lSeMHsd`%$5_@7U-XJwH&c-$$LAKeiUl(q6-Ku0w;itO*+M%Y<1N?p zP%$K8?g#=L91i8cTrG6v?%H+`sz2iQZndliN<85FwQs?(TPoBNxa}AIwvJ*4N!wX( z+IPQJA;zY(DP)i{^Bj~WSAU7-mICl4!t>V@nDGrTp?8lqM(B?N^t*h>_d(^7oGoQq z2`jdKvFu&S#gf&PgVDh{y6})Rnrf%_Zv*gAY4L8E>Qg^-KOk^&bQY%lgfoJTLY^{U z8$;OghsG~1!Zo0!kC{EK!)c^Q&n}~fXZ-j$cYy!aS;0uyb7O7W6Gxd{{N}0ZKM{q| zEPG^Jiv!WSO7Kn1aT!Zyb&dcCiV|LjUDL=}vI_$3xF|)uxD~i995JX3c%nxJ1FeZ@i~xY zV=)FxaI~+sdKAqzMQuF1Zp~mG2c+(8w_R)trr7##=8jDN|DdpE zNrXDF#H2mjz1jl-7bX04(8!+{Fl4Ei0&~ue33IQ%z$_m)Dxa;<8?bPjEhorQCPUiU z+m$lp#4Mc*=_jdkQZPa!DdDl7jS%|(nJcpL5lgz&X*#7UnGQxk?ek2n$6Wf#z|qL^QrgD#?sL;6b&hA`++rM}mI&G%JE7I*ej z!4drpMVpK&e;t3|3t~cax}9Kyl`JSYqg9hns4-DUOt#pR9i-G*Qcc#wm_a$Sz*pOS z^jvuml1q;J?JJWogZX4yAX1*3mI>67A`j)Wtyt>pjY+-x+kn^W?WECkJ!Zi7J7;i* zXM<^xj`{~`)$N8+?A}<7CQb;-Uu|S`C3k|xFNJLDCezSxoc69UJFy^s2si2lkA4GJ zBm1(kmS;3Y83#1a9sK0edIbsj!Wnd+~mCW9wEj^p*?k+&ns);{6P z5?ClQaWQwGGX?N1viYWo?`+lXR`m_}yk%ubxO5FUTvIc9qAcP!RNI@pO*b(q!)n(d zz`LXqao^*&s3$cH(mum^mIlL#doq40ytJF13v^(J7RLRFv+ai?l%8t;xbT3eCHHQf zHlM8!a_Js9ld~UFAl%w+HzrTr(Pd5hh;*OS(XWLz;qzITAEi^X;XeG8o{C)lbi^C- zn>!`?nYy&XVzFv>*W1YuoX}Bqm0x}&LlhWhBQ0M3sVD6Lv(36iC2M}gS(~fqKE?luK|KjcEuA!&r2A!@4qnci&+%?2uKX0; ztIQML(sAoL+GAmq`z+9^FLa&#@}S+b&=*Ftaa7T-0xh54uF^YAn?cak3X;1G4uKqq z1k=g-;j#a3tn(I?)nIJ>Lwn}?{E9K5kDpn{LV;f!iSgj3lggeJ)0F#}aQK$-4|n%5 zO}|T6{<0x6fzHbY4jjt5iHEuNOWZngvaYHG#L@X#?-UgHQp*P*LY@>bIbo`)O4H6~ulw$aOyVh0 z056A#=LSqJgPnNdq2)5A`d#pk^F!N-9*v|Vf{A9HWjeDJ^ch;k_?1uTbyC)L!?u_G zJ2z%X&6bd}sJn#Rg~j-xmRZ1{sC14pDc!BojA+p22XMaF;F3)VrRdY)!w*Xkk94Yq z*!ZMkFv`BE#=cpr1)=!EM9TP;~S5v5JdSi zuAmQ!@i6hLr0C`^MKpcx4jWqz7_k$m#=XM&BicNS#Fh3TEN;_}JSS{6RXPM(nT%tu)W~6MPgqrpT|&;7+CN!GaYDO)!z+IL5>7coWHtw90;UY2b=H= z>;;(>^eByySM^o$*!K+RlXH|N=lF1_n=009MocDbjpa+4q#oMS-EKpn1+oajnf$z| zdZPA2&QMB8aj7aP$Dcn~CW?!cej3TN!!BU_e4Rw5D^Aa%-?rY<&2U!N9;#H4GAf2w z`Wm&PTAqmN;2cKrrT)I@{(qBalf8lZ=R%m;n4>3qPCPO**$7GDXguPa6-**o!ftu^ z@Sn%zM&YI4&|SKWY=ru833||TG86h6%=-b+gkM|NU5}}qV$dBHMh|;tK2*>Gp`xE8 zvr76#QjWHb%I(6Oy>pONb5K!$()fo%Mj8;pBc3chs{cK{cf^^@?|t3bsI@x2vU^z& zi3uRO`M51_MDP3lNJH@vIuN8ed-_>!Z{KQyaua3XGCZhW=9E@VmU4&wZCalJ2Y)fa z7wRseV*ok2uW>kBW1`5iB<9RDSOX6ckS2D};!SQ;j0! z)cD!z?=9i=E+QGYK;=M^ll(-i&rTZdoTc3e%jX9zF?WEXVpEgB=jems4oc4Y&uTjZ|uoiW;1 zj6jhunLa_m1w20iAZ7rgBrP$pSoYi3Zel7&-_KPU&74cPRo`sT#3sl?ixTw?EFXG_ z1)zz#ktOCTt#_pX{W>4A)gp%1!^L65D|ME&Q&+Pd{oAz$!A#*JM1{YUiMm8UfxWqz ziae(^mI2qT^c3%9?~I2~oA!aWs=ZbgJliSWksOksj#^e-8KNoF?5b7|n|_OpvjsoU zp3#jO0!HvOXxbtlu$5@1rQc;t=Mt@`8}pm|9JFT9Zsyhb?Fv-l>}WT-3Y_JO{+fcI zrzArTx<+#IYgTp6u^-2yFcXR2$dNHcInMoTu^~DnL{oid%F)GuR1e*8(zRe8VQ^<< z%4$pJ=q=Q?zq~h$>pC@l(aID&*mP+1n_uTC7=MeEscIYajk#n`O@wv8^!@CI>4CG| z%+0#c`V|eyQwOD()AXW?oUx4hHq34yw!F*a*44=o&Eb%T%@XQDP>W?GiRyT zr<&o6^Fj?*dhXuJ{tyqkI6tK$Zy9%bG0$?(ukc-86nQw+Vb8ajD7yzV;3T(X$YpnN z7jrjvvTskywJxqa3WGf>$ZADiV)UVzqb!nBJXa~&X<;}PNWJD=9VX%XP)27k)a0(} z_@aHIQCwBwrSY>Zj{>QnfE5d>56lXnT43~F&w3&NOmXmC9i(oIxDjS?;p1_MSajsU zPdfO(73&zIik)zZ>VUO^E4*e%X<^kZ5!DiNtJZf}Nx#c;@JP(ZQOjcup7cKuUWP?F|C z1FDb-o7!=n+flyLBCjhgl7Jygzp7}Cd9i-Q*J&+(0<-=Y)yoS$+CUxcpC#@{peWH% zO-e_IYB}U8HNN`If@^kbk>0Rjw?b)QHA>IeRL9|0n$MG*{h>-M^K6Xz2UR)qdrFDbE!yUJQ_QdT1 zO&cuEf&zqqZiKu40`(IL?CF$AUidSdem**K46s}v7mW-pZkyQc7R8U2kF|U;c0|^b zt6S5h*bV!mF|~|iq1TU(UtN=eZ$hWW!mh-VUO4~>fuCG6-wLJG~B9eTeShYv%@bH zqbf^C5idyTON_aZ&aVV^)e-R?SwHdFb~mwGXzsn|g!L~0jtqU1%<_`&tS6_Gokcwu zC%ao6X0><{15O~Z=}v)X?>{CBc09Q3!+hGhx``nLli~0m3F~)8mBHiGlL#9!sd30I zr2f2zD@B-;W#D~pI{IBl?e3xbquu1we`3wRaM-P{Ge+t2{76IljAspZy77KtfQ2+n{BWg!6lOWoZ6(n=Aie z`a5IaXJ}Hw#F>m_2rZSvt^yp7cvVBdjUQWT5NFV(uf!_|su4L%R~(u#@P?Taoj54b zjh#6-G?M%+&9Qe{)eOK-_T6MXr%~l zm!fTLOu?lEWpL1yMA)lPDHfCh`G$if1? zq4O1^iILJDHGXVCU>HFTWW?P%zg=p8NmK;Dm*{e&scPgpQ~{e_F;Hqag&!s{K*X#^Wl7WmIEQ}-vnCYg+-mf^ zg#W`V%WoL&8@za$}V!b2>6quCf}+y_%L zos$u~&6K=;#yp8Np$L0$-yP5d+0?=nCypMyD7w;gH^;G8M0pDNsQ$cj9Zh5pw0}<4 zQ9jzvaO#8OM}r~aiVwKD;P#UqCqR@~?n!NVt0pb=h^xPpLu{Z0SqBG<8bPp`B0eSaD&0$M^zFy$9oUk zJaUbcP)a51^9QPVDfZQ4*k2I|9DNn&B@S5w$4;?aP~B4E+>2SGcq#(`07iMz0kL(+ zmp$afAHay5D#YHoFp8-qH~r5Q_`0MCydH~_N+%OKuV)O}grB(>LofO~Rb=1z*a^Hd z33#ZN;A4aGR?w3{RYYd1Yj}AgCVGP{!|_+S2n>P$nfKGp^|!vKNWNHMbOQSOQOY+P zTZ_hK6vDrpBu&HHp-C)-Ym!R6S4-OHA-A53N7fS;rcnV?AA^iy_%4W}mj9l)*+P;x zF)*N3oH=&6wD26V4z2Ri-gNNAKdq12x^&cK1uthPW3xQFPe z#Ox~y7~IiGq68$o@VAicY92KHDEkh3|cb(8=Tc z^%s1{FY{y!-d7hW6On7U6#n$=D_9|hRnd_5GHSRb?DPQ_yR0Bi^$2I;Yvksk*F0x4 z<_~;elh1;&-yC+-5{sp^q_25a_tRmRkGyWvx$=n2?CDZAeJJ4J-hk(y4Qs((UtmaR zZu9!X;ndaWY$x(Ozu+d?290l)vXQipDor5*R&Wbjss*VIlva8QZGbQgEs#?&oQqMT zwIC^mJ>k9_47c=ZpI49u~o(7`dz8_^K91qK}ESgRuRF){WgcfDoU@O+|-;8&UtXBEe=2rmEs zm_rdJKRm-x7NI|o&(O5^^Z0>hWLzh?Xu@iXFToCO{fMbWR=t1}UO?J=MH%vEOlrdK znad%cwsyPJSmx=lFo6C;xX}OAxt}uAwAX(irQ{EF@AIep@dxCPe~OIWeor2NYx3(u z2t22=?ucv@BkxiS37`k(rh|&y^oC7Q0002a2|FvraY!@|(863!j0!?b%_D$^SWAQ| z;R65~N!3$BCt~PSX9N-NWLsRi@;a|9P!o?c-xYkmn&XFDIi1W~_{U)?naInEIi1Bc zZ3bowDgyujZ;y9Yp0;|kirVt}>r4$akkHUGeQw%FKt2jb-vWai%_EOeB75+-04=!a zLfSHQt4~l4-$E@5os^5P?GGn#kWrTQ5v9*W2J-}vlMK18jJO)E_K#znvzi@9kYw6m zG!1SQ*ozo!pF~>aZMtD>+I4~Vb(bH#ABSzc$OEMThCLuoDC8um?ME&+IF8QXST)iN z|Ky_6Adnig`@S!6)`+P7vSD9ptSqh1GU><(fUI0xOH1>tygnJmVyT+-;@4I|^9e0j zpChQ&jb4?9iE6~9Tu|m{;i4O0T#kGR4M*v;6CL!)QjiBxb$)Y{rB3C;k}-so#$T$N zdYCPxAPgSK6J>_3K@4))AnOOQ)&S-|OK*?8<|YhyjV0eIuT{y|@u(~()O5)mUk^tV z_CeFMoZBTTYt12u9#6Bv*X-j6(_5>C@fohlaA`0JTE9tO^*t<%TA^P9mjEGMiqy+u zSbSvpOzlLUkSVA+WVhXYFNEig(X=t1EEwD#U{$ce39Z)FjCVoXMGo3V?6`pIRZWBh zH0P7yfhX9j+ZoLxcnSaA8U-BVb;W=9DZHIQB9*tN0Qt$? z9<^Cy4@xdZU?6_WgvweUaGw|?*;L+Hzk$+&*$RAYwyjG06@=Rto3i7`&UBsW*gBdc zazXXMmgfB_R`XF$0@twnYl`5M7}}7dzUGpFbE8`t0*?%Y0x`zn-{ciLPDu`PlOUle zx?jS$(1HjJE3dCW3me9{En}pw1ki};#KehDV5*=ed){uF^goQ`@*c_hORhC&|y&k004e+A?y(j3~`3T zLBE!j5o;K7HlmOigMbJ}L0Bi!do2J~Rzx&}D96Q-uGe-$_`c*56iiJoQutz7&NN^e z9Mk^EQ6!3Y?>AJH(Z4&W3Q6oi0ssI}C)J#wV-Js7yONVBMSJL#2Oagx(4dRk39uS3 zOiybNm1Ec;TYAasRHxTn4!@!xz2?p=+(0M!ax&HfB6BPZ=*Q=grT*F(yR4iKyEaIx z!;6{uzn$aSFMspZAJ>(YGthp(Nn-aBQdRb_SYQ>~AqvaxcSgd6L}*eEM8L9C&;puB z`PT5&HhEDyDejeNJCb8jMO_eS($$bsYp(9lo+*aaEFuaC~(eC^E`9D zdc_NBglGv5J=#H{n_qA|mM8yn;6u0)nfa5(Ka7>)p%T934sySMC zdkaZtvLG-P2MP?*nHXCGxkM$!$0K+Wrm?mauSIf6grru@+5vm1ua`g?!7?mPm^gFB zD@q-h6;YPw3*bW7jCZ&JHx_VU2kX#J;up#l zbZ>*~U$=dg0G@LBypSAW;`r=rNroP)`L`h(m!Bkq7K?W0XQ+fKzBu`X&I?%Z3U9_1 z?_n2LD=@I|7i3RupzZhb%qj^Sw<8R8VrbzuTbv#5{R}74G>IqggPp~_lx4;R*^xlD z*dJ16m#_Q@(acwr`41ktq^mNHEMG9l3qC}DIw=lFL;8?VpAG@Xu_@#OAfsc7pWKCm z!3wpm>%};BM2N6xl`C`7S5Z4k#?Wj+)q`F0P5=M^rTbaPW=pJ2S8HDXJsNfbaHiNEFx)hMsII)WpYbpc_KP=a&=`a zB5Q4PNp5p=VQyn(Nkk$#B1A<-K~+ptA}k_nZF5O(b97;DV`WuROd>iWL`6nHRZLVO zEFx`Tb97~LOJ#XUL?SvOW^ZqFYh`&NEFyPuVQ_F|WJEA6!i`|!dHmuSw*jLg6TMaovuVz0-p*h#Tovl? zNlwaf&z4e`!p`p{;;X_tL0{qKNzD~sg=@3RCL=?O^O~+%wqc;v|m7un7)_CE%(nU4aq7w z>Zg*Ik5IR793z6n-l2Od_%p`_c>eJNi`(>?P2?=e$+FT-ehVB#G*!8-QOSx zZX>;qTI?<-?%=QX5&B2;q+g`EmkeM4_In`36*itO?WZs1$PPV)Mz-x3<$sG^0ehh# zll?2q6Ox#K6T7y%?xRbuZerk65>{sGG0w zAfX*W5nM}Ia+()`j39SeX2z!$N{Hr;?wu{Qsf9WqTtFZJqh>maCWAMW>DE&>Emt}jR%CWHv;ATA*qfN}z%XEYa z?d!Cf-JODNz#FW|w_6~qRF~%r0!N%fGaqiYYLFM@YZ4sDJU&Z4i zYKzL0#UrPBIocd7CKA=4rNPzXdtmZ^WhWI?u5wwB%}mr z`J>;X-h>i)7qqH|mq3M2KTGwhY)fHzVn3@02jG;Wh_xxvER@T>HGxtI=;1=(xMQO_ zerP26f$#s3mf!VUgMP(S00+22L`Es|3Y9%_Ebc$q{+qflbyZ7&1GV=820Py8YJau- zT}qV2wt-&ychqvQf)B3Z7R{AM(FWo!ix6#*&60LS1(19 (generated with pyarrow 24.0.0) +""" + +import base64 +import sys +from datetime import timedelta +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +import pyarrow.parquet.encryption as pe + +NUM_ROWS = 500 +FOOTER_KEY = b"0123456789abcdef" +FOOTER_KEY_ID = "footkey" + + +class Base64KmsClient(pe.KmsClient): + """Not a secure KMS: wrapping a key is just base64 encoding it. For test fixtures only.""" + + def __init__(self, kms_connection_configuration): + pe.KmsClient.__init__(self) + + def wrap_key(self, key_bytes, master_key_identifier): + return base64.b64encode(key_bytes).decode("ascii") + + def unwrap_key(self, wrapped_key, master_key_identifier): + return base64.b64decode(wrapped_key) + + +def table(): + seq, ids, names, tags = [], [], [], [] + for i in range(NUM_ROWS): + seq.append(i) + ids.append(None if i % 7 == 0 else i) + names.append(None if i % 5 == 0 else "name-%d" % i) + if i % 4 == 0: + tags.append(None) + else: + tags.append([None if (i + j) % 3 == 0 else "tag-%d-%d" % (i, j) for j in range(i % 4)]) + schema = pa.schema([ + pa.field("seq", pa.int64(), nullable=False), + pa.field("id", pa.int32()), + pa.field("name", pa.string()), + pa.field("tags", pa.list_(pa.field("element", pa.string()))), + ]) + return pa.table( + {"seq": seq, "id": pa.array(ids, pa.int32()), "name": names, "tags": tags}, schema=schema) + + +def main(out_dir): + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + factory = pe.CryptoFactory(lambda config: Base64KmsClient(config)) + kms_config = pe.KmsConnectionConfig() + for algorithm, algorithm_name in (("AES_GCM_V1", "aes-gcm-v1"), ("AES_GCM_CTR_V1", "aes-gcm-ctr-v1")): + for codec in ("none", "snappy"): + encryption = pe.EncryptionConfiguration( + footer_key=FOOTER_KEY_ID, + uniform_encryption=True, + encryption_algorithm=algorithm, + double_wrapping=False, + internal_key_material=True, + cache_lifetime=timedelta(minutes=1), + ) + properties = factory.file_encryption_properties(kms_config, encryption) + path = out / ("arrow-v2-%s-%s.parquet" % (algorithm_name, "uncompressed" if codec == "none" else codec)) + with pq.ParquetWriter( + str(path), table().schema, encryption_properties=properties, + data_page_version="2.0", compression=codec, write_batch_size=100, + data_page_size=1, use_dictionary=False) as writer: + writer.write_table(table()) + print("wrote", path, path.stat().st_size) + + +if __name__ == "__main__": + main(sys.argv[1] if len(sys.argv) > 1 else ".") diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-ctr-v1-snappy.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-ctr-v1-snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d99a49f92cfdff9f6b51ebffc49774db98768958 GIT binary patch literal 7722 zcmb7}Wl){X)~0cS!^U9)!QI(}1b6q~1PugtC%F5DKyY_=cXxMp2tk8xe8Tz8)XbT$ z-l>^C-BtbTdhTnjRn`3{39E?vLP0^9hM~rtvibREuRIPmnK0_%x8FE&H5T)*pK%~U?V^T%T zOB^1Lk$GY*AgT6*NVPW?3GEQYY%~_W_%N8SP^sOgi_huDHJ{`;Rs6r4bn79OW|xBA zf!+W$rZNSppmrca?)@ppHYQB$-zIJOk}*ufGbw-_W^G~vq%ksm>@ZGO#C--T;<^b%E4 zk)fCiJuBu$-)Cx8VOdvXu+=MX3qyqUgGh~18rj@(qsvwL+gs&4I-Q@tvdbT&w~JpZ zaGJdV;&ZcqVV3ts^irS+morO%pzr`JUmtz8qTc|ux$CSL_E4{G>rDcdTD|TIp6FuB zi+5MqGBF(G){+=B2(&q)Tx6TgZg4qCkm9#&yqbUKj}-4##0qY$63hPRKCAQz=)*pA z`(u~a`Z}}f+zN3rpXNvlY{47F7B0gNQVdY^hD!Px|M80UImpI|cAs){;6!)3!}jp} zWRqu$*t>J{{Ou@`mByA?0^9<=nVxr0l8fD7KE#QIlXtpIj+7Vp zaPaft7Hz7oW`(}qj`Rn)4)xx;ONeZYv=!RZ)CoLlF;(4{M7LtusA~L_(q`}Yp6yaI zvAXqN%~fJnZg-EE(twYekQzSg7SYBJKk73^nn?aTaixHs6y?B%ctadSYRr&t~SX`C9~B0 zf#74Cv9c@f6#2aPZ6BKbEpJ4sB?S@<{Q=Bo-0qWoQ*Yi}mGq*F8#G=ZS*`^iyJJ~U z-0OHBUUymz#>=Xab zz~6GmctcmSGZ+JFU@m5il-xCOJJTBkdrK=2;9qnvC9Jhcyd(F8Mf1&kL!XSPX1cCF z%)I#cvGl=2azbvXsv^oV@3Z5~!csKUBg8dXLme}9&Y2{Tn7Yi2QRV%HeCJe;VVNnO zTUdvq-ij0KDCN{JlW8psLLGswal}AMQ9F59&jlok!qJ@J=K>x?A=2$nZ|Jr&L@_Vg)A&r*dSIE|Fr%Qw zRLZWBhFJZM?5aoJ3@uv7C-E;B3zv>T zZXF3~KLger6PL1dPpDv=t1gBqL7BsKWqcq7Z||e7GuPEh&JW$^%;N<**A>!^a9cOt z?gabGGY!NWp6Bg%|6VZM{t^*n|MswDn4&JEY|_P;ZP9cG1fa#ng!P8gf)kF$dh_5U zp_%2DL8W_4^SJRvyysFWQs2b0j~&JuC~~AvJ`A`7PN@NU>0Womy$Je=Ui`9sR-nGR z5IM*z+3;8t&ukqf}n7@3@=E`&Ct_-?X^$qp# z61ioV9v)3!sy3RUDf)yLg%X=I6FBUTL;dVv%zrN*(yjK@l0Rs($|A83y+2YfVpw#C zyKdJk=Ep^Bm?{IxHz~e!D#4;_iSnJ zN*fD+moCL?BILuSBH2HK-VOR~&1vj9yu&b!HnL6SwxvlgIjT~bpXib9)&2}5)~M4E zBd<`RuH8R%6awxefmEh2KNh)_G}e3JP{5s3fX`_U-|}#P^A20^wUb&B0e5*hC^LHp zDCQNAFt_c&RJ5T0iZ3jfBYz2|{f3Gp6@>Bs8k=`EhD8}3dM4xF%J^%QqSKEq;?=Yi zuD@u=G;lEJW6;QN^jdM$;8n}@B?*^QrvYM%KxhG#b^ z!}}@SG5YoWa8*~T~q3hwyDa{Sixzw)@UnlLPOA4)#S_EAdB?hzPXky+lx z$PbnzrX>Bm9WiqAfiKQU2ShV}aju(Y(?_vO09&+hh#VNNZxdeV5?Njl-TfOM1C56= zEtfD^!y>a|LTvYh3`@lV897Qzh|Q4+KdZ~vr`!it4<#CcQwc+uSIk55M8DSu&=fUb zzoGr_c?6d*9~9%*LOV|ter;LMQOSK6`VRV>+rXWoDM9F44mXqQLP6IBF;BkKJ&G6c)uJ{2c73LMuCuQ8|JO zbhNRhYHGc(_rQxxLcM9|UT`-5tjP7RnLXQ}9_DX6>(l1AXYmcWlB;%xqD4(cp|`j$ zgW!Fjo=OnM$ihqf?XMfR!olYr=)eJ2Md;DM03;s+e^?p~5uI+|!rb1zJ4`#Hkkc*0 zE9p@#VgZMZY=VzJE&=@%O7lHu{`nsF`5Rt_d+6{?Ji{Rrfxil=sB6(Q=LL`VQ4Rll z9=bscw_U!I5*$`F7kD;~A%jY;ap}LQNA9K{yW};#vDUs~ai)K>GiV+oX;^thk~paA zhh-wcOavflRW$@%wABVuT780Zss?jc^=WHZcR({`Ze2lI*f-9pIqZ6bJ|iw3W9Inc z_}z*XhQwuwLfc-T zlCB&R8B{rrezD_s7qh7MbjBRG6v2K!FasNTIo$V`{QRfBP70=Bh?1%R5|mP08DpJ- z=HHJrxMf^@J`Rvq-@X&u6sHyH#_@nGh}<9aV?Sq`9(d69N*!*5Uq~MEAzLt;aS=sH zxqD4!eDjcPVegTp5D~~Hsp$X(P zuGHsW6Y*BVe~puBcj8O@MIK$>DQk1KI0{*8mQ7hb`tk=3j5Q} zH^hrkklK>emagi7g6w62cbet2LS$o}%xA?oSrS_S+z?G4R(1m_gBQGcteT1K8^xQe zn6g9VD*eGN1Vi-GU40n@&lBWgv7T}*Iu5!)W>r&xI75aCE7dL6I%pCy!#FmxAdk)} z4m3?}sA$7=xEaZRa{B)y^WWNu|2k!HopD>_|90Pha)f~cUwz}rNNDTGOmfS4(Fk*# zWiF-lCuu&0NIpO52fH%A7~D+Uw?~>s{P#EQ;=SugKQT8ICF*J`BV7a@KauUsaf9>h zL)L^t$@6b006cf}?ahdMTb^1?nFLGpJue&qtj~Jh`5cwitug6G8l1yI&}3I!ad6cM z8=*b@QZ>L>GEC|e9&EADNPYw;#W@6WNH>gjj=VC_^gI$X@bFlIY&&%AT7&<^%2Ad3 z!|Yy%4{h$k-KGLpi8)ry2Rl1+1KckICqc?g)WKJBIEAAK8(be#qhK;4o|}T}!)+Hl z#u7EWOwisV1g@%ri@-4_yt4UNr8<)p2cW!++hU(B2X2 z=tSq7jOS$`ed6;s5t(FD4-)4$w(s8_m;*>NPIex+FLF(bmy4CeT#9M&lFn_qIoy$H zx%Yq=Gki?<9zq_TeXE^hl`M-eD~4@R4a3_SsswmvQQ5hblAa6=PSqr0lAH_7nGWO~ z|Ga)_@|k#Qvjc!GRMC3+Vu!Qa<^AOoe?^lW3)3`j))^SqMs0$1M3aVz*>94UcTRh^DaRv!-7@!92 zB$uUUw>Ar+EW=dJqpy;KLyjNyO}5KvPQ5qEiAJ-f)h)(DMFF~6E ztkcKLM9aS#H-0^~fFmnRGg#;5h#oHBRM8edV0Xus=8?ugqQlSLud(KkSge##Zf&;Wi%o~f$K<&Fv$eMUCjyuzvAOeNXRY$D_vTtljn8q{r0 zXVGv4Pry4jg%VzjdA#A>ezRz{uLD;yCL(hoJJg41@heLW;A7xNnqT?iKT=Ihe7D@Q z%Ql>q;f)-2yGFibs-vDJ+;w$`%WjSYqyCfOf0)DnsrKI!8|$y(;vr))$&&_?-N-Ga zV+V=s?A?@5EQqiUJh0wg!iNuYRSSiDzGb>RkzV8&+YxW1k|e^=BPF~|&jRG7!@h&O zJ;>_F-0eSQmKttG+X;y)3T6(8Em~1*7?j#V$6?x(eh~Mv=Ni?AXjWzF$_!8Qy|?+X za}ff(mj1(i@)9a@N(lE39wuQvz^azQcZ1lKGIhv#X33cLmxOFRZ~TU8h*Lvo7w$a- z$0Oy#Qn5$jP%t zedHo$ZYuqZwRp0!yU&)RFENP);lsv*?OfBG5*xD$Id>6>R`Cn+vsV~7>dS*ICig-t z3U6{7YxAWa5}0)8dob3_E9y+Cw1@iZ{je3RoPWf}=XCv1!yRRug^a|BtXA+e5`w~z z5U`+|AbzND(4bROAda{0B#~me;`znb_%G=ze{jiBfOpVNb0u_Q`KcHIJ0%OKJC?Mk z^~&N`;>;?cpDZl7lTE^x`qnf393AMKJEFG#N$&sel-~5ER^3%ek-v&ZE(hC4(-FHh zkU`#0&<%X-g6j-SU$xk}$+zJ>C*FP{J`QpET~=4nVlxSZj>xU+U9%s^ttc>hF%fXAXYN5U#DmzAQmyitl$NKtHs0&HHvR2{o*qntUW`*8ZQLL zC&BsmVsRq8>fBuSuI-J(HGnKT=ZL{xOs~NV(k|AC2-d@h=RG$W5?i>GW6DPd_9 zN3@15%x?|8wR}<4e%I!VLj3mtWQ3`4FE9;K7VpTx4wy`K{(~k5bZP8n zDT`QAI6qU6Rf<1TIDdQqMbX1_uX5f;ZK5;wgU1NO4ab6mt%ew9<66I$( zG9ps64Chy)A5>qni6N2p-gJ_3#P57yBCC0VU!3I+-o!*=BNpe#RuZfIPmY-3BBG{9 zv$HTzT*Yrx`*Qj;`iH(X{7UG)$g*?tuN0`lD~-imxgAaxm)-(Z9RU}X$)pug%2#mL zGLxrN`O3`Vv-T%s{W}Y)5oiFWeVq*<$Q``d>oZF}LdHdJ-QCoWOoNni_ZjBqamB$N?o|%Bw{pe^dMI!&K$UV7d_tgH*M!PO$d*A{ zh@=^Q7ePwq4%H-m@6c`zdVG5Q?uFaXdJ39gDtX`BtqT}cbntvEoX&`G@jT#oB7b%z z#-58dcx5@6N8RrlWU}y_$EHDYc%0$TJys-FE1!p2f+k+X!c` z`aL~rZ$z#obU=-zlZ9Z%sM8VTu}i?_5gN+`u+J_xNXR*LqlJ@YxY}iHnX2kXYNnW7 zVfS8wsY!p@G`Pb1lkslD`t$bX`K9KVj%lZQ4e%MF>;4_a4JXXB?ItS7lW`!8B#rfG zeh*->cWVOu^Iaek>dmX9c{oCAOFlL{h!Cfqb!vtKVL0U?_hW;q_oNV?o+-S&j_8gw z2lpfx6@!_Bo)#RUiNj^H9@Aj9$sRB``1YNTghh0XzN&E^)F@mn=vqS82{^jXwgr?; zkaJwxE@wQ&mVRoz$3RO1VDStDA?n9JPO?Mx4_T+_HV@6*bYH zReAJp-r=(*eCVR>z9--luAo^lZZa+30?AVa0F-)w+J@x$Y4UiPv%Q(L;Lr>_h1d+C zh@DvOeTT;nzJSeUEY2H(i`#72DJ9Bbx0&H2YdWP_Dn03p=8TQHj>yaFwUY7t?n?g% zohbRB&~*-e1hlluRBkNG`XDLttn5-;s{vwF{~x>~CZk>;at#_1d!g2ElCv+Wq%aRz>+OLCmH$?!uP8B$zvrel9yQHmO)ob|q`ji%( z{hm?b$q!U>3i%=Z_68&@eop-|vTqFa5T1UFNb9)?8KFgKKE&ZhSRKDs(YPjy=5`v& zzmGNBiZZddLq1RAX(6n>WVDA-sdNlM7c;&#jsuQ*%Y^dBIFs{qa!A_(w`eGKP0Z;$ zDnFTUPFhRWlRays=#Y&t-0Vg+m36%7lBDpUMr7A|6UWz-lpawj&*oyG$i8pBF@!;F z3O7#{t~v6uga8rzUw%sJpvT|ir;>eYxN?c&(!sOk6A!fSAz;Rl-J4{lrTjbXRkK8# zvfz5zPZYW%v0gUdo{{s&zXEIP{nZ+qNR#_v|zn><;oZ2HD9iwf>?*+M}9a_*FQ3OfGm{7^_M<9RM!U2)WX zJBMQ7?00uQGR4~Pa$4N{1o~kCP>BZsh!y#F7a3I}kp+0Zsv=~JhJRWgYSRlEQ1fZ& z^OhH$3mctC3z#MG+H>>??Vjll+uRsC1Rnb3~a_~>Eu61(k$eQLk(L( z07b}i21UAWGpRE!#u_BM?_DrIM71cb)K5Z3J97^>6LToKJG3fPJ|CV!>*6d^UIDAr zG_t)aO=Z6LmP37JbeT~sl{aahk#Vl;I@rcu$TxeTOQo^qdzD_E2M)P}jr8ZFH^C#r zrvCKyB@vac_l3QT_lxW>tSOb8v5?c+MU0JG$NVw{kQH48Va`nfb2S^*AK**W18gHM zeC=MGms6aioZ2Od8q+`8C&9))pEk;3@YJGNH>Gzv0bs|s=)_Ih@frdL>kQ)LvW1J= zC1Ras_ZFC5Qf`*5Y|4QY-od!q@4C+UjzN`9$1!)(JtKV8i!Puof4uKNfLiG|1WCB# z>GYKXcO^gd`M9ZbqKgcoi_$ih1vpcO-@~6rE>*`}_kSddBonEOliI*w332-6tM9I+#B8LO{D`hr>bA0l_MuK&E4gIi+~KDK;GZI&xr3h)VUadN8dX>_Q$VA z$I&Di888pJ2{Bve3Vx<2xmBDn^Oq=Z?<0BvRq$cno%POcY#|zBYfmLdMQKGN5DVr{ zSWTq7%eL8Ss!3rT?@I8)ZTUjGhMJs*m(Aoko5*?42Qi#qeZ2#Q+Mh*fp8rTDHE(+|CL%QZE7IB1&a*1g&Fa?>I?x5uhA zNg^^T=@?$GS99Y$;FRBtPJZC+!o##l?bRk^whj&u@N z`sY2z_Xl&>#xI}P9c0rdVdnukI#w_7HQi%+T_IqJr^+8kCU(GyG9fy@y~i7FI;Po| zFe)Y;Ado4}SY9|O^HxGJC}Avy>CfO$Sw8fB-0wv>D?EEiq2Igvo}>@kLezE~Oro%> i9`~;vVzCBFo0Iomn3w#H9`(ZAVOoFtN9pfh;{O4MFCo_e literal 0 HcmV?d00001 diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-ctr-v1-uncompressed.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-ctr-v1-uncompressed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..805361aca069ec8dc17326d72fd9a5fa08a90ad6 GIT binary patch literal 8684 zcmb7~Wl$a6qF@j1?(Xgo+}(qFa3@%BClK5Y?(QBS5G=U6y9W&%+&Rb#^X}Blom=13 z%%9#>{cG)2YfE*ts<^uJR{#L81oY;%Z-l@0e5CUagfw)Bj3Ss;m`>_CT2zop$}n>K za1k*fR=?^s8#WWFrlFZ#;4d^VYjaj#3sS(TE+DuZX_|6ADmMmAnTk0HojPPv=Lcu$ z)HS)JHyd(nm8g=@x$H_?&A%hTTlK*Si~{$;SL%)e3J zxAA)BLNH;GM2hOy6}X_}HQrGE99x%&E)B;uf4J!jZ*MQS(afciz&VtX^u& zj*%a<)OD(L6@8MKKYD>w9+|aOIGIU)(2jv1lALJOZ_PK0@DiIdE2fV3mx2B)MTJdp zZ8jKL_4TbCL3k33_?m&={EjDn5S7;P(_6KQK*dWJyEWN=9)Tx!-G-#12@&MHMfdPx zJmB|5a*u=lvvtjIW5F)UG{ZCRouvFxGKV_Z*us3T%<8?UrSJqMF@fBPhn_23bC>Nt zMHy7QvPfCd&QUQXLi$O!F;X+o2s|x{GFigcf9Cwt`U>itML2*(X#UT2j2=viY~V&SQwbSkCq~n@*B_{ z;-ct7P_Ni+KC*7Vyqp@%6K4A5)ao+p2Lnc#)?WQ->+w_RCZcc$D7siZ{UA@RZy|xx zY4xY&a^@B+4Gj%mS^27Xi_{}uuM2wD4Q36?JAlWZo_K&UX?MSE&EuV z3VKRChR`;6ii9Rb&jw_7NHeBl>A3ZYJTKQ7`3HCrkK617iWz0psu#%6;0pq=s}Z}LGB?GvQTQtx1QF>UiZ{)!8`u*ElW3w9UUY)Prp+f z!`Hb;cp%8La1rfw)(BaaGbih*<0~IcxMBc27Enj?wiFq;g8`DH~oN;QfIN?Qi_L;+n}=l9^3|1ls@ z{w3PHlWdR!JS5HSV}8;4P~1`TO{^D&4>k|M1=k=sM|kS3@||0x-oASr>tz_Ih%tcG zijTM0rh5H6xVchoqZer3ec|S+1XV$7Mjxas4|DCL&I93rVl*P^&H4l>brD4#RTUwS z-Bg^-5|fZwi-NJN2*d{! z8`T&p{t1;FB)KE{U)10iRI0y3B<;8p*(WZHd}6G5Z*U2>-y@~G?Zg>oFoT@R{1IN^ zS|!uo!3!Jq?N3A=5n^XPA@7a|4Qgi4BXNN9jy&1X5Tz1;rPL8i61h3bSVg}UqnaIP zKNjGvJ9IFW=A-dor5c&hxoAJctF4)YjvxB zB_OJzDQ(jFN_rUVn+2SV3j_Yfy5S z3j){BBI_DFRn{ZVgjN#WqG_2D!gQ4GL{#S&1UP4KI4m{x>+vsGv1*^B?;a@_ME~8$ zf2RWX4z6tx)QIg!q27EeUFi?=H!1yMee1j@?o zO*h<#xrlk3O4f98EuXaA_fQF4>`L34vE*PwbX&rZ?)quB5X)?zj?eL+0-Kh<*Si zZ5BJ%dv002Rq}E`kOqhU_tfO3vkUg)_xjJ>iZOGuM4P_n9ryowpmqKw(h#eu#n&g% zo;^lF8EqP}(vZ4@}nnoQ12!!5x>O$k^>gd5utLO!i0; zX&S52`IQpL6(GelCyRTDI1_0g{gWYSRVz1UHnA4T?|W?HrmA4}(GJ8)0NP3q^au$a z@iySnZi&geDTRkvrUelk;=Ur}iSnD_uvPWlDI}a$Z?u?X0IvwMtu%qIye?I#C)#wC z>%$e+%^WFc1PZRhBf-b1ypPs z-Rn?mYIpoNG|-lhcAeu7WYl60r)l8ValLXsx>bl--sSjtLhp)w?N&AU;R+a#f4{`e zg6+)G3;f)>#dlNY zpnF?np57c`aZXF$C9jng^_e5O%bwsA(OVfZ*NKk4;cAqz6s@WmmnA9El1X0M*GXJD zUW(-~Tl1K@=I@08hUo>;WtcJ?)v_pX_-f2@SWfCXA^&OlGoeI0t&&e4Q>^>t1v2xY z+Jn3eJub)Pxa!gG%&G;_4$`WFs$Nz9XFwAKNXI6IhlRsfn?yJ1?re@XZ=A$Z=x*HXh%mOgVvx<7jRB3 zgX8eP`RWEN(Ge4C=-28WYi~!wIFNGqS&FG{XdSgh^gMetsSMt@`1@&cxmmr41|U2G zY|mT+?$RO+`UMiV*nk;ItFh%_8F=y8ykYd!TvY0uI^lzRx7-agtD5`6Rp}3Ng!9fO zXxZ%sTj6&;*YMZJUI`^tuLCFq`kIp9C?w7ZU2V*22HYfCRHG&{aXMTa2WnlUtM*ZW zx9UNcYcX<=dSm@*eo^isBauDz?(eaz<=E2C%(o(usigvGywQhjU!8E{gUj4Av_{Pt zzA7$O_yp;4H~9Rg<%RVs*Pl?Ue(YoA%i=!O=8!uNI{uoPVemRGITOE<@b|tfd*En# zXr1eomt8A@SH7fP2!^}4U!!0ILgn8?j$MfyeX4hTCejyOdc|j|yjwRWv%zvPoqp4V zyLVHgGy(QJ0(Vr$quJLLWsHNbXOPjLu^A6r1Lt`L69+}tg(P1(mmK2am3cV1ZN&rF z3k?0pC0JXSj!)xyaqc(OLe88`@up!7q$Mu>1a%50TYh@gb~S%OABs$^P1RuQWL#r^ zfs=4^(rkKlZ&x#H41TvvgiyA=oFw^pFo%=ryY~i~h5%Z^(D%qvUh|Rj#AV)w)AS7O z)~^%p@MUhdiX_%Mma-V`Da-4>(lcpeFet2Fu*VnIuBp(^$^&c(R~IG@%*52;@IMdYnQjd;*= zmPq8LmE$&+XsJ=Pmp8$}W70MA`_>eYIC>uYCLFgEzH59VXLeNDbs$zpT~SYbr+8Y#jp2awErjXNClV~uNO-#B-z zG_aZ`zz}psy%DiK{=?%#`sb3GgZpY{NR(<58kq?(3Yj{NQRMu`MZ&vfAD~SJ*8IHy zZ_rOP$3x>U>qSMp9mbGph&3o2ucFYR4VJyKKYS(3xc%UDr<_w%5s%A6*ZE-=|)GpZ~*4 zPY3{jeSdx!-^4D(v%*Pp(?Y`#KW1v)>!}A4GaT=#1Y2f&+yZ@RLBg7^qGCkJsqVL; zRW>Yr9+COHOsDNym2PL4q70NmsV#+v%)Ct4|#dhTMfrq+7_ukM5J_SE6tpo{6n)&{hfswO*;W+n0*EKHPC4q2W_FC%#K zej1XuVO~3vF(F2SV@?!1eM8=hF_qCK7GhGws5UlJARuE|Z|GHM?R!Lb5!e z^hcxlwxkS#TW0?Q(n|V!$MgtQz?w<|!KU@~xJsns|Dli*?Ft}>7UBO|?PWr)l<-FDTG2D4BoQ2|_$Ep%K|dvJVs3Y?ISE ziLzkq`C#T=Y4p6$U(h*q*L1<}P7{9Cb4jbJ4G~CA*Rzi^Nwsfhi1dOkA-?kSbfhUO z4Rup)6qZZR#MG2dURV2M|JpF(rZD}w`V$M2K(EjQzUQToPAB|5=|3g(|J2UEDvSJ^ zkYAvTx5`6+XJN)|WO{G)-MnY6NcW_PmLt?47UmGUXl{s#OzO4rhtQ(s67eq+deDM1 z?zVBOVt#1Wz3Ju61k7JS2E2omnqd>KIkkGI_ZV}O9r-|NcpEqoYY4?4TyY^PNE)|* zu9bxNZwU~$#87%gU14?Ch0(0kp)YWKNsG!DAA7LqCjN{uZb6fy*ye2Rs8){1okikq z5gl4reG>U0b;W?WaxV?wm?F*xuL1ZK2V}s_h(DN5l~~b%Cp%JX8v=#+dg^`>Q|MH5 z&rq+Q9edn_0jLEcNmzv%YIEVO!)}HAY>gasu14F5l`H<%chDkR!y*`@^PF(vq2BSu zHQc7_+n)J&%hg(aYf1+*6;`RGdKbAV9A7a${OCI!5NuamP+a_0kP#mYA-xpQ_a(-# zgs}BMIoaQ$RobCJNI!3LeTGYK$$K>Wn-#cF4QOxHiONV|M?9r%xyDIV;iaDwXqR61 zbCo{$U@SZ}t=6P27CcLY5;&eXPpyNSTyyU*V^i@S)rb%n#uR9 z^ZMFmDI^0dUAx(fXT1COvpjo!eGayRi(t}A-|eH>hITC7;#1_V;-V`#gd;&=b6K?< zbK`K^`*~v0G%Fvxqt_fmW)sr~*}7(BUR0*Me(Zwsxse`y3p>u=STr z87ZU0HvbgW|9vO=_dP56Z&dCnzoW%8I6JpLAG!LYd=wn(zf9YjeHo5^X}vqI03U-? zTRYC?-JG0FC+y;n*doUC~`K!$wG0>|#MfM4C!h62W zteAof85KDg<$WSqr~I=fAg0ttH8B0mMy#xf!w_L>n$X>ziaK)11O}$P?~W&)*MzAjCdM;@JKM|*ZyqW|XyeY;qSN9u6;#A0E^y#bRe{{vR<&rKC^%r5)G2`DpWm}G zUkKgBP>DRbhzDgM4^9un0BPjq1)iK$w78R&L*S3v8A>?$eoQ%;_UKmXC*Q%6OJ^cd z9m1un(P?V%mLO9k2+lhkBDX=2vmGCe(;`TKtNpcBFA%kPXZs>=sAomw6ZswKo2sk= zL0y6~Q7I)Ykz6`KbZdXYXF)3_yoT5Rl+{0+Z2#WVzs|t-y~==Q? zAce!FQKT4|W_l>t@Cc`yzRI8RZK=m9iE-u>8M>%$c1|mzzm*NM9%-*ck7pKEDt1h!MsWNCi-|f+dJA%bb)QCE4v`b2y6e^dp zbo&dT{3OXJg#9jP67K~}#~2=~xA~-uivkY)bQsNtAGhjO#(@gDY28-lH<(mW+P>t<*|p6}Vu)Rnin0gT21_tg{=lrB>wLs{bIG-Annbz)qf!UYq!gyyf@+D#XN+_#LDY$qr zbNdmyrFQ2_xs-CMPAe1xyP$*XNDqzQN4TqokoBORu;k=+r$&oa5&0iMo7aF7K3pSw};#FHD(>UJv#Ykqk21 znJNDHAY68XYccG*V@8(mU4H+|l52$&C8hjj0ddg2dFZ)!nJZMj+?_TAdi2hklPPwH zRon_+xK6#?Ip=T?Aawh%Y`EJrg0a$2D#*LPm1iTw z<%*|d&?2`N^{4D_X-qu*D3<7!s=gk#BmI2Mm-w!DgSEy0d+ybWW4VxSMS#V(c=M60 z^^Iw|Irh>#Wg&WX0N?*Z!*3my~(vHGtSM`#-YH!ddO>~*SoY^^v0S2QMsx3}i zgdF^HMWaIFQ%!h^6t+jK=E~826k`J)<@UioWd2E@u%Z{!T{DuVUfqe8q~QzonFB+p z03URUAmyT(B=onUVv{~)qldz4Rf#S-C5Vl^zft+7X4LleQc-h$lJFJw@Wq-^Wyt}+ zS;Otx!?V1ECle_IT#O$DEg;e}>j~~A!vPB9b1TMBsql@VdD&uxuI?5ks}@HoR$U6W z_a|=IwkW zrBg%B8b_WVo1ul}JOo#RcNa3hXn#tNPJb;w(71KpmHsK8dwGP5wX<(p<;X7?p};MA z)n%;nIKaK1ajuW`p}MUERk(;p0B)sc7C-8etfhPOVJrT36-JnkB*|B2x}G|5Dr z!ndt#lUWG@#1DIkKjy^_=3Q0a^L#ewOm8LPrl%juG2-n|_tEjiBBDYN5nLLhNQ(#4 z8of;9H1qfRld?lS$b3US-ZbhBv^tIPjm~j$@^=+jg6Q&lv`aG@+X61RI5$6_Dan(2Wa8dW`Bk|KtFyjM;uMv~afqSwoWdkIVUls{`VREFbF_C8OZ zDZtjisgLS;+-?}d9gnJeLzp;Wv>?Nu8@_A+#2;zYPx`l<&s6%JkLN{}5#^y7f(Re} ze&6(+V{j|`v-i!GmkfF6OF%tU*u0Aia0RV&sBnXT-V;Y>U&w3atJM<#>lTt;UWnKMFy9EBK-ztF_%4f1>3UAg%WLTPf zZN++u#JG?!AAtf46-c2_CkMK=2r<^(M1sxz89syIZsAh}L6!i#LuIz&OZEcmBB(Dt z0oxV%TfnxlCOsx&$R&&#FCZRx|0lF&HCBG2G0plW=y5clhRWdjD<$6fqeSZ7=Ig8H zWHxr%)4l=w>k(C09swzRTP={vR~MkSy2PAH?$v|AcjyW=9NZ=WwFf5nE|?3!J8bp$ zR@I1h_xGl5M*3C{bBzGQCl|cB_RbKRLacJ|MCBge4sls*?sLzq8t{aEs?tH6IPC>( zdrYR`xMQ8^ikvVEv^(xf0SQWf73~$|N#8TFvN-b)0JJ>Q#CVyoD$;a%px5we-mL+- zm=FzgpPrCZ%Uv01^O9Qcco*T5JdQHvANdg(Fuy1z_hOkReD7qDEYuwi#9(@7Nc&hh zm1kwLQ;2=IdA(Q%2!^Y28?Y1bar3Xl&S9M~=R!MbjHC<33aaGeP}Vt{_4WDo@&WN? zLs?F!5rFgBHznUPjUgjIn9t1%_*ldi;o3TOlSg8+_+|9uyOCAf2ZlBhZG(#bfhX;e zY3k&)T_9IApSHh3|kB?|9ve5yK&b&Z@6V14Y=XZ{@c^gF{3gxkT?6)qn+2G8s-9oKNyIVbHx7 zU0_2?svjAdkf_@|SZMjcb{F|;9KI@OH!s784K6Rj!*DQ`w6Wfw4X2hfH{;U$d>BRX z+Z{Ae;M(H1A!-b$fRe9=_ooc6Obz;dY9`X@Aq)d8lGQDr8dmjeY7Ubhy} zZwHA{U3-1?B>Jl%e|!a8!z${i;!aDIya2PWMj{m>nT?ctlvVq&s^*ghJ0NIpb4aC^ zkvJud1vo35IEre|Oi2q0;(JX{z$?|0en=IAP~+x7(K85dN1UYJZ^h4}yscLf>S(e% z0!^9P5wFzZ1*Ro?w@&F#yan=7N#B~Sz%OgPBW%sWh3fMXnyP?@_3WFxu`@BfgRDM! zfi|;pW(04p5F0^!nF11YPdC3nt5;r`jbw@O;}YwkQ)faXZ`!xjd|}1cW}k+^`;}W~ zldpEDZDu{?;FuNT<3Y=}?~!bwg(q?JGVH&Nyl0s{%yWUz$^8z;lL2R`i~?ODTP|wz zww2-2(Q6Vi)3AxKx>Bu6Mp|q!l*%79@;(9Ycg$7ipLttJu=acaU-moo2aNMCZlI1L zHqq0qkqupc0R5noC>%kU89ger9BZh1*heutM0T>pnBc`Ujuf5Qz5QY1K%YLemy{n$ z{nL5v4q9r1zH4x6p-;wCzPM@#iOzj$9^c3_>+KoAa2osa2j6H4inbXG-R=99I5r;G zi9dO#dGs-NY~9tI7JSfO%U0+YbKcdm zT}G|k|6tduf4-Z16DxkAdp+WHUBsixqO*~j2))@ud$L4mT#qd_?-}wax@jWlIQ8?L z?wbKm+VK|Qke4hhZV*byRntBC82ae1^I@gM=M1X?qkX@ZPJgnD_F!lqH876e-}R&V I_dk#S0@jP^!~g&Q literal 0 HcmV?d00001 diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-v1-snappy.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-v1-snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e0d18ce38ae212f08d79c6ce9ceb7e86381f9b36 GIT binary patch literal 8042 zcmb7}RZv~sx~3QI?h@QxgR^jF;U3(AI|O$P1PCFxySoKP*_5eSZOHA@P-`TTB*jmV+!G zt2}8g8in8av!6X{HEm`;WT#((mU_sDwT-)zFX^q<&G$t%%%6!`80t?4=>9TP9wZv_ zPIb)C`TcH!vq(ByU&hUVkCE#PlffGCwwEUS@{3$w6++$mop%ObKl>MP9}50 zW%U{DM2Sy}>9Lw*f_eqcdv!`?klaN+-CAp#YYi4P$v+jOTI}4|nvRE1Myh-)UgcD3 z*OkBVA#||Se6CEGn~&fG zoV#@jL@jzWjL4rwiE#UP|B&|%?4X8WS+%`<=7>HB9gP0@m$x1Xz0j__c9N?`CY~S^ zfBOgo7K3)>>WafsYyutEiCqpR>6Sb!z=^_^T@ae_z*NTZBTar%;t+y({5iasYzn1# zzcH+I?*8**1=}z|k&|27f`^aerYcXJA#Du zX|b=b+%(dwM2vYcDK=vEG|%-PJt6rjX(d6ihuJySJGN!Ag%YD+7yvPsG(Iq5NLeV~ zz`a2A{!B6ed0@Q*|EA4IkU`e;o#ChIP3PYPw!Wwu4{{|$koOOs@Jt>;Dg({ExIvd3 z+wZ_$+QOylHIFL9{l2{!N+1jEl5ie(8_0*RG!{K~X@3=58NlM6h~jTOG+B zY;0cNbIw=bW>hdsljnL#ngqh1uLFteFVW=RCPx=rPzWko5162?miM1IL4JVtmz>iS}J>#JP*=aYs{&MsX-bk zCoYINIwAVSnqFBimXnbS4 zNjA3AccG%s!`VSCSFo9$=zrySdO_v>OGNey|LO(x*(DhuZ2$ouX34VwC9r65^Z8js zMO5Wj+g{X&HQbEq%vsRlJB%p049NTr0(15~3QQSkDle}%pL_&+HPe~hNL2T*c$(y# zm@fz+Z6}>{9*fKc0xh!$BdoAGhX8;T`;z4+LYF1jW3T%)MMxXZ7JsV7f&XxeYDFw4 z+2ZmC)2>wd&8?g04>Ed50*pcXvop+*w7M_QjCY{`Dq;dtmdOJ#QJWW;_z88`^@l3e z5TgGn$q@>mFQDj+;)2bXQn)1X7gC2s>&An`k%Qe?bD$PbqJ&Xs>_wK)3h>o_Sq(TRpSIca6C}?TpbIaC;y0Zhy5-$386uE zX=B~c!Ioo%igq8a&G_niQL}6UXsl>-f86O1i*f#0i{6$Iq($`Kll9*N2KOH2>(ti} zlo%Ml7mywL*ZTEq1GDwqYIUTbB+?-9RL&|@`DZqCs3iQzX-52?H?S zVq&;57wIZ>2pS+_(NaLV3`fCYh8Q74qYcY& zsIWNtO-PV~9z;s)Tj8A#+#AMiag7yhGi8W#?yKJ&GcfVZOF zL2K>{!zt04_gVYhzdmRYe~BO*?2un>ZQ$6K%vQYMy0l;9581xUv`z|*WNtjGn1u{m z%_;oJempiS;pYF)fx{#;zDIkx*-{ZpAVL?$4z_ESx+K`DRxv3p7!)Rs*LemKRLtig zJyT_xanY^+GEEbHGZSV;>=yYs-3U(ZhQimT!0h7O*YWJ^A|pel`$nc@aySo1Gm1nLd&qSwFXg^b0|o@)`9C$j8!1DVOg>^?Qq#A0^T+ z(x)k1Cz0-8N4fcy_KwznXWkTL)5k8d+HU6w{Z&y8fGTJ8v3f4fw~M8cQhtDfwv{k;7RDxg3zyUV7d~?UDTIzdUyr}Z@KwAdQ|*^jmBsjU z;}h~}2iCGsSIYF4q#3FC%CYER!-w4LN;g`twVMcjt?#;osj%V3)Zd*Gi$mAa7+>Mz zB=`Y?$WN>z@oSb)|Gggz55h4Xu50p*^%~wFPM%{FS$;NdESH5nm=A{rEREKit#qK- zMT8b)ftuiH=r%RYd8RPtZ}rYbQGr2;1}^A{@%Q-y$xrTUt_Wl3``3#>BF@>6o$=J} z!A8W#DyLQA<81?rDLyj%la=>6&@19UO!g6XD)TwEOmy%ikjhH`v!B4xD;3)%HRCR; z=p<%RaNTj!F%e0*BM^bfBdkmYbJ5us3)hrr=rO_$#S_O&w@I$aAWf7@!~u42gIT8-_*aikqr&(FzlJ7xQ|2heL2raeTy^ts!u;f zLKG$u6Beo=nTjGy!-~-0g;^tM+cT#t_TCRK8b`R)rxCQ%3llyL{yZ|%MbbPLtu%L= zfujuKE@`D!D{$jg8p3Gd+T_*dE+d+8eI@;{K~O}R37`P-t~hf+a5>x zKl`D7%WnX*VGySJxXGi}=RvtSgl3j~7z}8S-WD}C%cr-!ubI-!!!JA~go@pHlm8C(TfA0qaf*4P|H)^CXr9zeL zkD^OmXz@?hU1j};JD2j_{Cq#D0QmR>I*F!Yff%cL1%Hs`2`4B=2Fhcy4h*UO)TG~~ z|K$fm`aSwM497_eFC?n2SV3T&{b3e+#)WZ^y?Kdt6v>a9{L)_H*QlM>-IK8O?b<@T<$Zq%&6)DHYnwBs@8vB&|T&NtbQsoF_W} zXnTjKL@sk|b!Ba)S%k2-)e<3^YVAbU)w3bZ=S_UsEr(h*SVt>$EyQHL^_aYwn{MxKfT_P^iq8a6%i(RDU->|n8R8)eagFbB1J)Zf{$(! zgAjS#6ls$HXC^%R2Q)JB$r1}|U_bL`F2We8yy^erU~UtB`r3K*c+C1baa7lNFHkl4 zx^?ysKk*>|0LZXP9#Bsns?#TtQbXDP?XfKpoJI@1F{lz*t==9hkf73tf$1$lrv9lk z;?2bDk26(CX_Um6!^vj2*a_m9xXR<`eZE0O%!j|puVrM^mLC^ggw^@RgIGcWDK;Z!=G&QUw_ zor08Sw~e|;)bzU~xEijlwUwxs&YEWhz_*`O{JD#YRudTQ7tVd*t!(Pb_jm5TLi~Cf-ycmvYG@k=*eVbT@YXs%!)uoC;t; zO3AFTTQY}X>hTFG-}k-sQxxJGAR^IBMfk+|!^Ooce6j{T6Gc@;ce=}8FgLz&0hp|~ zD4i1WPmceeu1&8pNR9QK1u%A}8WIMzYLLcYdO)xgf>3`H71>sDh~14tdl zc#wqOhiygB#_n?IKR@xnYGBf{3PNnSj<0pWYMd__mbKW=fB$1ardV1L!JV_z;gokC zYkz&3g#{s1p?eS_n?Zl@^LG>p_)%mBUeW39xJOnE+dQ;G7iP;nh4}iz5ZTX%WmJjA zu8RWvW1ab_uZkkbTqekVEwbh-$WeH}9DuB1r6;mzY^&9_1Mx@vjy)+y3-if4IxE(% zADni|qv6I1o}p~GINnSU8@XDHqG9U9W+NO!5T_t|fr)6AdOE7ih(*UJQ%Z;NE5o;D zMN@O=`&GAAa4$W*I8bb{k0gOrK&=zY#7~ZpvM?Zf*Lm?XvfCYbVg5NVx4#Hr5;^~8 zI~n+0`#Ot#+aOoENLxnUIWCo=nebN@T@3{E0EOt~gD%d483y#KA+@^3Vt_a|I-`$X z+dv7Y{?_%Wxw%oMA>Tern{HcRAE}W&cU}!Z0Mib^kp70nyK&XxmNQPh&9#tO8fKUB z3ME(XOc~U;=S?z0sotS5Nvi5+w|lzB zY|v86{!R~auyxC4Qct$zZDTg;o0I-Aj8kf0!LG%#o_8d~XnXouV$9)+2OO&$)lFeJ z;kFkqoRN`cohyybD8qk!7A-|2V@SsA6+KG2l5Qmkt!@)cm$&p*J5xwr(QblU~ zND}YHYK-zz#fP(yQ|>)|iUOZZ__jvZ*|9H^G?pZhRp8qy@~0rfrG*UDqN3vhDReze$37P%I3y z*l|;N-$rh5(?s!Mk9B($b}N1&35RH>!HdDZW5xr6;`cn)n|y@4LIw9>I3Y(rh(t!C z(KJ}Pz-=E8hq;+|6F@MmgC#@^Wq0D-6(EBWtB5a+z)GTV#I_JziwGAHpHyYIx%f}U z|KZ;LchmQ9RZ=b3HT3NB>!+~Hk8Rc))F8xQOh%8wo#OuVhX#V{&Y>>}bOwk@=q_Mr z;^Ilm4K}iSG_Ox1UL!-?;bWR2kF8e=GUDVVeQweBBzl{N&`%4BrIc4`A5wDie<+_{ zec?hyfMFiY4qTC?Q+pHx6ZUwclCwn*n>t95MkBeg=eRd0Th(6ylM_Kt!}G7 z)l$&b2Qa)`2@Rz6;gE%{FRz@%!rO#D=)Q{alVep*ygudG@=vfvb_gDI6k>4{g@8j_ ziA5;lKQbf1YVgAS7UD0SbG*KRUf7^ZfB~rwS_Q`+D&YDn+Eg6U=5FHX1=59pJf04-s)@&YEP!|Fdhq)X0q1=!XEQ@rUDXK75opG>F!h; zFn$OfTLQA_GU@us^OkHMVIXAZ=$VWSqX%GGqg?;6UnlOe%7f>Jr&)ne#G&v504pc* zfW$}sDA8EhORCW!twuKh`W93rQ$Z`cKXJcmRwbFr4pTF2V3#N_rCm$<1k98#Q>ok( zt#jAB1cQdkS@cfK0B`&fQ0zm4>hpeAJhwqQqnL%RPEzu^cMl7QF`1{cA+Qc9VrucO z>h5}RoPgcs_Xy#1C=O+aD=O^?%?R!)Xj`G9cOqKFuC6gY)R`;w)vttV&33s>qLmSr z%c3aC>8;ZxZ!-1;QOO4ByUN?1!@@3gqdS`U(Dy^u>Wx0)w+*XC8vLQvu(V9-qVp6B zvIX@-p$C2L%bma4D=8hCUarPNT%Ddrg*biF4u=%oMp{$?@>Ir6(Jg|@YpqZpteUJZ zr+ejSb4n6H66w}Hv5lqGnRe@_&1@}$fbTtBmZht$;lSjH=NMnTBtVQgbYd77Z?IVR za>?as>y5-ofgsm(f=RMbv(88QB_nTob^v-Xn z@)1lQy{fIjHcfamu5|n1s@!L5KBal*NO1v-Rmbl3Bxm(;?b3q_@SO-Xsvw`*;g0f5WxpzHmz0g^WuS8_tn` ziF1#293*ky*VL6gq$qdtx;(asS|@fc?Bx`M8{MvjAuxwqsC zKKd@$h~g+L#&}lo%3Pow+^XM~x$O;;wSV-HjrhK(2;AG<(yv^(Y$WIsPGd9KtS`pZ zh<`k&<`SdGo#0lYergS|G{jJ^ypjqzeMzufIWT_dE`7~%u%D4G3MNV_C1GClHG^*& zAF9Rne_O@o|2hj`a{ANldoqr+jrc0!$QB(q7&_{8$~8)sTddDzCRn;p|6J}R{L5t3 zrdo2<@=FmzS1I?vZIWXbY4kV0kHZHKe@t_c1mRc-ZDuv8fSY%CB#Zjzd4JZFjz8RA z7)1oi=e?#M3KNCBwv)hTYE}vSp#isRm@0%rdt<-n#LBq;XrRM|DDq7!VyGOiTTDTi zUXVb0@moogsTSqjB=bxog7A}kQkNuj1!%R$Y+#%`F~haU1Ij*d)d_=KCS9<{wE*pq z;Jj@Dqsn5PvJ6#=Ja72(#jfVzLMslkv(#lgT|5v6HIF5|L5*2Lfnq z`(&t{hhP4PY>V!$x0H1>W}H!!?G(Uit)C>ac1x|EcdfWBPL*+|@+Zq-=}H^(TPYZ0 zk%N2HRmSTycGt5NBv_}@La$GdXR>q9XF;)A00{I%bViC3ETFbjNn&{VvvKf_Q%-Em z=KDV8P9Nl{-kkXr2I(7R6+0Ko?n-PmJEr>-qJX|pV*IK{*_!AHc zAh;iJv@|aM&NDqFKYgTOsbYz_;Rk6lDxqfSWrQ(}y!U z^6We7ZI(t{J?!_MEQ}~|CQZH9YHOx;vW!i7XSkQTy8Auq@U^wqe1VP>QP?>dJBt3H z(t%Tl@Sv&E7Rrb8&x&XF#oA9`+1q@MLSx}slSdm$7XMWLU~du^q8(?Z!GU?#q9=C! zf@!963xDK(Ku*sKE7YB^29wq}j6>+@^KN0Yz|g2y*g>7AGnuUNCn{^H&=LP%yMIi1 zG^pPh5MP5?fNpprp>4rl3)53|6%3^-`I5$?YmhPl42!d`Q2ku$6Y4o7u14)a8hkeb z7?bVV%^#<;8`RtGMAAIsLVJQN5%>)b$F!0UI7VQLa8r(5e{S;Jqgg-v6pGE_l5h4( zuZfTgMJRv_Pf16*H+Lb?#Oo;TEw=UQP)2W|d1t`s1Gj8z|15Qu)uQ}qw|^^bP>BVh zdf}{v`p{%wPp92()wC~|2l1VxG5Ys`3blbeg89T&vCSAR2_?H|rjE(p7*ecuk@`&L zXX1D*r8QP~webQ3UdJcN+Avwb>o`gD{$+p}H9a>yX|w75RN8g>=688|LF?YEwUEBc zRP;?^VmkO}$&b6lrSSro002Img@c2ujkza45DEZb^(qL@*CS88@FQtiPJQi-C=w!+mcou%Z`z5r5R{05k8{DW%}dfy@;{l(c+ve|2OmR?~?GzjZNN2ZE^CFq7_yVDQ1h|7UOOv{JOts|USAq=_N_1PtY1hze zBxrG%n!305L;t0Ybnm&BL_`2eAp{`sMbJq`aXIICG4*Y&HXCc-{Tb{wUGICU1`mO) zRx}dE03Fe!biWgMy2_>2wi;J33XeV^ak}*j_Z9QS#Vmg|{AmPIog(bWeY$E$@EQFD z=hSAJ^i#5`>4`YJLdEqabDuu$){oaw*NpkGZ1H%yM|qA($vpKfx8ZCU>*Px^Ue+el z&ns%Sp6@!!^-i8=lDtN!bTp}ieKB7#RL=(dxcaRmUz^?r|()qhT%70(k{|7>=nx_B& literal 0 HcmV?d00001 diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-v1-uncompressed.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/java-legacy-v2-aes-gcm-v1-uncompressed.parquet new file mode 100644 index 0000000000000000000000000000000000000000..754253eb1dc08943aae5c4518926191e70b94fd5 GIT binary patch literal 9004 zcmb7~Wl&vTm#1+F?(Xhx7k3Ztt`~y46WkJly99T+T%6$UZow0RyE}Av`<qYEcXWndOWx5ODsqsl3MCd$5^62HNxR$`%HKSG<4AV z%CpLxutybAnhB&st>6~e-HeY}%Uw+94a!S9X4EkBO7@|wg3x+Zl(Ka`Aa&9HWvHf_ zq}!#|(|WD3DC6k>jy)$(KK0{E z>B1^{d7Y%7E58s+2An8rv(~kBP2+sS>>DmdArtXzag+A!b@@X3coZ$}r$RW<6I}Yn zZ1`fVDrfoN@HJlY7F(yx@)xEUfh{x6ug2A3um(+|@q4~F}m@%x^&lVSsX zpX@W_^zDGE_T>$q04==W86qa?Ge>s9cn9+Q8gbZk6XAD<))P{GS4QPhnM-$i&)hgj zruK*+Q_p?-5W!xNmNS)Eqx*mcdoAe|>XqN9-KG``q;n~R@Igc?;9892_=KHqRvC$KX4z-1w-}O`x#D@6X_kdalH^yH2xtcsWRXHGc9RGvT_R( zeXy+-ddEHKYg)M{ z&?^;Yh6$~spDZ`r6s#b{84Ng>HQ<;<;0#2t>U8yc4xEas1y^fL-jlSc=SF>Qqt)l5{*AY|g;BC9MI$m-(q}O)dIWvM`JCm^ne(50pP|Yp@BGBV zLF2R#C5d%h4QC)|6H70_)74)Sf|qkKunUFCy50E_p4P<;^_sI>o1Yp_k^Z(Hv`P!02rP@RVb5Qqr%d1dM{C-f>CwR!U3Igt?%J} zM{Fx*DK5H|x3)dk#?gPA9UaT2@SJeOl;`)STo=E5M8sI&M zxNAP~Wz3F)c_WsFKI_;iZeJRXKEJ`jma)MB6LC1&sg$h2&~KBex$n9mpK z@!HD&f#@)zfaBFZX(v44PI;4jCK7#CCi+jO{!=aZ_aI3E>>hW`q!aq*AqfhqRbl1~ zHF_@Oadl^kgWu`wOq`Drb`6^-=byC4$**o?u;B&}z2AdKv!ZFnId50!uxZts%Jzc*LeQ;gx~;o}QD>xcawAR<$yk^VoN=d6N8#q|T~=*vwvp_K@3Af} zVI^PcFWv7)^naX?CzSUfpB9UW0$zuRP@CwkJFWKjqPaSJwi@Zd?uBkTxH;cQTtc!3 z*e*0kzJ#abJywVdN!1O%2l>jJWFKpul?#UmD;8|}nm5{gyjLS=(>Av)oO*wH#-5ypfeG>A6*&N}Y?{Xm=i zOGE{Qy|wO+oWTK>{Y>iq+;+agwUM2$2DZJcC2S~k~-yv50iQ!(T1F`LoHfw0vtxFdn)#PS& zFdrEwgRy*O$J-M^`>jy8Qu54^gsGNR!avHwV}CGQB{bb#K_yrx|7vJ(Uks-t4JAL$ z>xDg-+Ypb8rcXZ>>@L;nE$l0K=a$#p9oAjyO0y*=o$bbHoPncEsS| zy~cx#`p!FesD61w0>U$X6VPe3g3Bk>$fcUonfwBfAOlOZBLZGy3ej3fnP>VKC-dTd zC2>WfZ;I{>%)CG1Ox0xCt$c+W<=KW}*6a~^9*@6K)PoApo3VES4+UitjNKcL?9$Y4^X^T& z=6J@A&OxGA=&8ruRUjZn*5sW#8oFH(`--M79djL-*0~LcZ|4Ed`4NvJOle$97#gTU zB&!(8Ez@&liqFI92%Vi?W7)8@5N=el>|;I>lYJ78NX6%*xFruRAPy(5Sb9BeqcEH6 z;S7?G(9j2GF7{F`w~SdawF; zVenjg^C&c-uF9^J(mt#ghcM3dmV=*+Hvi|_Hw8M>+vxG?Ypd03uV3LxX@IEk-Ur(? ze$2FThUiFw)fjd@nyD;rbnn0)5J*J=vtXVnQC6q7^e*K3z0Zhjy4#K@E4*GPDCC?4L? zq-(&1vyuD`ZEZafg0pm$Yk0~dq~wz^zx@Ms4&_YiF(y-a}?_&RbFGKfc|H7jR>E8=8DB=N~HtpL~`2V&ORp+cOGjtK_JOBse6bmE1jte9E; z!=OT*33;62Ct?+vvMiLml|*9i@!MAjWV zUg$Ql2cR&RS}6N8v21`ihya6=S?-I*vW!}?@w~qu_G2$(+~o{(01rJifv{|^6~#hy z0{a20pcwizT`2^L5Fr%-YEJcfvV>hYpZ<4;;we4L>^~3Mc9fs7w;5+6z)&MD7ukBw z{sT#&I-rYFUwMyP%F#}WSBHaoc0O`rEbwPA-4-j9Ry052d(|xh*hiRKOm7F~>Mqr? zs1+A_@5UX_(+7NDS&%G6F7iM6yKiuNp1tq_|@~i2+)R69i{3^0>6-Q-= zzSr^HFn4(9uS(|e%H|jyN?q`NfWMk~JuS!ojU+UBS2WtaN~td6O|S0N>Hsf$$ek@> z0fCB3C@dA_1^RDYeCsb!0DD%|7?>Oe8{DRloB0Sx^j(c-IIR0o_^XF_;cB?2NmqzI z%7XUG(Q^Cuc=HAyz98aGlGhZ6A`@tgvN#FVlkD~Kz+X#5t6ioyqR77_N(KL*rCC6K z{bgZ|?!Xnbn(Gn52PGpf1|%*~Ny#&Zagl<&rIeKp8@l5rGJldSY7T#Dvm3{8bxlLS zSiUKsr0n!eOLUao2jDthOQP&R8(~A;H`8<2RfVnYeV&14NEL`EcPi>sl-fi`J=zXT z8OxDTy(8|&8{v`ZU$L)SSRbJpbqw0L13IYaW`#Wc>!ayfg4{!^*%mdilkAS=NuxCM zV29VFj@)-fggdIAyL!7MwZWi_G(d#%_rmEu#nK#*Z8=sLm+JRKizH`VXBIX=5aQ`* z-F0i?SCS8`cnopI=j}tW0scwSoY|hYNEb@Hgb>t3*S>h<^8KfH{Dgn)33Tdm)dTud5t@5VkU z$yKMqi`PhY>SR>7QZ1X9iSdi^9y)6|Y&rvlJ08 zo`P{Q6Gy~Oi_AUysS1|&zGjy5Xu)81O?Hs|@ap9#Kp|ikEHq=?SpHaky8$|A1_6v+ z_mWkmt*uyC=wP>VGBB_lx*^jo-ynO!5t+P1Jnd3{Rzme49i(6Y>-Kq;i()qs4suu2 z#WuWI#tW_E>*Uc;3so@+`{DG?o`irzYB;`DNghLtFJB9e!x#}B6iv&pRsP&kynVF) z2|&YX&hnEjn7^iIr`@H?$zitU{g+ZZ4lN3?V`?F~s5xI79#eVajbZiqHTQP+id%!x zA1sMs8L*6NT4n5gn1-J?nR{`q_*lrfNvGx5W~*9=Q@?OU)3kr`*khK%D}CbuYI^B0Z@SCfUf)VRw`jZ3WjlT!ar zF8$lOjK4}PJaAU#3g;73afsKOAil=TctvZ6uXUp|OgFf9qx5^5mBWd}ViFF)w%y_> zT|>XXIMVjJNX9};oP`s?**^*0$n{Ce8 z3CEnUrAlD^)N^MCOmvtHGdCjs;2m^2eQOY{kdT}u*?R6@X(lohrQf}`)xvZ@tTRc1 z$abqZrTa-BK+`gZtyE zP~WG8-WTDI$!%|uwADs|?#NT7uF8=t_Y?y=x~zR&<=mf6uk{?r;ZYaEo4p1)CTNSRcW42i~>d}K0I;d-*``i?-mcCK?-t2WFV^ND(F%SkP_Pf;S&#m zB%d3>mHxDIsiQm}l=nx9&UdpLQHPCzF|0LP}y;J?yo|g4jvV%M4-W)T3Q=Vb; zS$F0kGGVeHZ~a6l0u5CKp7{G~Q5!eY0%Eoc zBFk#~w{|(eDKCW_tosG;mQY*zS{`EoJSU&D?xPy;k07qpNcw38H!)hB9iVVC+qW8w zG#O^gxEFk3JF(SXMzrGBd@Jd-_QElF$nY<)3#uzlWTzCmc- z?7b|poMM(XUSSz0a(#!%xkf@&kV?^P7i~)6&s2KL){Zj3o23Ew15oz)$OGdllGQVs zCyX!RXV%bP@Su^Dnwh*wnqktgj^X@LWT{*hor6F1`Q!z;x8`7<)q!64T)eV-BHYZz zJ=-b`m_f$b^CL_fv(wvAQMHUIIHiSDzG!Ww$J(1?Q71eH9EN|qz(PvR5PAlWre9B$ z`^pHrZc`rXts$^`@Fn62EjeGg<{)DWmK|x$NuOw*1Dk+vLL#HF@KB@cdK`h;7CA1k z0L_ieB$r&8o1no>)VRlz>*-Lw=ow7*%ephzsBn}m{S26^cg-B2TpMf&`5q3jT8`dw zj`)Wsw8cqjuT>f#^r&9AhUD>MeCl9wVBf4?8MAEURW%b2HbeLT(Eu2EOCnvFnDv<} z{DwDpjo)9Z4`qbHS}JxFBXR>noKfA#C%GxNQd(XNIYYVE6{(wVruN^{KClD#G`mWB@Yx_&0b zQKs9M%=t-gh~!_;w5bajXpS)~ihR4d{bNSloMlz0a1@^borEx)Ia~W=-KHDl+{lsn zL2CobsLxcfxN)H^N4Q@=mX7{3v@yMNZZ~NO$&A6OP!od2%{j1hF)x=dnZD;^0zs%f zM;#k%y5Fu>BC3+Xu_kJ!KbqeqkHIhw*Y^rud!ID0{U;QmNF+byZ7|>B1(m|VMX(Cj zvKH%FIHISZV>gD=v+1znki=${7Iy?NMw!B4F>U-BQF>x7{|{!rnX!DUNj2foJDnw z4A4gsMmH<>4RN1)M*q1}s(g?XGmx=m0fQn1<6?XL@na*jAdFl|z10>sPlZQ+O8+SO zFTpJDeOlD~guOMFfzd#&KLaGsSmR1HqCIO59&Gpay`MXVL@*63Q1>0Y&x%CveWuCX zD8kBclpBKz({d+bPkAepEb-nY8*=X(INzo(0?Q~F_;V4>z>fKm@uEtYTv4IpmvT@E zk;~*{?U3+`y(N>7UG#pZjGaYaX8QA0m4U=MyyyNk4KQ@3fdlF9*;+p)uk$u#4X8UN zs)^(WKWxxqd$Ddi79j zO}jvz`3<#R>*ZN49VI?q^%x02(vaYk*A~2gmS`4BiD*l2g~`hx<6%Gs8!Ox88*u$+ z^6s3ii#sk&umf2r94El6a3+|p;>;deqzxhLLx8KbWmGC64o4WANbji|Tj`r2l?m4w z@u}qaRKXnVqa0>kgJy-Oyb9JPWc#@4C_OC5eFzQasdgJzT}x0s$T~DuB?9mZtwQVr zn;T+&F<>PC#u>Y8^bARSyJvDc_%oLdD~};P(ft7`iQfHU_ZP6R)cG)d^^+AjnObMu zmo)KjRt4uE{a>QkSZ&q>U&2yAhr`J|FfUFyiRr;i(DBGh%0p2~VdKD}2L??U*r^6O zX_CXln9FZ90b6*#@FJ~0sN-eONBeWV@zx3`kh=-kJ{_Sk=CpL0Q8f=p{P`;x9|-cI z`XmN{8KD!^H@gQLkhdOXk7?wb**cKxRQCfV&i~fp4#My$Q{O#KjK1ulINQ8S3$>2S z0m-SGfFS2pKbU=+JaoNoYe z|GtBah)(9By-N-?OWvRMt7pW4!_Q$|nD!KkxCknVKK+gDO=G|2YaJclYfX+jnLq_^ z8BasbDs&}S+&rshHlmVXODi1B+3t*9HZ1ogB+I_4OUfY&Mky?M$?TI)gsnsXb$=XQ zPZQb?w;>Y?0RiP)^c8zkLBtenlqlFtBO22`XzF$>zhZT7f4=fm5&RCzUF-gM)juUV zJ+;qdoA4qPQ0UUH2Uqx7n}=35vzBU`6@(E7NOgwD@Zuoid2zhw!+L2f9U9br)3mlP&DWe z|Kmm8%hP({%{Bs(5+~7nPD)Y#G^4G!z)SW&KKK!T(o6-qK|uFn=+!)jb6@*E}XnQ z6uA!?CqKKKF!KcUmNq(+vwrlX#qi|@P(Vsv>ypKgcj(9#>{{?YrhU^4lwAOX9Yn#S zd#iW|t`h?|xd(IC-o4h$@<^1#NU9uw;Nv{LFaGA8TG$Yq3MazxBO-<3JEI`#no{o@TmR{b_38hr^V zUc;WLs27DNr{X&d&&v%LFg>fU%-6E~{D7&MhZ5)gZ%PEw&SLKjk{IUM~BkF>Xs&X&9V19O{j9~?`J6F?U zBkw6x(bynY9@g-*zB4Go&X-6#z{!S>MByh7JtHxuOP0l=>g@^qxpqk?83grZ8J)0~ z3eeIO6?;$V^@q)np)0E3$uowMn5MkZs;%1P8%7SK6fL5S`y0?2e~iY zh>NupIPS)m;3_<8|7KLnifi98i>nb)iL8ty$>`jFrSmo+J#{4}rbk$+Y!1{~L0-m$ zfWU{hc5-sJv+{uuf`)(q4sVLZ)g3-zZ;T2@Nt!=gX}8@sAZln+H3bbS_K}m%ZHyxs zPibEw!JVn+i;nlzTTz0@L%6qREV|>)5!3Cwj5{!!N9U60OfpI`4`jZxr}!X-7P{ z#h5Z4_iR8I+1(3^!v#ekH`vnMYkU~|TyQy!QDEegeoC68@SQBJ@H}5zr&z+o+r=9b z3GUf|FLDa~1r9XPnVyN>32 zv&TAq(wrZMmDHu4ocvviPhGI z;pq!fw}Al7L>EDmn6m5W(e^fzWqyI7sm$UKCDqGg-OH)GZTJ}(vHZzfAkuk=;$xO6 z16x4JX7^w{Vq;iXhcUP}P3TMp_>IQbFYM1eYvnDo7B42WfS~B~#b+TCTvW?VRZ;eQ ztTX^zu~4`b8;_!`DB5;ltYgw1_VB4aE$Ti0)d{TtAs6J?pC&>=5; zKZIbvhmx-qV~b4xP!2XE27;S5`~)VKb{Zlk`0d zkw<1(P1wY@KLzzD5zvwcHJ3FzFfi0TgttA-DC~c~EKsGU6ssE5@WBtDd^#d`>GUvE zK)4d2J{%GIRe=n8UOAe7aj{{@DGk`x-&M6DD+9?$bQ>c~~ zDJMk=X!1==Reqk(H^Fk#gSX_o5u aQ6 literal 0 HcmV?d00001 From 425429f9389957ae593c8e5ebd61f8d41c14d491 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 5 Nov 2026 11:55:44 -0600 Subject: [PATCH 2/6] review: address round 1 and 2 feedback on DataPageV2 encryption - detect the page layout with both markers, resolve ambiguous GCM pages by the tag check and prefer the legacy layout for unauthenticated CTR pages - validate the level lengths of the page header - replace the public writeEncryptedDataPageV2 by a writeDataPageV2 overload that takes the page block encryptor, so the writer owns the layout - tests: checksums on reads, several row groups, page skipping, a crafted ambiguous legacy page, leak tracking, KMS cache isolation - document the compatibility change in the parquet-hadoop README --- parquet-hadoop/README.md | 5 + .../hadoop/ColumnChunkPageReadStore.java | 196 +++++++++------ .../hadoop/ColumnChunkPageWriteStore.java | 7 +- .../hadoop/FullBodyEncryptedDataPageV2.java | 88 +++++-- .../parquet/hadoop/ParquetFileReader.java | 172 +++++++------ .../parquet/hadoop/ParquetFileWriter.java | 79 +++--- .../hadoop/rewrite/ParquetRewriter.java | 54 ++--- .../crypto/DataPageV2EncryptionTestData.java | 59 ++++- .../TestDataPageV2EncryptionInterop.java | 225 ++++++++++++++++-- .../{ => keytools/mocks}/Base64KmsClient.java | 3 +- .../TestFullBodyEncryptedDataPageV2.java | 74 ++++-- .../datapagev2/generate_arrow_fixtures.py | 1 - 12 files changed, 672 insertions(+), 291 deletions(-) rename parquet-hadoop/src/test/java/org/apache/parquet/crypto/{ => keytools/mocks}/Base64KmsClient.java (94%) diff --git a/parquet-hadoop/README.md b/parquet-hadoop/README.md index 2025243143..11ea7b6b55 100644 --- a/parquet-hadoop/README.md +++ b/parquet-hadoop/README.md @@ -451,6 +451,11 @@ Note: nested column names must be specified as full dot-separated paths for each **Property:** `parquet.encryption.algorithm` **Description:** Parquet encryption algorithm. Can be `AES_GCM_V1` or `AES_GCM_CTR_V1`. +Since 1.19.0, the repetition and definition levels of the DataPageV2 pages of an encrypted column are encrypted +together with the values, as the Parquet encryption specification has it and as Arrow does. Earlier versions left +those levels in plaintext. 1.19.0 reads files of both kinds, but 1.18 and earlier cannot read the DataPageV2 pages +that 1.19.0 writes for encrypted columns that have levels: with `AES_GCM_V1` they fail to decrypt the page, with +`AES_GCM_CTR_V1` they decode garbage levels. **Default value:** `AES_GCM_V1` --- diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java index 0bfdc40ad5..0b09834165 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java @@ -42,7 +42,9 @@ import org.apache.parquet.column.page.PageReader; import org.apache.parquet.compression.CompressionCodecFactory.BytesInputDecompressor; import org.apache.parquet.crypto.AesCipher; +import org.apache.parquet.crypto.AesCtrDecryptor; import org.apache.parquet.crypto.ModuleCipherFactory.ModuleType; +import org.apache.parquet.crypto.TagVerificationException; import org.apache.parquet.filter2.columnindex.RowRanges; import org.apache.parquet.format.BlockCipher; import org.apache.parquet.internal.column.columnindex.OffsetIndex; @@ -219,65 +221,69 @@ public DataPage visit(DataPageV1 dataPageV1) { @Override public DataPage visit(DataPageV2 dataPageV2) { if (dataPageV2 instanceof FullBodyEncryptedDataPageV2) { - return readSingleEncryptedModulePageV2( - (FullBodyEncryptedDataPageV2) dataPageV2, currentPageIndex); + return readFullBodyEncryptedPageV2((FullBodyEncryptedDataPageV2) dataPageV2, currentPageIndex); } - if (!dataPageV2.isCompressed() && offsetIndex == null && null == blockDecryptor) { - return dataPageV2; + return readPageV2(dataPageV2, currentPageIndex); + } + }); + } + + /** + * Decrypts, if needed, and decompresses a DataPageV2 whose repetition and definition levels are + * separate from the data: the levels are plaintext and only the values are encrypted. + */ + private DataPageV2 readPageV2(DataPageV2 dataPageV2, int currentPageIndex) { + if (!dataPageV2.isCompressed() && offsetIndex == null && null == blockDecryptor) { + return dataPageV2; + } + BytesInput pageBytes = dataPageV2.getData(); + try { + if (options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer()) { + ByteBuffer byteBuffer = pageBytes.toByteBuffer(releaser); + if (!byteBuffer.isDirect()) { + throw new ParquetDecodingException("Expected a direct buffer"); } - BytesInput pageBytes = dataPageV2.getData(); - try { - if (options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer()) { - ByteBuffer byteBuffer = pageBytes.toByteBuffer(releaser); - if (!byteBuffer.isDirect()) { - throw new ParquetDecodingException("Expected a direct buffer"); - } - if (blockDecryptor != null) { - byteBuffer = blockDecryptor.decrypt(byteBuffer, dataPageAAD); - } - long compressedSize = byteBuffer.limit(); - if (dataPageV2.isCompressed()) { - int uncompressedSize = Math.toIntExact(dataPageV2.getUncompressedSize() - - dataPageV2.getDefinitionLevels().size() - - dataPageV2.getRepetitionLevels().size()); - ByteBuffer decompressedBuffer = - options.getAllocator().allocate(uncompressedSize); - releaser.releaseLater(decompressedBuffer); - long start = System.nanoTime(); - decompressor.decompress( - byteBuffer, (int) compressedSize, decompressedBuffer, uncompressedSize); - setDecompressMetrics(pageBytes, start); - decompressedBuffer.flip(); - pageBytes = BytesInput.from(decompressedBuffer); - } else { - pageBytes = BytesInput.from(byteBuffer); - } - } else { - if (null != blockDecryptor) { - pageBytes = - BytesInput.from(blockDecryptor.decrypt(pageBytes.toByteArray(), dataPageAAD)); - } - if (dataPageV2.isCompressed()) { - int uncompressedSize = Math.toIntExact(dataPageV2.getUncompressedSize() - - dataPageV2.getDefinitionLevels().size() - - dataPageV2.getRepetitionLevels().size()); - long start = System.nanoTime(); - pageBytes = BytesInput.copy(decompressor.decompress(pageBytes, uncompressedSize)); - setDecompressMetrics(pageBytes, start); - } - } - } catch (IOException e) { - throw new ParquetDecodingException("could not decompress page", e); + if (blockDecryptor != null) { + byteBuffer = blockDecryptor.decrypt(byteBuffer, dataPageAAD); + } + long compressedSize = byteBuffer.limit(); + if (dataPageV2.isCompressed()) { + int uncompressedSize = Math.toIntExact(dataPageV2.getUncompressedSize() + - dataPageV2.getDefinitionLevels().size() + - dataPageV2.getRepetitionLevels().size()); + ByteBuffer decompressedBuffer = options.getAllocator().allocate(uncompressedSize); + releaser.releaseLater(decompressedBuffer); + long start = System.nanoTime(); + decompressor.decompress(byteBuffer, (int) compressedSize, decompressedBuffer, uncompressedSize); + setDecompressMetrics(pageBytes, start); + decompressedBuffer.flip(); + pageBytes = BytesInput.from(decompressedBuffer); + } else { + pageBytes = BytesInput.from(byteBuffer); + } + } else { + if (null != blockDecryptor) { + pageBytes = BytesInput.from(blockDecryptor.decrypt(pageBytes.toByteArray(), dataPageAAD)); + } + if (dataPageV2.isCompressed()) { + int uncompressedSize = Math.toIntExact(dataPageV2.getUncompressedSize() + - dataPageV2.getDefinitionLevels().size() + - dataPageV2.getRepetitionLevels().size()); + long start = System.nanoTime(); + pageBytes = BytesInput.copy(decompressor.decompress(pageBytes, uncompressedSize)); + setDecompressMetrics(pageBytes, start); } - - return uncompressedPageV2( - dataPageV2, - dataPageV2.getRepetitionLevels(), - dataPageV2.getDefinitionLevels(), - pageBytes, - currentPageIndex); } - }); + } catch (IOException e) { + throw new ParquetDecodingException("could not decompress page", e); + } + + return uncompressedPageV2( + dataPageV2, + dataPageV2.getRepetitionLevels(), + dataPageV2.getDefinitionLevels(), + pageBytes, + currentPageIndex); } private DataPageV2 uncompressedPageV2( @@ -315,19 +321,59 @@ private DataPageV2 uncompressedPageV2( return decompressedPage; } + /** + * Reads a DataPageV2 whose body, repetition and definition levels included, was found to be one + * encrypted module. + * + *

If the layout of the page body could not be determined from the page alone, it may also be a + * page of the layout that earlier parquet-java versions wrote, with plaintext levels. Both + * layouts are tried then, the one of the specification first. AES_GCM_V1 is authenticated, so a + * page of the other layout fails the tag check. AES_GCM_CTR_V1 is not authenticated, and cannot + * tell the layouts apart. A page body of the specification layout, which starts with a random + * nonce, looks like one of the legacy layout only by a coincidence of about one in 2^32 on average, + * a little more often if the levels take fewer than four bytes. The legacy layout is preferred for + * that cipher. + */ + private DataPageV2 readFullBodyEncryptedPageV2(FullBodyEncryptedDataPageV2 page, int currentPageIndex) { + DataPageV2 plaintextLevelsPage = page.getPlaintextLevelsLayoutPage(); + if (plaintextLevelsPage == null) { + return readSingleEncryptedModulePageV2(page, currentPageIndex, false); + } + if (blockDecryptor instanceof AesCtrDecryptor) { + return readPageV2(plaintextLevelsPage, currentPageIndex); + } + try { + // Decrypting on the heap leaves the encrypted buffers untouched for the second attempt + return readSingleEncryptedModulePageV2(page, currentPageIndex, true); + } catch (TagVerificationException notASingleModule) { + LOG.debug( + "DataPageV2 body is not a single encrypted module, reading it with plaintext levels", + notASingleModule); + try { + return readPageV2(plaintextLevelsPage, currentPageIndex); + } catch (TagVerificationException notPlaintextLevelsEither) { + notPlaintextLevelsEither.addSuppressed(notASingleModule); + throw notPlaintextLevelsEither; + } + } + } + /** * Decrypts a DataPageV2 whose body, repetition and definition levels included, is one encrypted * module, splits the levels from the values, and decompresses the values. */ - private DataPageV2 readSingleEncryptedModulePageV2(FullBodyEncryptedDataPageV2 page, int currentPageIndex) { - final int levelsByteLength = page.getRepetitionLevelsByteLength() + page.getDefinitionLevelsByteLength(); + private DataPageV2 readSingleEncryptedModulePageV2( + FullBodyEncryptedDataPageV2 page, int currentPageIndex, boolean forceHeap) { + final int repetitionLevelsByteLength = page.getRepetitionLevelsByteLength(); + final int definitionLevelsByteLength = page.getDefinitionLevelsByteLength(); + final int levelsByteLength = repetitionLevelsByteLength + definitionLevelsByteLength; final int valuesUncompressedSize = Math.toIntExact(page.getUncompressedSize() - levelsByteLength); BytesInput repetitionLevels; BytesInput definitionLevels; BytesInput values; try { BytesInput encryptedBody = page.getData(); - if (options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer()) { + if (!forceHeap && options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer()) { ByteBuffer encrypted = encryptedBody.toByteBuffer(releaser); if (!encrypted.isDirect()) { throw new ParquetDecodingException("Expected a direct buffer"); @@ -335,20 +381,19 @@ private DataPageV2 readSingleEncryptedModulePageV2(FullBodyEncryptedDataPageV2 p // The decrypted bytes are laid out as [repetition levels | definition levels | values] ByteBuffer decrypted = blockDecryptor.decrypt(encrypted, dataPageAAD); checkDecryptedPageSize(decrypted.limit(), levelsByteLength); - ByteBuffer repetitionLevelsBuffer = slice(decrypted, 0, page.getRepetitionLevelsByteLength()); - ByteBuffer definitionLevelsBuffer = slice( - decrypted, page.getRepetitionLevelsByteLength(), page.getDefinitionLevelsByteLength()); - ByteBuffer valuesBuffer = slice(decrypted, levelsByteLength, decrypted.limit() - levelsByteLength); - repetitionLevels = BytesInput.from(repetitionLevelsBuffer); - definitionLevels = BytesInput.from(definitionLevelsBuffer); + repetitionLevels = BytesInput.from(decrypted.slice(0, repetitionLevelsByteLength)); + definitionLevels = + BytesInput.from(decrypted.slice(repetitionLevelsByteLength, definitionLevelsByteLength)); + ByteBuffer valuesBuffer = decrypted.slice(levelsByteLength, decrypted.limit() - levelsByteLength); if (page.isCompressed()) { - long compressedSize = valuesBuffer.limit(); ByteBuffer decompressedBuffer = options.getAllocator().allocate(valuesUncompressedSize); releaser.releaseLater(decompressedBuffer); + // The decompressor consumes the buffer, so the compressed size is taken before it runs + BytesInput compressedValues = BytesInput.from(valuesBuffer.duplicate()); long start = System.nanoTime(); decompressor.decompress( - valuesBuffer, (int) compressedSize, decompressedBuffer, valuesUncompressedSize); - setDecompressMetrics(encryptedBody, start); + valuesBuffer, valuesBuffer.limit(), decompressedBuffer, valuesUncompressedSize); + setDecompressMetrics(compressedValues, start); decompressedBuffer.flip(); values = BytesInput.from(decompressedBuffer); } else { @@ -357,14 +402,15 @@ private DataPageV2 readSingleEncryptedModulePageV2(FullBodyEncryptedDataPageV2 p } else { byte[] decrypted = blockDecryptor.decrypt(encryptedBody.toByteArray(), dataPageAAD); checkDecryptedPageSize(decrypted.length, levelsByteLength); - repetitionLevels = BytesInput.from(decrypted, 0, page.getRepetitionLevelsByteLength()); - definitionLevels = BytesInput.from( - decrypted, page.getRepetitionLevelsByteLength(), page.getDefinitionLevelsByteLength()); + repetitionLevels = BytesInput.from(decrypted, 0, repetitionLevelsByteLength); + definitionLevels = + BytesInput.from(decrypted, repetitionLevelsByteLength, definitionLevelsByteLength); values = BytesInput.from(decrypted, levelsByteLength, decrypted.length - levelsByteLength); if (page.isCompressed()) { long start = System.nanoTime(); - values = BytesInput.copy(decompressor.decompress(values, valuesUncompressedSize)); - setDecompressMetrics(encryptedBody, start); + BytesInput compressedValues = values; + values = BytesInput.copy(decompressor.decompress(compressedValues, valuesUncompressedSize)); + setDecompressMetrics(compressedValues, start); } } } catch (IOException e) { @@ -381,12 +427,6 @@ private void checkDecryptedPageSize(int decryptedSize, int levelsByteLength) { } } - private ByteBuffer slice(ByteBuffer buffer, int offset, int length) { - ByteBuffer duplicate = buffer.duplicate(); - duplicate.position(offset).limit(offset + length); - return duplicate.slice(); - } - private void setDecompressMetrics(BytesInput bytes, long start) { final ParquetMetricsCallback metricsCallback = options.getMetricsCallback(); if (metricsCallback != null) { diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java index 5dd49a45d8..f1536ec10a 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java @@ -317,11 +317,8 @@ public void writePageV2( compressedData = compressor.compress(data); compressed = true; } - // The bytes that follow the page header. The "data page" module of Parquet Modular Encryption - // is the whole page body of a DataPageV2 page, so when the column is encrypted the repetition - // and definition levels are encrypted together with the values, as one module. This is the - // layout of the specification and the one written by Arrow C++ and arrow-rs; the levels - // must not be left in plaintext. + // The bytes that follow the page header. With encryption, the levels and the values of a v2 page + // are encrypted together, as the one "data page" module of the page. List pageBody = new ArrayList<>(3); if (null != pageBlockEncryptor) { AesCipher.quickUpdatePageAAD(dataPageAAD, pageOrdinal); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java index 6659c052b2..9c96962702 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java @@ -28,40 +28,56 @@ import org.apache.parquet.column.statistics.Statistics; /** - * A still encrypted {@link DataPageV2} whose entire page body is a single encrypted module. + * A still encrypted {@link DataPageV2} whose entire page body is, or if its layout is {@link + * Layout#AMBIGUOUS} may be, a single encrypted module. * *

Parquet Modular Encryption encrypts the "data page" module, which for a DataPageV2 page is * the concatenation {@code [repetition levels | definition levels | (compressed) values]}. This is - * what Arrow C++ and arrow-rs write, and what parquet-java writes since the layout was aligned. - * Older parquet-java versions left the levels in plaintext and encrypted only the values; those - * pages are still read as plain {@link DataPageV2} pages whose levels are already separated out. + * what Arrow C++ and arrow-rs write, and what parquet-java writes since 1.19.0. Earlier + * parquet-java versions left the levels in plaintext and encrypted only the values; those pages + * are still read as plain {@link DataPageV2} pages whose levels are already separated out. * *

The page body can only be decrypted, and thereby split into levels and values, once the page * AAD is known. That happens in {@code ColumnChunkPageReader}. Until then the whole encrypted * module is held as the {@link #getData() data} of this page, and the repetition and definition - * levels are empty. The sizes of the levels, which the page header carries, are kept separately. + * levels are empty: they must not be used before the page has been decrypted. The sizes of the + * levels, which the page header carries, are kept separately. */ final class FullBodyEncryptedDataPageV2 extends DataPageV2 { + /** How the body of an encrypted DataPageV2 page that has levels is laid out. */ + enum Layout { + /** The whole body is one encrypted module, as in the specification. */ + SINGLE_MODULE, + /** The levels are in plaintext, followed by an encrypted module of the values. */ + PLAINTEXT_LEVELS, + /** The body fits both layouts, which can only be told apart by trying to decrypt it. */ + AMBIGUOUS + } + private final int repetitionLevelsByteLength; private final int definitionLevelsByteLength; + private final DataPageV2 plaintextLevelsLayoutPage; /** * @param encryptedBody the encrypted page body, in the serialized form of an encrypted module: * {@code length (4 bytes) | nonce | ciphertext [| tag]} * @param uncompressedSize the uncompressed page size of the page header, levels included + * @param plaintextLevelsLayoutPage the same page with the levels separated out, if the page + * body is {@link Layout#AMBIGUOUS}, otherwise null */ FullBodyEncryptedDataPageV2( int rowCount, int nullCount, int valueCount, - BytesInput encryptedBody, Encoding dataEncoding, + BytesInput encryptedBody, int repetitionLevelsByteLength, int definitionLevelsByteLength, int uncompressedSize, Statistics statistics, - boolean isCompressed) { + boolean isCompressed, + DataPageV2 plaintextLevelsLayoutPage) { super( rowCount, nullCount, @@ -75,6 +91,7 @@ final class FullBodyEncryptedDataPageV2 extends DataPageV2 { isCompressed); this.repetitionLevelsByteLength = repetitionLevelsByteLength; this.definitionLevelsByteLength = definitionLevelsByteLength; + this.plaintextLevelsLayoutPage = plaintextLevelsLayoutPage; } int getRepetitionLevelsByteLength() { @@ -86,31 +103,58 @@ int getDefinitionLevelsByteLength() { } /** - * Tells which of the two DataPageV2 layouts an encrypted page body has. This must only be called - * for a page that has repetition or definition levels: without levels both layouts are + * @return the page as it would be if its body had the {@link Layout#PLAINTEXT_LEVELS} layout, to + * fall back to when decrypting it as a single module fails. Null unless the layout of the + * page body is {@link Layout#AMBIGUOUS}. + */ + DataPageV2 getPlaintextLevelsLayoutPage() { + return plaintextLevelsLayoutPage; + } + + /** + * Tells which of the two layouts the body of an encrypted DataPageV2 page has. It must only be + * called for a page that has repetition or definition levels: without levels both layouts are * byte-for-byte identical. * *

An encrypted module starts with its own length, a 4 byte little endian integer holding the - * size of the rest of the module. If the page body is a single module, that length is the size - * of the page body minus 4. If the levels are stored in plaintext, which is how earlier - * parquet-java versions wrote the page, the body starts with the RLE encoded levels instead, and - * the length of the encrypted values is found after the levels. Matching the first four bytes - * with the page size is therefore sufficient to recognize the single module layout. A legacy page - * would only be mistaken for it if the first four bytes of its levels happened to hold exactly - * that value; in that case the page fails to decrypt, it is not silently misread. + * size of the rest of the module. Each layout therefore announces itself: + * + *

+ * + *

A page body that is a single module has a random nonce and ciphertext after its length, so + * it matches the second condition only by a coincidence of about one in 2^32, a little more often if + * the levels take fewer than four bytes, as the four bytes then overlap the length of the module. A page body that has + * plaintext levels starts with RLE encoded levels, so it matches the first condition only if + * those happen to spell out the size of the page. When both conditions are met, the layout can + * only be found by decrypting the page, see {@link Layout#AMBIGUOUS}. When neither is met the page + * is corrupt, and it is treated as a single module so that decryption reports the problem. * * @param pageBody the page body, as sized by the {@code compressed_page_size} of the page header - * @return true if the whole page body is one encrypted module, false if the levels are stored in - * plaintext in front of the encrypted values + * @param levelsByteLength the size of the repetition and definition levels, which is above zero + * @return the layout of the page body * @throws IOException if the page body cannot be read */ - static boolean isSingleEncryptedModule(BytesInput pageBody) throws IOException { + static Layout detectLayout(BytesInput pageBody, int levelsByteLength) throws IOException { long size = pageBody.size(); - if (size < Integer.BYTES) { - return false; + boolean singleModule = size >= Integer.BYTES && readLength(pageBody, 0) == size - Integer.BYTES; + boolean plaintextLevels = size - levelsByteLength >= Integer.BYTES + && readLength(pageBody, levelsByteLength) == size - levelsByteLength - Integer.BYTES; + if (plaintextLevels && singleModule) { + return Layout.AMBIGUOUS; } + return plaintextLevels ? Layout.PLAINTEXT_LEVELS : Layout.SINGLE_MODULE; + } + + private static int readLength(BytesInput pageBody, int offset) throws IOException { try (ByteBufferInputStream in = pageBody.toInputStream()) { - return BytesUtils.readIntLittleEndian(in) == size - Integer.BYTES; + in.skipFully(offset); + return BytesUtils.readIntLittleEndian(in); } } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index b46464d034..56ea915e1b 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -2215,76 +2215,17 @@ public ColumnChunkPageReader readAllPages( break; case DATA_PAGE_V2: DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2(); - final int repetitionLevelsByteLength = dataHeaderV2.getRepetition_levels_byte_length(); - final int definitionLevelsByteLength = dataHeaderV2.getDefinition_levels_byte_length(); - final BytesInput repetitionLevels; - final BytesInput definitionLevels; - final BytesInput values; - boolean singleEncryptedModule = false; - if (null != pageBlockDecryptor && repetitionLevelsByteLength + definitionLevelsByteLength > 0) { - // The page body of an encrypted DataPageV2 is either one encrypted module that also - // covers the levels (the layout of the Parquet encryption spec, as written by Arrow - // and by current parquet-java), or, in files written by earlier parquet-java versions, - // plaintext levels followed by an encrypted module of the values. - BytesInput pageBody = this.readAsBytesInput(compressedPageSize); - singleEncryptedModule = FullBodyEncryptedDataPageV2.isSingleEncryptedModule(pageBody); - if (singleEncryptedModule) { - repetitionLevels = BytesInput.empty(); - definitionLevels = BytesInput.empty(); - values = pageBody; - } else { - ByteBufferInputStream pageBodyStream = pageBody.toInputStream(); - repetitionLevels = - BytesInput.from(pageBodyStream.sliceBuffers(repetitionLevelsByteLength)); - definitionLevels = - BytesInput.from(pageBodyStream.sliceBuffers(definitionLevelsByteLength)); - values = BytesInput.from(pageBodyStream.remainingBuffers()); - } - } else { - int dataSize = compressedPageSize - repetitionLevelsByteLength - definitionLevelsByteLength; - repetitionLevels = this.readAsBytesInput(repetitionLevelsByteLength); - definitionLevels = this.readAsBytesInput(definitionLevelsByteLength); - values = this.readAsBytesInput(dataSize); - } - if (options.usePageChecksumVerification() && pageHeader.isSetCrc()) { - verifyCrc( - pageHeader.getCrc(), - "could not verify page integrity, CRC checksum verification failed", - repetitionLevels, - definitionLevels, - values); - } - DataPageV2 dataPageV2; - if (singleEncryptedModule) { - dataPageV2 = new FullBodyEncryptedDataPageV2( - dataHeaderV2.getNum_rows(), - dataHeaderV2.getNum_nulls(), - dataHeaderV2.getNum_values(), - values, - converter.getEncoding(dataHeaderV2.getEncoding()), - repetitionLevelsByteLength, - definitionLevelsByteLength, - uncompressedPageSize, - converter.fromParquetStatistics( - getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), - dataHeaderV2.isIs_compressed()); - } else { - dataPageV2 = new DataPageV2( - dataHeaderV2.getNum_rows(), - dataHeaderV2.getNum_nulls(), - dataHeaderV2.getNum_values(), - repetitionLevels, - definitionLevels, - converter.getEncoding(dataHeaderV2.getEncoding()), - values, - uncompressedPageSize, - converter.fromParquetStatistics( - getFileMetaData().getCreatedBy(), dataHeaderV2.getStatistics(), type), - dataHeaderV2.isIs_compressed()); - } + DataPageV2 dataPageV2 = readDataPageV2(pageHeader, type, null != pageBlockDecryptor); // Copy crc to new page, used for testing if (pageHeader.isSetCrc()) { dataPageV2.setCrc(pageHeader.getCrc()); + if (dataPageV2 instanceof FullBodyEncryptedDataPageV2) { + DataPageV2 plaintextLevelsPage = + ((FullBodyEncryptedDataPageV2) dataPageV2).getPlaintextLevelsLayoutPage(); + if (plaintextLevelsPage != null) { + plaintextLevelsPage.setCrc(pageHeader.getCrc()); + } + } } pagesInChunk.add(dataPageV2); valuesCountReadSoFar += dataHeaderV2.getNum_values(); @@ -2333,6 +2274,103 @@ private int getPageOrdinal(int dataPageCountReadSoFar) { return offsetIndex.getPageOrdinal(dataPageCountReadSoFar); } + /** + * Reads the DataPageV2 page of which the header has just been read. The page of an encrypted column + * that has repetition or definition levels is read as a {@link FullBodyEncryptedDataPageV2} if its + * body is, or may be, one encrypted module that also covers the levels, which is the layout of the Parquet + * encryption specification, written by Arrow and by parquet-java since 1.19.0. Files written by + * earlier parquet-java versions have the levels in plaintext in front of the module of the + * values; these pages are read as plain {@link DataPageV2} pages that are still encrypted. + */ + private DataPageV2 readDataPageV2(PageHeader pageHeader, PrimitiveType type, boolean encryptedColumn) + throws IOException { + DataPageHeaderV2 header = pageHeader.getData_page_header_v2(); + int compressedPageSize = pageHeader.getCompressed_page_size(); + int uncompressedPageSize = pageHeader.getUncompressed_page_size(); + int repetitionLevelsByteLength = header.getRepetition_levels_byte_length(); + int definitionLevelsByteLength = header.getDefinition_levels_byte_length(); + org.apache.parquet.column.Encoding encoding = converter.getEncoding(header.getEncoding()); + org.apache.parquet.column.statistics.Statistics statistics = + converter.fromParquetStatistics(getFileMetaData().getCreatedBy(), header.getStatistics(), type); + + if (encryptedColumn && repetitionLevelsByteLength + definitionLevelsByteLength != 0) { + if (repetitionLevelsByteLength < 0 + || definitionLevelsByteLength < 0 + || (long) repetitionLevelsByteLength + definitionLevelsByteLength + > Math.min(compressedPageSize, uncompressedPageSize)) { + throw new ParquetDecodingException("Invalid DataPageV2 header in column " + descriptor.col + + ": repetition levels of " + repetitionLevelsByteLength + + " bytes and definition levels of " + + definitionLevelsByteLength + " bytes in a page of " + compressedPageSize + + " bytes, of " + uncompressedPageSize + " bytes uncompressed"); + } + int levelsByteLength = repetitionLevelsByteLength + definitionLevelsByteLength; + BytesInput pageBody = this.readAsBytesInput(compressedPageSize); + if (options.usePageChecksumVerification() && pageHeader.isSetCrc()) { + verifyCrc( + pageHeader.getCrc(), + "could not verify page integrity, CRC checksum verification failed", + pageBody); + } + FullBodyEncryptedDataPageV2.Layout layout = + FullBodyEncryptedDataPageV2.detectLayout(pageBody, levelsByteLength); + DataPageV2 plaintextLevelsPage = null; + if (layout != FullBodyEncryptedDataPageV2.Layout.SINGLE_MODULE) { + ByteBufferInputStream pageBodyStream = pageBody.toInputStream(); + plaintextLevelsPage = new DataPageV2( + header.getNum_rows(), + header.getNum_nulls(), + header.getNum_values(), + BytesInput.from(pageBodyStream.sliceBuffers(repetitionLevelsByteLength)), + BytesInput.from(pageBodyStream.sliceBuffers(definitionLevelsByteLength)), + encoding, + BytesInput.from(pageBodyStream.remainingBuffers()), + uncompressedPageSize, + statistics, + header.isIs_compressed()); + } + if (layout == FullBodyEncryptedDataPageV2.Layout.PLAINTEXT_LEVELS) { + return plaintextLevelsPage; + } + return new FullBodyEncryptedDataPageV2( + header.getNum_rows(), + header.getNum_nulls(), + header.getNum_values(), + encoding, + pageBody, + repetitionLevelsByteLength, + definitionLevelsByteLength, + uncompressedPageSize, + statistics, + header.isIs_compressed(), + plaintextLevelsPage); + } + + int dataSize = compressedPageSize - repetitionLevelsByteLength - definitionLevelsByteLength; + BytesInput repetitionLevels = this.readAsBytesInput(repetitionLevelsByteLength); + BytesInput definitionLevels = this.readAsBytesInput(definitionLevelsByteLength); + BytesInput values = this.readAsBytesInput(dataSize); + if (options.usePageChecksumVerification() && pageHeader.isSetCrc()) { + verifyCrc( + pageHeader.getCrc(), + "could not verify page integrity, CRC checksum verification failed", + repetitionLevels, + definitionLevels, + values); + } + return new DataPageV2( + header.getNum_rows(), + header.getNum_nulls(), + header.getNum_values(), + repetitionLevels, + definitionLevels, + encoding, + values, + uncompressedPageSize, + statistics, + header.isIs_compressed()); + } + /** * @param size the size of the page * @return the page diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index 4c02bc5725..f8048efd52 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -1169,7 +1169,10 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page + * Writes a single v2 data page. The repetition and definition levels and the data bytes are written + * as given: nothing is encrypted. The page of an encrypted column has its levels in the encrypted + * page body, so the data bytes of such a page must not be encrypted by the caller; use the overload + * that takes a page block encryptor. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1223,7 +1226,7 @@ public void writeDataPageV2( * @param compressedData compressed data bytes * @param uncompressedDataSize the size of uncompressed data * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for block data + * @param metadataBlockEncryptor encryptor for the page header * @param pageHeaderAAD pageHeader AAD * @throws IOException if any I/O error occurs during writing the file * @deprecated will be removed in 2.0.0. Use {@link ParquetFileWriter#writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[])} instead @@ -1259,7 +1262,10 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page + * Writes a single v2 data page. The repetition and definition levels and the data bytes are written + * as given: nothing is encrypted. The page of an encrypted column has its levels in the encrypted + * page body, so the data bytes of such a page must not be encrypted by the caller; use the overload + * that takes a page block encryptor. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1271,7 +1277,7 @@ public void writeDataPageV2( * @param compressed whether the data bytes is compressed * @param uncompressedDataSize the size of uncompressed data * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for block data + * @param metadataBlockEncryptor encryptor for the page header * @param pageHeaderAAD pageHeader AAD * @throws IOException if any I/O error occurs during writing the file */ @@ -1317,7 +1323,7 @@ public void writeDataPageV2( * @param compressedData compressed data bytes * @param uncompressedDataSize the size of uncompressed data * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for block data + * @param metadataBlockEncryptor encryptor for the page header * @param pageHeaderAAD pageHeader AAD * @param sizeStatistics size statistics for the page * @throws IOException if any I/O error occurs during writing the file @@ -1356,8 +1362,9 @@ public void writeDataPageV2( /** * Writes a single v2 data page. The repetition and definition levels are written as given, in - * front of the data bytes. To write the page of an encrypted column, whose levels are part of the - * encrypted page body, use {@link #writeEncryptedDataPageV2}. + * front of the data bytes, and the data bytes are written as given too: they are not encrypted. + * To write the page of an encrypted column, whose levels are part of the encrypted page body, use + * the overload that takes a page block encryptor. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1369,7 +1376,7 @@ public void writeDataPageV2( * @param compressed whether the data bytes is compressed * @param uncompressedDataSize the size of uncompressed data * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for block data + * @param metadataBlockEncryptor encryptor for the page header * @param pageHeaderAAD pageHeader AAD * @param sizeStatistics size statistics for the page * @throws IOException if any I/O error occurs during writing the file @@ -1406,55 +1413,67 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page of an encrypted column, for a caller that has already encrypted - * the page body. + * Writes a single v2 data page of a column that is encrypted, if a page block encryptor is given. * - *

With Parquet Modular Encryption the data page module of a v2 page is the whole page body, - * so the encrypted page body holds the repetition levels, the definition levels and the values, - * in that order. The levels must not be written in plaintext in front of the encrypted values, - * which is what {@link #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, - * BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[], SizeStatistics)} does with - * the bytes it is given. + *

With Parquet Modular Encryption the data page module of a v2 page is the whole page body: the + * repetition levels, the definition levels and the (possibly compressed) data are encrypted + * together, as one module. Without a page block encryptor this method behaves like {@link + * #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, + * Statistics, BlockCipher.Encryptor, byte[], SizeStatistics)}, which writes the levels as given and + * does not encrypt the data. * * @param rowCount count of rows * @param nullCount count of nulls * @param valueCount count of values - * @param repetitionLevelsByteLength length of the repetition levels, before encryption - * @param definitionLevelsByteLength length of the definition levels, before encryption + * @param repetitionLevels repetition level bytes + * @param definitionLevels definition level bytes * @param dataEncoding encoding for data - * @param encryptedPageBody the encrypted concatenation of repetition levels, definition levels and - * (possibly compressed) data, as serialized by the page block encryptor - * @param compressed whether the data bytes were compressed before they were encrypted - * @param uncompressedDataSize the size of the uncompressed data, without the levels + * @param bytes data bytes, not encrypted + * @param compressed whether the data bytes is compressed + * @param uncompressedDataSize the size of uncompressed data * @param statistics the statistics of the page * @param metadataBlockEncryptor encryptor for the page header * @param pageHeaderAAD pageHeader AAD * @param sizeStatistics size statistics for the page + * @param pageBlockEncryptor encryptor for the page body, or null if the column is not encrypted + * @param dataPageAAD AAD of the data page module of this page, that is with the ordinal of the page, + * see {@link AesCipher#quickUpdatePageAAD}; it is required with a page block + * encryptor, and ignored without one * @throws IOException if any I/O error occurs during writing the file */ - public void writeEncryptedDataPageV2( + public void writeDataPageV2( int rowCount, int nullCount, int valueCount, - int repetitionLevelsByteLength, - int definitionLevelsByteLength, + BytesInput repetitionLevels, + BytesInput definitionLevels, Encoding dataEncoding, - BytesInput encryptedPageBody, + BytesInput bytes, boolean compressed, int uncompressedDataSize, Statistics statistics, BlockCipher.Encryptor metadataBlockEncryptor, byte[] pageHeaderAAD, - SizeStatistics sizeStatistics) + SizeStatistics sizeStatistics, + BlockCipher.Encryptor pageBlockEncryptor, + byte[] dataPageAAD) throws IOException { + List pageBody; + if (pageBlockEncryptor == null) { + pageBody = Arrays.asList(repetitionLevels, definitionLevels, bytes); + } else { + Preconditions.checkArgument(dataPageAAD != null, "dataPageAAD is required to encrypt the page body"); + pageBody = Collections.singletonList(BytesInput.from(pageBlockEncryptor.encrypt( + BytesInput.concat(repetitionLevels, definitionLevels, bytes).toByteArray(), dataPageAAD))); + } writeDataPageV2Body( rowCount, nullCount, valueCount, - repetitionLevelsByteLength, - definitionLevelsByteLength, + toIntWithCheck(repetitionLevels.size(), "page repetition levels"), + toIntWithCheck(definitionLevels.size(), "page definition levels"), dataEncoding, - Collections.singletonList(encryptedPageBody), + pageBody, compressed, uncompressedDataSize, statistics, diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java index 15c1fb35fc..212bd75254 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java @@ -738,8 +738,8 @@ private void processChunk( BytesInput dlLevels = readBlockAllocate(dlLength, reader); int payLoadLength = pageHeader.getCompressed_page_size() - rlLength - dlLength; int rawDataLength = pageHeader.getUncompressed_page_size() - rlLength - dlLength; - // The page body of an encrypted DataPageV2 is encrypted as a whole, levels included, so the - // payload is not encrypted here but together with the levels below. + // The payload is not encrypted here: for an encrypted column the writer encrypts it together + // with the levels. pageLoad = processPageLoad( reader, headerV2.is_compressed, @@ -767,40 +767,22 @@ private void processChunk( } readValues += headerV2.getNum_values(); readRows += headerV2.getNum_rows(); - if (encryptColumn) { - BytesInput encryptedPageBody = BytesInput.from(dataEncryptor.encrypt( - BytesInput.concat(rlLevels, dlLevels, BytesInput.from(pageLoad)) - .toByteArray(), - dataPageAAD)); - writer.writeEncryptedDataPageV2( - headerV2.getNum_rows(), - headerV2.getNum_nulls(), - headerV2.getNum_values(), - rlLength, - dlLength, - converter.getEncoding(headerV2.getEncoding()), - encryptedPageBody, - headerV2.is_compressed, - rawDataLength, - statistics, - metaEncryptor, - dataPageHeaderAAD, - null); - } else { - writer.writeDataPageV2( - headerV2.getNum_rows(), - headerV2.getNum_nulls(), - headerV2.getNum_values(), - rlLevels, - dlLevels, - converter.getEncoding(headerV2.getEncoding()), - BytesInput.from(pageLoad), - headerV2.is_compressed, - rawDataLength, - statistics, - metaEncryptor, - dataPageHeaderAAD); - } + writer.writeDataPageV2( + headerV2.getNum_rows(), + headerV2.getNum_nulls(), + headerV2.getNum_values(), + rlLevels, + dlLevels, + converter.getEncoding(headerV2.getEncoding()), + BytesInput.from(pageLoad), + headerV2.is_compressed, + rawDataLength, + statistics, + metaEncryptor, + dataPageHeaderAAD, + null, + encryptColumn ? dataEncryptor : null, + dataPageAAD); pageOrdinal++; break; default: diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java index b6b3dacf5a..3764607a38 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java @@ -27,8 +27,10 @@ import org.apache.hadoop.fs.Path; import org.apache.parquet.bytes.DirectByteBufferAllocator; import org.apache.parquet.bytes.HeapByteBufferAllocator; +import org.apache.parquet.bytes.TrackingByteBufferAllocator; import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.filter2.compat.FilterCompat; import org.apache.parquet.hadoop.ParquetInputFormat; import org.apache.parquet.hadoop.ParquetReader; import org.apache.parquet.hadoop.example.GroupReadSupport; @@ -47,7 +49,7 @@ *

The row generator must stay in sync with {@code generate_arrow_fixtures.py} in the fixtures * directory. */ -public final class DataPageV2EncryptionTestData { +final class DataPageV2EncryptionTestData { public static final int NUM_ROWS = 500; @@ -123,27 +125,58 @@ public static FileDecryptionProperties decryptionProperties() { } /** - * Reads all of the rows of a file, rendered as strings. + * Reads all of the rows of a file, rendered as strings, with the verification of the page checksums + * switched on. * - * @param conf may hold the configuration of a decryption properties factory + * @param conf may hold the configuration of a decryption properties factory, and whether to verify the + * page checksums, which is the default; it is not modified * @param decryption the decryption properties, or null to take them from the configuration * @param directMemory whether to read the file into direct buffers and decrypt it off-heap */ public static List readRows( Path file, Configuration conf, FileDecryptionProperties decryption, boolean directMemory) throws IOException { - conf.setBoolean(ParquetInputFormat.OFF_HEAP_DECRYPT_BUFFER_ENABLED, directMemory); - ParquetReader.Builder builder = ParquetReader.builder(new GroupReadSupport(), file) - .withConf(conf) - .withAllocator(directMemory ? new DirectByteBufferAllocator() : new HeapByteBufferAllocator()); - if (decryption != null) { - builder.withDecryption(decryption); + return readRows(file, conf, decryption, directMemory, FilterCompat.NOOP); + } + + /** + * Like {@link #readRows(Path, Configuration, FileDecryptionProperties, boolean)}, for the rows that + * pass the filter. With a filter, the pages that cannot hold matching rows are skipped by way of the + * offset index. + */ + public static List readRows( + Path file, + Configuration conf, + FileDecryptionProperties decryption, + boolean directMemory, + FilterCompat.Filter filter) + throws IOException { + Configuration readConf = new Configuration(conf); + readConf.setBoolean(ParquetInputFormat.OFF_HEAP_DECRYPT_BUFFER_ENABLED, directMemory); + readConf.setBoolean( + ParquetInputFormat.PAGE_VERIFY_CHECKSUM_ENABLED, + conf.getBoolean(ParquetInputFormat.PAGE_VERIFY_CHECKSUM_ENABLED, true)); + if (filter != FilterCompat.NOOP) { + // Without this the rows of the pages that are read would be filtered too, which hides whether + // the pages that cannot hold matching rows were skipped + readConf.setBoolean(ParquetInputFormat.RECORD_FILTERING_ENABLED, false); } List rows = new ArrayList<>(); - try (ParquetReader reader = builder.build()) { - Group group; - while ((group = reader.read()) != null) { - rows.add(group.toString()); + // Closing the allocator fails the test if the reader leaks a buffer + try (TrackingByteBufferAllocator allocator = TrackingByteBufferAllocator.wrap( + directMemory ? new DirectByteBufferAllocator() : new HeapByteBufferAllocator())) { + ParquetReader.Builder builder = ParquetReader.builder(new GroupReadSupport(), file) + .withConf(readConf) + .withFilter(filter) + .withAllocator(allocator); + if (decryption != null) { + builder.withDecryption(decryption); + } + try (ParquetReader reader = builder.build()) { + Group group; + while ((group = reader.read()) != null) { + rows.add(group.toString()); + } } } return rows; diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java index bf843381af..276e67eaa8 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java @@ -35,10 +35,12 @@ import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; import java.net.URISyntaxException; import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -47,18 +49,28 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.ParquetReadOptions; +import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.BytesUtils; +import org.apache.parquet.column.ColumnDescriptor; +import org.apache.parquet.column.Encoding; +import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; +import org.apache.parquet.column.statistics.Statistics; import org.apache.parquet.crypto.ModuleCipherFactory.ModuleType; import org.apache.parquet.crypto.keytools.KeyToolkit; import org.apache.parquet.crypto.keytools.PropertiesDrivenCryptoFactory; +import org.apache.parquet.crypto.keytools.mocks.Base64KmsClient; import org.apache.parquet.example.data.Group; +import org.apache.parquet.filter2.compat.FilterCompat; +import org.apache.parquet.filter2.predicate.FilterApi; import org.apache.parquet.format.BlockCipher; import org.apache.parquet.format.FileCryptoMetaData; import org.apache.parquet.format.PageHeader; import org.apache.parquet.format.PageType; import org.apache.parquet.format.Util; import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetFileWriter; +import org.apache.parquet.hadoop.ParquetInputFormat; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.example.ExampleParquetWriter; import org.apache.parquet.hadoop.metadata.BlockMetaData; @@ -68,7 +80,12 @@ import org.apache.parquet.hadoop.rewrite.ParquetRewriter; import org.apache.parquet.hadoop.rewrite.RewriteOptions; import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.hadoop.util.HadoopOutputFile; import org.apache.parquet.io.ParquetDecodingException; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -91,16 +108,36 @@ * java-legacy-v2-*} fixtures are. They were written by parquet-java 1.19.0-SNAPSHOT at commit * e265ac7f5, just before the layout changed, with the footer key as the key of all columns, * the same rows as {@link DataPageV2EncryptionTestData#row(int)}, 100 rows per page and - * {@code ExampleParquetWriter} with {@code WriterVersion.PARQUET_2_0}. + * {@code ExampleParquetWriter} with {@code WriterVersion.PARQUET_2_0}. To regenerate them, + * check out that commit, and write the rows with the {@code writeWithParquetJava} method of + * this class for each cipher and codec. The Arrow files come from {@code + * generate_arrow_fixtures.py}, which sits next to them. * */ public class TestDataPageV2EncryptionInterop { private static final String FIXTURES = "/encryption/datapagev2/"; + /** + * Matches the second half of the rows. With 100 rows per page, the first two pages cannot hold a + * match and are skipped, so the rows that are read start at the third page. + */ + private static final int FIRST_ROW_OF_PAGE_WITH_SEQ_250 = 200; + + private static final FilterCompat.Filter SEQ_AT_LEAST_250 = + FilterCompat.get(FilterApi.gtEq(FilterApi.longColumn("seq"), 250L)); + @TempDir private java.nio.file.Path tempDir; + @BeforeEach + @AfterEach + public void clearKmsClients() { + // The KMS clients are cached by instance id, across the tests of a JVM, and the Arrow fixtures use + // the default one + KeyToolkit.removeCacheEntriesForAllTokens(); + } + static Stream fixturesAndMemoryModes(String prefix) { List arguments = new ArrayList<>(); for (String cipher : new String[] {"aes-gcm-v1", "aes-gcm-ctr-v1"}) { @@ -121,6 +158,17 @@ static Stream legacyFixtures() { return fixturesAndMemoryModes("java-legacy-v2-"); } + static Stream legacyFixtureNames() { + return Stream.of("aes-gcm-v1", "aes-gcm-ctr-v1") + .flatMap(cipher -> Stream.of("uncompressed", "snappy") + .map(codec -> "java-legacy-v2-" + cipher + "-" + codec + ".parquet")); + } + + static Stream ciphersAndMemoryModes() { + return Stream.of(ParquetCipher.values()) + .flatMap(cipher -> Stream.of(Arguments.of(cipher, false), Arguments.of(cipher, true))); + } + static Stream ciphersAndCodecs() { List arguments = new ArrayList<>(); for (ParquetCipher cipher : ParquetCipher.values()) { @@ -152,9 +200,9 @@ public void readsFilesWrittenByEarlierParquetJavaVersions(String fixture, boolea .containsExactlyElementsOf(expectedRows()); } - @ParameterizedTest(name = "{0}, directMemory={1}") - @MethodSource("legacyFixtures") - public void legacyFixturesHaveThePlaintextLevelsLayout(String fixture, boolean directMemory) throws Exception { + @ParameterizedTest(name = "{0}") + @MethodSource("legacyFixtureNames") + public void legacyFixturesHaveThePlaintextLevelsLayout(String fixture) throws Exception { // The test above only proves the legacy code path is covered if the fixtures really are legacy files List pages = decodeDataPagesV2(fixture(fixture), FOOTER_KEY, false); @@ -171,6 +219,9 @@ public void writesTheLevelsInTheEncryptedPageBody(ParquetCipher cipher, Compress List pages = decodeDataPagesV2(file, FOOTER_KEY, false); assertThat(pages).isNotEmpty(); + assertThat(pages).extracting(page -> page.column).containsOnly("id", "name", "tags.list.element"); + assertThat(pages) + .anySatisfy(page -> assertThat(page.repetitionLevelsLength).isPositive()); for (DecodedPage page : pages) { assertThat(page.singleModule) .as("the levels of %s must be inside the encrypted page body", page) @@ -195,13 +246,42 @@ public void readsWhatItWrites(ParquetCipher cipher, CompressionCodecName codec) @Test public void detectsTamperingWithTheLevels() throws Exception { + Path file = writeFileWithTamperedLevels(); + // The page checksum would notice the change first, which is not what this test is about + Configuration withoutChecksums = new Configuration(); + withoutChecksums.setBoolean(ParquetInputFormat.PAGE_VERIFY_CHECKSUM_ENABLED, false); + + assertThatThrownBy(() -> readRows(file, withoutChecksums, decryptionProperties(), false)) + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("Can not read value at 0 in block -1") + .cause() + .isInstanceOf(TagVerificationException.class) + .hasMessage("GCM tag check failed"); + } + + @Test + public void pageChecksumDetectsTamperingWithTheLevels() throws Exception { + Path file = writeFileWithTamperedLevels(); + + assertThatThrownBy(() -> readRows(file, new Configuration(), decryptionProperties(), false)) + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("Can not read value at 0 in block -1") + .cause() + .isInstanceOf(ParquetDecodingException.class) + .hasMessage("could not verify page integrity, CRC checksum verification failed"); + } + + /** + * Writes a file of which the first byte of the levels of the first DataPageV2 page was changed. When + * the levels were stored in plaintext, changing them went unnoticed by the decryption. + */ + private Path writeFileWithTamperedLevels() throws Exception { Path file = writeWithParquetJava(ParquetCipher.AES_GCM_V1, CompressionCodecName.UNCOMPRESSED); DecodedPage page = decodeDataPagesV2(file, FOOTER_KEY, false).get(0); - // Flip a bit of the first byte of the ciphertext, which is the first byte of the levels. When the - // levels were stored in plaintext, changing them went unnoticed. + // Flip a bit of the first byte of the ciphertext, which is the first byte of the levels File source = new File(file.toUri()); - try (java.io.RandomAccessFile raf = new java.io.RandomAccessFile(source, "rw")) { + try (RandomAccessFile raf = new RandomAccessFile(source, "rw")) { long position = page.bodyOffset + Integer.BYTES + AesCipher.NONCE_LENGTH; raf.seek(position); int original = raf.read(); @@ -210,12 +290,7 @@ public void detectsTamperingWithTheLevels() throws Exception { } // Drop the local filesystem CRC side-file so the tampered bytes reach the Parquet reader. Files.deleteIfExists(new File(source.getParent(), "." + source.getName() + ".crc").toPath()); - - assertThatThrownBy(() -> readRows(file, new Configuration(), decryptionProperties(), false)) - .isInstanceOf(ParquetDecodingException.class) - .hasMessageContaining("Can not read value at 0 in block -1") - .hasRootCauseInstanceOf(TagVerificationException.class) - .hasRootCauseMessage("GCM tag check failed"); + return file; } @ParameterizedTest(name = "{0}, {1}") @@ -251,6 +326,9 @@ public void rewriterWritesTheLevelsInTheEncryptedPageBody(ParquetCipher cipher, List pages = decodeDataPagesV2(rewritten, COLUMN_KEY, true); assertThat(pages).isNotEmpty(); + assertThat(pages).extracting(page -> page.column).containsOnly("id", "name", "tags.list.element"); + assertThat(pages) + .anySatisfy(page -> assertThat(page.repetitionLevelsLength).isPositive()); assertThat(pages).allSatisfy(page -> assertThat(page.singleModule).isTrue()); assertThat(readRows(rewritten, new Configuration(), decryptionProperties(), false)) .containsExactlyElementsOf(expectedRows()); @@ -258,13 +336,127 @@ public void rewriterWritesTheLevelsInTheEncryptedPageBody(ParquetCipher cipher, .containsExactlyElementsOf(expectedRows()); } + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("ciphersAndCodecs") + public void readsWhatItWritesInSeveralRowGroups(ParquetCipher cipher, CompressionCodecName codec) throws Exception { + Path file = writeWithParquetJava(cipher, codec, 4096); + + try (ParquetFileReader reader = ParquetFileReader.open( + HadoopInputFile.fromPath(file, new Configuration()), + ParquetReadOptions.builder() + .withDecryption(decryptionProperties()) + .build())) { + assertThat(reader.getFooter().getBlocks()).hasSizeGreaterThan(1); + } + assertThat(decodeDataPagesV2(file, FOOTER_KEY, false)) + .allSatisfy(page -> assertThat(page.singleModule).isTrue()); + assertThat(readRows(file, new Configuration(), decryptionProperties(), false)) + .containsExactlyElementsOf(expectedRows()); + assertThat(readRows(file, new Configuration(), decryptionProperties(), true)) + .containsExactlyElementsOf(expectedRows()); + } + + @ParameterizedTest(name = "{0}, {1}") + @MethodSource("ciphersAndCodecs") + public void skipsPagesOfWhatItWrites(ParquetCipher cipher, CompressionCodecName codec) throws Exception { + Path file = writeWithParquetJava(cipher, codec); + + assertThat(readRows(file, new Configuration(), decryptionProperties(), false, SEQ_AT_LEAST_250)) + .containsExactlyElementsOf(expectedRows().subList(FIRST_ROW_OF_PAGE_WITH_SEQ_250, NUM_ROWS)); + assertThat(readRows(file, new Configuration(), decryptionProperties(), true, SEQ_AT_LEAST_250)) + .containsExactlyElementsOf(expectedRows().subList(FIRST_ROW_OF_PAGE_WITH_SEQ_250, NUM_ROWS)); + } + + @ParameterizedTest(name = "{0}, directMemory={1}") + @MethodSource("legacyFixtures") + public void skipsPagesOfFilesWrittenByEarlierParquetJavaVersions(String fixture, boolean directMemory) + throws Exception { + assertThat(readRows( + fixture(fixture), new Configuration(), decryptionProperties(), directMemory, SEQ_AT_LEAST_250)) + .containsExactlyElementsOf(expectedRows().subList(FIRST_ROW_OF_PAGE_WITH_SEQ_250, NUM_ROWS)); + } + + /** + * A page of the legacy layout whose plaintext levels happen to start with what the length of a + * module that covers the page body would be, so that the layout cannot be told from the page body + * alone. With AES_GCM_V1 the tag check settles it, AES_GCM_CTR_V1 is not authenticated, and the + * legacy layout is read. + */ + @ParameterizedTest(name = "{0}, directMemory={1}") + @MethodSource("ciphersAndMemoryModes") + public void readsALegacyPageThatLooksLikeASingleModule(ParquetCipher cipher, boolean directMemory) + throws Exception { + MessageType schema = MessageTypeParser.parseMessageType("message schema { optional int32 v; }"); + // Definition levels: a bit-packed run of one group of 8 values of which the first is defined, and + // two more bytes, which the reader does not look at. Read as an integer they are 0x0103 = 259 + byte[] levels = {0x03, 0x01, 0x00, 0x00}; + int bodySize = 259 + Integer.BYTES; + int moduleSize = bodySize - levels.length; + int overhead = Integer.BYTES + + AesCipher.NONCE_LENGTH + + (cipher == ParquetCipher.AES_GCM_V1 ? AesCipher.GCM_TAG_LENGTH : 0); + byte[] values = new byte[moduleSize - overhead]; + System.arraycopy(BytesUtils.intToBytes(12345), 0, values, 0, Integer.BYTES); + + Path file = new Path(tempDir.resolve(UUID.randomUUID().toString()).toUri()); + ParquetFileWriter writer = new ParquetFileWriter( + HadoopOutputFile.fromPath(file, new Configuration()), + schema, + ParquetFileWriter.Mode.CREATE, + ParquetWriter.DEFAULT_BLOCK_SIZE, + ParquetWriter.MAX_PADDING_SIZE_DEFAULT, + FileEncryptionProperties.builder(FOOTER_KEY) + .withFooterKeyMetadata(FOOTER_KEY_METADATA) + .withAlgorithm(cipher) + .build(), + ParquetProperties.builder().build()); + ColumnDescriptor column = schema.getColumns().get(0); + InternalColumnEncryptionSetup setup = writer.getEncryptor().getColumnSetup(ColumnPath.get("v"), true, 0); + byte[] fileAAD = writer.getEncryptor().getFileAAD(); + writer.start(); + writer.startBlock(8); + writer.startColumn(column, 8, CompressionCodecName.UNCOMPRESSED); + // What parquet-java used to write: the levels as they are, in front of the encrypted values + writer.writeDataPageV2( + 8, + 7, + 8, + BytesInput.empty(), + BytesInput.from(levels), + Encoding.PLAIN, + BytesInput.from(setup.getDataEncryptor() + .encrypt(values, AesCipher.createModuleAAD(fileAAD, ModuleType.DataPage, 0, 0, 0))), + false, + values.length, + Statistics.createStats(column.getPrimitiveType()), + setup.getMetaDataEncryptor(), + AesCipher.createModuleAAD(fileAAD, ModuleType.DataPageHeader, 0, 0, 0), + null); + writer.endColumn(); + writer.endBlock(); + writer.end(Collections.emptyMap()); + + List rows = readRows(file, new Configuration(), decryptionProperties(), directMemory); + + assertThat(rows).hasSize(8); + assertThat(rows.get(0)).contains("v: 12345"); + assertThat(rows.subList(1, 8)).allSatisfy(row -> assertThat(row).doesNotContain("v:")); + } + /** Writes a DataPageV2 file with 100 rows per page, encrypted with the footer key if there is a cipher. */ private Path writeWithParquetJava(ParquetCipher cipher, CompressionCodecName codec) throws IOException { + return writeWithParquetJava(cipher, codec, ParquetWriter.DEFAULT_BLOCK_SIZE); + } + + private Path writeWithParquetJava(ParquetCipher cipher, CompressionCodecName codec, long rowGroupSize) + throws IOException { Path file = new Path(tempDir.resolve(UUID.randomUUID().toString()).toUri()); ExampleParquetWriter.Builder builder = ExampleParquetWriter.builder(file) .withType(SCHEMA) .withWriterVersion(WriterVersion.PARQUET_2_0) .withCompressionCodec(codec) + .withRowGroupSize(rowGroupSize) + .withMinRowCountForPageSizeCheck(10) .withPageRowCountLimit(100); if (cipher != null) { builder.withEncryption(FileEncryptionProperties.builder(FOOTER_KEY) @@ -332,7 +524,8 @@ public String toString() { * *

As the specification has the page body start with the length of the encrypted module, the body * is a single module when the first four bytes hold the body length minus four. Otherwise the - * levels are in plaintext and the module of the values follows them. + * levels are in plaintext and the module of the values follows them. That is enough for the files + * of these tests, which have no page for which both could be true. * * @param file file with an encrypted footer, whose encrypted columns all use {@code columnKey} * @param onlyEncryptedColumns true to skip the columns that are not encrypted, for a file where @@ -388,8 +581,8 @@ private static List decodeDataPagesV2(Path file, byte[] columnKey, -1); PageHeader header = Util.readPageHeader(in, headerDecryptor, headerAAD); long bodyOffset = bytes.length - in.available(); - byte[] body = new byte[header.getCompressed_page_size()]; - in.read(body); + byte[] body = in.readNBytes(header.getCompressed_page_size()); + assertThat(body).hasSize(header.getCompressed_page_size()); if (!dictionaryPageRead) { dictionaryPageRead = true; continue; diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/Base64KmsClient.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java similarity index 94% rename from parquet-hadoop/src/test/java/org/apache/parquet/crypto/Base64KmsClient.java rename to parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java index e02e8355af..c84f5c7e37 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/Base64KmsClient.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java @@ -17,10 +17,11 @@ * under the License. */ -package org.apache.parquet.crypto; +package org.apache.parquet.crypto.keytools.mocks; import java.util.Base64; import org.apache.hadoop.conf.Configuration; +import org.apache.parquet.crypto.KeyAccessDeniedException; import org.apache.parquet.crypto.keytools.KmsClient; /** diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java index 2fb2f2d8a5..7095163df0 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java @@ -25,10 +25,14 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import org.apache.parquet.bytes.BytesInput; +import org.apache.parquet.hadoop.FullBodyEncryptedDataPageV2.Layout; import org.junit.jupiter.api.Test; public class TestFullBodyEncryptedDataPageV2 { + /** RLE encoded levels: a bit-packed run of one group, 0xff, followed by an RLE run of 8 values of 1. */ + private static final byte[] LEVELS = {0x03, (byte) 0xff, 0x10, 0x01}; + /** The serialized form of an encrypted module: its length, then that many bytes. */ private static byte[] module(int contentLength) { ByteBuffer module = ByteBuffer.allocate(Integer.BYTES + contentLength).order(ByteOrder.LITTLE_ENDIAN); @@ -47,21 +51,17 @@ private static byte[] concat(byte[] first, byte[] second) { } @Test - public void bodyThatIsOneModuleIsRecognized() throws IOException { - byte[] body = module(50); - - assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(body))) - .isTrue(); + public void bodyThatIsOneModuleIsASingleModule() throws IOException { + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(module(50)), LEVELS.length)) + .isEqualTo(Layout.SINGLE_MODULE); } @Test - public void bodyWithPlaintextLevelsInFrontOfAModuleIsRecognized() throws IOException { - // RLE encoded levels: a bit-packed run of one group, 0xff, followed by an RLE run of 8 values of 1 - byte[] levels = {0x03, (byte) 0xff, 0x10, 0x01}; - byte[] body = concat(levels, module(50)); + public void bodyWithPlaintextLevelsInFrontOfAModuleHasPlaintextLevels() throws IOException { + byte[] body = concat(LEVELS, module(50)); - assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(body))) - .isFalse(); + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(body), LEVELS.length)) + .isEqualTo(Layout.PLAINTEXT_LEVELS); } @Test @@ -70,24 +70,54 @@ public void bodyThatIsOneModuleIsRecognizedWhenItIsNotInOneBuffer() throws IOExc ByteBuffer first = ByteBuffer.wrap(body, 0, 2).slice(); ByteBuffer second = ByteBuffer.wrap(body, 2, body.length - 2).slice(); - assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(first, second))) - .isTrue(); + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(first, second), LEVELS.length)) + .isEqualTo(Layout.SINGLE_MODULE); } @Test - public void bodyThatIsTooShortToStartWithALengthIsNotOneModule() throws IOException { - assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.empty())) - .isFalse(); - assertThat(FullBodyEncryptedDataPageV2.isSingleEncryptedModule(BytesInput.from(new byte[] {1, 2, 3}))) - .isFalse(); + public void bodyWithPlaintextLevelsIsRecognizedWhenTheModuleLengthIsNotInOneBuffer() throws IOException { + byte[] body = concat(LEVELS, module(300)); + // The split is inside the length of the module + ByteBuffer first = ByteBuffer.wrap(body, 0, LEVELS.length + 2).slice(); + ByteBuffer second = ByteBuffer.wrap(body, LEVELS.length + 2, body.length - LEVELS.length - 2) + .slice(); + + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(first, second), LEVELS.length)) + .isEqualTo(Layout.PLAINTEXT_LEVELS); } @Test - public void bodyIsNotConsumedByTheCheck() throws IOException { - BytesInput body = BytesInput.from(module(40)); + public void bodyThatFitsBothLayoutsIsAmbiguous() throws IOException { + // Plaintext levels that start like the length of a module that spans the whole body + byte[] module = module(50); + int bodySize = LEVELS.length + module.length; + byte[] levels = ByteBuffer.allocate(LEVELS.length) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(bodySize - Integer.BYTES) + .array(); + byte[] body = concat(levels, module); + + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(body), levels.length)) + .isEqualTo(Layout.AMBIGUOUS); + } - FullBodyEncryptedDataPageV2.isSingleEncryptedModule(body); + @Test + public void bodyThatFitsNeitherLayoutIsTreatedAsASingleModuleSoThatDecryptionReportsIt() throws IOException { + byte[] body = concat(LEVELS, module(50)); + body[LEVELS.length] ^= 0x01; - assertThat(body.toByteArray()).isEqualTo(module(40)); + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(body), LEVELS.length)) + .isEqualTo(Layout.SINGLE_MODULE); + } + + @Test + public void bodyThatIsTooShortToHoldAModuleLengthIsNotAnythingElseThanASingleModule() throws IOException { + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.empty(), 1)) + .isEqualTo(Layout.SINGLE_MODULE); + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(new byte[] {1, 2, 3}), 1)) + .isEqualTo(Layout.SINGLE_MODULE); + // The levels take all of the body + assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(LEVELS), LEVELS.length)) + .isEqualTo(Layout.SINGLE_MODULE); } } diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py b/parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py index 39ac44eb6f..5db7f091b9 100644 --- a/parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py +++ b/parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py @@ -40,7 +40,6 @@ import pyarrow.parquet.encryption as pe NUM_ROWS = 500 -FOOTER_KEY = b"0123456789abcdef" FOOTER_KEY_ID = "footkey" From 5d8c446c2b3fd23a28291a7483b40f6e9e70f90d Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 5 Nov 2026 16:57:13 -0600 Subject: [PATCH 3/6] Split DataPageV2 reading into plaintext-levels and encrypted-levels paths; simplify Javadoc --- .../hadoop/ColumnChunkPageReadStore.java | 277 ++++++++---------- .../hadoop/ColumnChunkPageWriteStore.java | 3 +- .../hadoop/FullBodyEncryptedDataPageV2.java | 70 ++--- .../parquet/hadoop/ParquetFileReader.java | 9 +- .../parquet/hadoop/ParquetFileWriter.java | 39 +-- .../hadoop/rewrite/ParquetRewriter.java | 3 +- .../crypto/DataPageV2EncryptionTestData.java | 39 +-- .../TestDataPageV2EncryptionInterop.java | 77 ++--- .../keytools/mocks/Base64KmsClient.java | 5 +- .../TestFullBodyEncryptedDataPageV2.java | 10 +- 10 files changed, 200 insertions(+), 332 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java index 0b09834165..72e0916fd8 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java @@ -221,136 +221,65 @@ public DataPage visit(DataPageV1 dataPageV1) { @Override public DataPage visit(DataPageV2 dataPageV2) { if (dataPageV2 instanceof FullBodyEncryptedDataPageV2) { - return readFullBodyEncryptedPageV2((FullBodyEncryptedDataPageV2) dataPageV2, currentPageIndex); + return readPageV2WithEncryptedLevels( + (FullBodyEncryptedDataPageV2) dataPageV2, currentPageIndex); } - return readPageV2(dataPageV2, currentPageIndex); + return readPageV2WithPlaintextLevels(dataPageV2, currentPageIndex); } }); } /** - * Decrypts, if needed, and decompresses a DataPageV2 whose repetition and definition levels are - * separate from the data: the levels are plaintext and only the values are encrypted. + * Reads a page whose levels are not encrypted. If the column is encrypted, only the values are an + * encrypted module. */ - private DataPageV2 readPageV2(DataPageV2 dataPageV2, int currentPageIndex) { - if (!dataPageV2.isCompressed() && offsetIndex == null && null == blockDecryptor) { - return dataPageV2; + private DataPageV2 readPageV2WithPlaintextLevels(DataPageV2 page, int currentPageIndex) { + if (!page.isCompressed() && offsetIndex == null && null == blockDecryptor) { + return page; } - BytesInput pageBytes = dataPageV2.getData(); + boolean offHeap = decryptsOffHeap(); + BytesInput values = page.getData(); try { - if (options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer()) { - ByteBuffer byteBuffer = pageBytes.toByteBuffer(releaser); - if (!byteBuffer.isDirect()) { - throw new ParquetDecodingException("Expected a direct buffer"); - } - if (blockDecryptor != null) { - byteBuffer = blockDecryptor.decrypt(byteBuffer, dataPageAAD); - } - long compressedSize = byteBuffer.limit(); - if (dataPageV2.isCompressed()) { - int uncompressedSize = Math.toIntExact(dataPageV2.getUncompressedSize() - - dataPageV2.getDefinitionLevels().size() - - dataPageV2.getRepetitionLevels().size()); - ByteBuffer decompressedBuffer = options.getAllocator().allocate(uncompressedSize); - releaser.releaseLater(decompressedBuffer); - long start = System.nanoTime(); - decompressor.decompress(byteBuffer, (int) compressedSize, decompressedBuffer, uncompressedSize); - setDecompressMetrics(pageBytes, start); - decompressedBuffer.flip(); - pageBytes = BytesInput.from(decompressedBuffer); - } else { - pageBytes = BytesInput.from(byteBuffer); - } - } else { - if (null != blockDecryptor) { - pageBytes = BytesInput.from(blockDecryptor.decrypt(pageBytes.toByteArray(), dataPageAAD)); - } - if (dataPageV2.isCompressed()) { - int uncompressedSize = Math.toIntExact(dataPageV2.getUncompressedSize() - - dataPageV2.getDefinitionLevels().size() - - dataPageV2.getRepetitionLevels().size()); - long start = System.nanoTime(); - pageBytes = BytesInput.copy(decompressor.decompress(pageBytes, uncompressedSize)); - setDecompressMetrics(pageBytes, start); - } + if (null != blockDecryptor) { + values = BytesInput.from(decryptDataPageModule(values, offHeap)); + } else if (offHeap) { + values = BytesInput.from(toDirectBuffer(values)); + } + if (page.isCompressed()) { + int levelsByteLength = + Math.toIntExact(page.getRepetitionLevels().size() + + page.getDefinitionLevels().size()); + values = decompressValues(values, valuesUncompressedSize(page, levelsByteLength), offHeap); } } catch (IOException e) { throw new ParquetDecodingException("could not decompress page", e); } - return uncompressedPageV2( - dataPageV2, - dataPageV2.getRepetitionLevels(), - dataPageV2.getDefinitionLevels(), - pageBytes, - currentPageIndex); - } - - private DataPageV2 uncompressedPageV2( - DataPageV2 source, - BytesInput repetitionLevels, - BytesInput definitionLevels, - BytesInput values, - int currentPageIndex) { - final DataPageV2 decompressedPage; - if (offsetIndex == null) { - decompressedPage = DataPageV2.uncompressed( - source.getRowCount(), - source.getNullCount(), - source.getValueCount(), - repetitionLevels, - definitionLevels, - source.getDataEncoding(), - values, - source.getStatistics()); - } else { - decompressedPage = DataPageV2.uncompressed( - source.getRowCount(), - source.getNullCount(), - source.getValueCount(), - offsetIndex.getFirstRowIndex(currentPageIndex), - repetitionLevels, - definitionLevels, - source.getDataEncoding(), - values, - source.getStatistics()); - } - if (source.getCrc().isPresent()) { - decompressedPage.setCrc(source.getCrc().getAsInt()); - } - return decompressedPage; + page, page.getRepetitionLevels(), page.getDefinitionLevels(), values, currentPageIndex); } /** - * Reads a DataPageV2 whose body, repetition and definition levels included, was found to be one - * encrypted module. - * - *

If the layout of the page body could not be determined from the page alone, it may also be a - * page of the layout that earlier parquet-java versions wrote, with plaintext levels. Both - * layouts are tried then, the one of the specification first. AES_GCM_V1 is authenticated, so a - * page of the other layout fails the tag check. AES_GCM_CTR_V1 is not authenticated, and cannot - * tell the layouts apart. A page body of the specification layout, which starts with a random - * nonce, looks like one of the legacy layout only by a coincidence of about one in 2^32 on average, - * a little more often if the levels take fewer than four bytes. The legacy layout is preferred for - * that cipher. + * Reads a page whose levels and values are one encrypted module. If the layout is ambiguous, the page + * is decrypted as one module first, and read with plaintext levels if the GCM tag check fails. CTR + * has no tag, so ambiguous CTR pages are read with plaintext levels. */ - private DataPageV2 readFullBodyEncryptedPageV2(FullBodyEncryptedDataPageV2 page, int currentPageIndex) { + private DataPageV2 readPageV2WithEncryptedLevels(FullBodyEncryptedDataPageV2 page, int currentPageIndex) { DataPageV2 plaintextLevelsPage = page.getPlaintextLevelsLayoutPage(); if (plaintextLevelsPage == null) { - return readSingleEncryptedModulePageV2(page, currentPageIndex, false); + return decryptPageV2WithEncryptedLevels(page, currentPageIndex, decryptsOffHeap()); } if (blockDecryptor instanceof AesCtrDecryptor) { - return readPageV2(plaintextLevelsPage, currentPageIndex); + return readPageV2WithPlaintextLevels(plaintextLevelsPage, currentPageIndex); } try { - // Decrypting on the heap leaves the encrypted buffers untouched for the second attempt - return readSingleEncryptedModulePageV2(page, currentPageIndex, true); + // Not off heap: decrypting in place would overwrite the bytes the fallback reads + return decryptPageV2WithEncryptedLevels(page, currentPageIndex, false); } catch (TagVerificationException notASingleModule) { LOG.debug( "DataPageV2 body is not a single encrypted module, reading it with plaintext levels", notASingleModule); try { - return readPageV2(plaintextLevelsPage, currentPageIndex); + return readPageV2WithPlaintextLevels(plaintextLevelsPage, currentPageIndex); } catch (TagVerificationException notPlaintextLevelsEither) { notPlaintextLevelsEither.addSuppressed(notASingleModule); throw notPlaintextLevelsEither; @@ -358,65 +287,26 @@ private DataPageV2 readFullBodyEncryptedPageV2(FullBodyEncryptedDataPageV2 page, } } - /** - * Decrypts a DataPageV2 whose body, repetition and definition levels included, is one encrypted - * module, splits the levels from the values, and decompresses the values. - */ - private DataPageV2 readSingleEncryptedModulePageV2( - FullBodyEncryptedDataPageV2 page, int currentPageIndex, boolean forceHeap) { - final int repetitionLevelsByteLength = page.getRepetitionLevelsByteLength(); - final int definitionLevelsByteLength = page.getDefinitionLevelsByteLength(); - final int levelsByteLength = repetitionLevelsByteLength + definitionLevelsByteLength; - final int valuesUncompressedSize = Math.toIntExact(page.getUncompressedSize() - levelsByteLength); - BytesInput repetitionLevels; - BytesInput definitionLevels; - BytesInput values; + /** Decrypts the body, splits it into levels and values, and decompresses the values. */ + private DataPageV2 decryptPageV2WithEncryptedLevels( + FullBodyEncryptedDataPageV2 page, int currentPageIndex, boolean offHeap) { + int repetitionLevelsByteLength = page.getRepetitionLevelsByteLength(); + int definitionLevelsByteLength = page.getDefinitionLevelsByteLength(); + int levelsByteLength = repetitionLevelsByteLength + definitionLevelsByteLength; try { - BytesInput encryptedBody = page.getData(); - if (!forceHeap && options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer()) { - ByteBuffer encrypted = encryptedBody.toByteBuffer(releaser); - if (!encrypted.isDirect()) { - throw new ParquetDecodingException("Expected a direct buffer"); - } - // The decrypted bytes are laid out as [repetition levels | definition levels | values] - ByteBuffer decrypted = blockDecryptor.decrypt(encrypted, dataPageAAD); - checkDecryptedPageSize(decrypted.limit(), levelsByteLength); - repetitionLevels = BytesInput.from(decrypted.slice(0, repetitionLevelsByteLength)); - definitionLevels = - BytesInput.from(decrypted.slice(repetitionLevelsByteLength, definitionLevelsByteLength)); - ByteBuffer valuesBuffer = decrypted.slice(levelsByteLength, decrypted.limit() - levelsByteLength); - if (page.isCompressed()) { - ByteBuffer decompressedBuffer = options.getAllocator().allocate(valuesUncompressedSize); - releaser.releaseLater(decompressedBuffer); - // The decompressor consumes the buffer, so the compressed size is taken before it runs - BytesInput compressedValues = BytesInput.from(valuesBuffer.duplicate()); - long start = System.nanoTime(); - decompressor.decompress( - valuesBuffer, valuesBuffer.limit(), decompressedBuffer, valuesUncompressedSize); - setDecompressMetrics(compressedValues, start); - decompressedBuffer.flip(); - values = BytesInput.from(decompressedBuffer); - } else { - values = BytesInput.from(valuesBuffer); - } - } else { - byte[] decrypted = blockDecryptor.decrypt(encryptedBody.toByteArray(), dataPageAAD); - checkDecryptedPageSize(decrypted.length, levelsByteLength); - repetitionLevels = BytesInput.from(decrypted, 0, repetitionLevelsByteLength); - definitionLevels = - BytesInput.from(decrypted, repetitionLevelsByteLength, definitionLevelsByteLength); - values = BytesInput.from(decrypted, levelsByteLength, decrypted.length - levelsByteLength); - if (page.isCompressed()) { - long start = System.nanoTime(); - BytesInput compressedValues = values; - values = BytesInput.copy(decompressor.decompress(compressedValues, valuesUncompressedSize)); - setDecompressMetrics(compressedValues, start); - } + ByteBuffer body = decryptDataPageModule(page.getData(), offHeap); + checkDecryptedPageSize(body.limit(), levelsByteLength); + BytesInput repetitionLevels = BytesInput.from(body.slice(0, repetitionLevelsByteLength)); + BytesInput definitionLevels = + BytesInput.from(body.slice(repetitionLevelsByteLength, definitionLevelsByteLength)); + BytesInput values = BytesInput.from(body.slice(levelsByteLength, body.limit() - levelsByteLength)); + if (page.isCompressed()) { + values = decompressValues(values, valuesUncompressedSize(page, levelsByteLength), offHeap); } + return uncompressedPageV2(page, repetitionLevels, definitionLevels, values, currentPageIndex); } catch (IOException e) { throw new ParquetDecodingException("could not decompress page", e); } - return uncompressedPageV2(page, repetitionLevels, definitionLevels, values, currentPageIndex); } private void checkDecryptedPageSize(int decryptedSize, int levelsByteLength) { @@ -427,6 +317,83 @@ private void checkDecryptedPageSize(int decryptedSize, int levelsByteLength) { } } + private static int valuesUncompressedSize(DataPageV2 page, int levelsByteLength) { + return Math.toIntExact(page.getUncompressedSize() - levelsByteLength); + } + + private boolean decryptsOffHeap() { + return options.getAllocator().isDirect() && options.useOffHeapDecryptBuffer(); + } + + /** Decrypts in place if off heap, otherwise into a new heap buffer. */ + private ByteBuffer decryptDataPageModule(BytesInput module, boolean offHeap) throws IOException { + if (offHeap) { + return blockDecryptor.decrypt(toDirectBuffer(module), dataPageAAD); + } + return ByteBuffer.wrap(blockDecryptor.decrypt(module.toByteArray(), dataPageAAD)); + } + + private ByteBuffer toDirectBuffer(BytesInput bytes) { + ByteBuffer buffer = bytes.toByteBuffer(releaser); + if (!buffer.isDirect()) { + throw new ParquetDecodingException("Expected a direct buffer"); + } + return buffer; + } + + private BytesInput decompressValues(BytesInput compressed, int uncompressedSize, boolean offHeap) + throws IOException { + long start = System.nanoTime(); + BytesInput decompressed; + if (offHeap) { + ByteBuffer input = toDirectBuffer(compressed); + ByteBuffer output = options.getAllocator().allocate(uncompressedSize); + releaser.releaseLater(output); + decompressor.decompress(input, input.limit(), output, uncompressedSize); + output.flip(); + decompressed = BytesInput.from(output); + } else { + decompressed = BytesInput.copy(decompressor.decompress(compressed, uncompressedSize)); + } + setDecompressMetrics(compressed, start); + return decompressed; + } + + private DataPageV2 uncompressedPageV2( + DataPageV2 source, + BytesInput repetitionLevels, + BytesInput definitionLevels, + BytesInput values, + int currentPageIndex) { + final DataPageV2 decompressedPage; + if (offsetIndex == null) { + decompressedPage = DataPageV2.uncompressed( + source.getRowCount(), + source.getNullCount(), + source.getValueCount(), + repetitionLevels, + definitionLevels, + source.getDataEncoding(), + values, + source.getStatistics()); + } else { + decompressedPage = DataPageV2.uncompressed( + source.getRowCount(), + source.getNullCount(), + source.getValueCount(), + offsetIndex.getFirstRowIndex(currentPageIndex), + repetitionLevels, + definitionLevels, + source.getDataEncoding(), + values, + source.getStatistics()); + } + if (source.getCrc().isPresent()) { + decompressedPage.setCrc(source.getCrc().getAsInt()); + } + return decompressedPage; + } + private void setDecompressMetrics(BytesInput bytes, long start) { final ParquetMetricsCallback metricsCallback = options.getMetricsCallback(); if (metricsCallback != null) { diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java index f1536ec10a..32f2ceb804 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java @@ -317,8 +317,7 @@ public void writePageV2( compressedData = compressor.compress(data); compressed = true; } - // The bytes that follow the page header. With encryption, the levels and the values of a v2 page - // are encrypted together, as the one "data page" module of the page. + // When encrypted, the levels and the values are encrypted together as one module List pageBody = new ArrayList<>(3); if (null != pageBlockEncryptor) { AesCipher.quickUpdatePageAAD(dataPageAAD, pageOrdinal); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java index 9c96962702..36fc1771ea 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java @@ -28,30 +28,19 @@ import org.apache.parquet.column.statistics.Statistics; /** - * A still encrypted {@link DataPageV2} whose entire page body is, or if its layout is {@link - * Layout#AMBIGUOUS} may be, a single encrypted module. - * - *

Parquet Modular Encryption encrypts the "data page" module, which for a DataPageV2 page is - * the concatenation {@code [repetition levels | definition levels | (compressed) values]}. This is - * what Arrow C++ and arrow-rs write, and what parquet-java writes since 1.19.0. Earlier - * parquet-java versions left the levels in plaintext and encrypted only the values; those pages - * are still read as plain {@link DataPageV2} pages whose levels are already separated out. - * - *

The page body can only be decrypted, and thereby split into levels and values, once the page - * AAD is known. That happens in {@code ColumnChunkPageReader}. Until then the whole encrypted - * module is held as the {@link #getData() data} of this page, and the repetition and definition - * levels are empty: they must not be used before the page has been decrypted. The sizes of the - * levels, which the page header carries, are kept separately. + * An encrypted DataPageV2 whose body, levels included, is one encrypted module. The body is held as + * {@link #getData()} and the levels are empty until {@code ColumnChunkPageReader} decrypts the page + * and splits the body using the level lengths. */ final class FullBodyEncryptedDataPageV2 extends DataPageV2 { /** How the body of an encrypted DataPageV2 page that has levels is laid out. */ enum Layout { - /** The whole body is one encrypted module, as in the specification. */ + /** {@code [module of levels and values]} */ SINGLE_MODULE, - /** The levels are in plaintext, followed by an encrypted module of the values. */ + /** {@code [plaintext levels][module of values]} */ PLAINTEXT_LEVELS, - /** The body fits both layouts, which can only be told apart by trying to decrypt it. */ + /** Both length checks match; only decryption can tell. */ AMBIGUOUS } @@ -60,11 +49,10 @@ enum Layout { private final DataPageV2 plaintextLevelsLayoutPage; /** - * @param encryptedBody the encrypted page body, in the serialized form of an encrypted module: - * {@code length (4 bytes) | nonce | ciphertext [| tag]} - * @param uncompressedSize the uncompressed page size of the page header, levels included - * @param plaintextLevelsLayoutPage the same page with the levels separated out, if the page - * body is {@link Layout#AMBIGUOUS}, otherwise null + * @param encryptedBody {@code length (4 bytes, LE) | nonce | ciphertext [| tag]} + * @param uncompressedSize uncompressed page size, levels included + * @param plaintextLevelsLayoutPage the page split as {@link Layout#PLAINTEXT_LEVELS}, if the layout is + * {@link Layout#AMBIGUOUS}; otherwise null */ FullBodyEncryptedDataPageV2( int rowCount, @@ -102,43 +90,27 @@ int getDefinitionLevelsByteLength() { return definitionLevelsByteLength; } - /** - * @return the page as it would be if its body had the {@link Layout#PLAINTEXT_LEVELS} layout, to - * fall back to when decrypting it as a single module fails. Null unless the layout of the - * page body is {@link Layout#AMBIGUOUS}. - */ + /** @return the page split as {@link Layout#PLAINTEXT_LEVELS}, or null if the layout is not ambiguous */ DataPageV2 getPlaintextLevelsLayoutPage() { return plaintextLevelsLayoutPage; } /** - * Tells which of the two layouts the body of an encrypted DataPageV2 page has. It must only be - * called for a page that has repetition or definition levels: without levels both layouts are - * byte-for-byte identical. - * - *

An encrypted module starts with its own length, a 4 byte little endian integer holding the - * size of the rest of the module. Each layout therefore announces itself: + * Detects the layout of the body of a page that has levels. A module starts with the length of the + * rest of the module as a 4 byte LE int, so: * *

    - *
  • if the page body is a single module, the first four bytes hold the size of the body minus - * four; - *
  • if the levels are stored in plaintext, the module of the values starts right after the - * levels, and the four bytes there hold the size of the body, minus the size of the levels, - * minus four. + *
  • {@link Layout#SINGLE_MODULE}: the int at offset 0 is {@code size - 4} + *
  • {@link Layout#PLAINTEXT_LEVELS}: the int at offset {@code levelsByteLength} is {@code size - + * levelsByteLength - 4} + *
  • {@link Layout#AMBIGUOUS}: both hold *
* - *

A page body that is a single module has a random nonce and ciphertext after its length, so - * it matches the second condition only by a coincidence of about one in 2^32, a little more often if - * the levels take fewer than four bytes, as the four bytes then overlap the length of the module. A page body that has - * plaintext levels starts with RLE encoded levels, so it matches the first condition only if - * those happen to spell out the size of the page. When both conditions are met, the layout can - * only be found by decrypting the page, see {@link Layout#AMBIGUOUS}. When neither is met the page - * is corrupt, and it is treated as a single module so that decryption reports the problem. + * If neither holds, the page is corrupt and {@link Layout#SINGLE_MODULE} is returned, so that + * decryption fails. * - * @param pageBody the page body, as sized by the {@code compressed_page_size} of the page header - * @param levelsByteLength the size of the repetition and definition levels, which is above zero - * @return the layout of the page body - * @throws IOException if the page body cannot be read + * @param pageBody the page body, of {@code compressed_page_size} bytes + * @param levelsByteLength the length of the repetition and definition levels, above zero */ static Layout detectLayout(BytesInput pageBody, int levelsByteLength) throws IOException { long size = pageBody.size(); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 56ea915e1b..0bac86af3d 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -2275,12 +2275,9 @@ private int getPageOrdinal(int dataPageCountReadSoFar) { } /** - * Reads the DataPageV2 page of which the header has just been read. The page of an encrypted column - * that has repetition or definition levels is read as a {@link FullBodyEncryptedDataPageV2} if its - * body is, or may be, one encrypted module that also covers the levels, which is the layout of the Parquet - * encryption specification, written by Arrow and by parquet-java since 1.19.0. Files written by - * earlier parquet-java versions have the levels in plaintext in front of the module of the - * values; these pages are read as plain {@link DataPageV2} pages that are still encrypted. + * Reads a DataPageV2 body. For an encrypted column with levels, returns a {@link + * FullBodyEncryptedDataPageV2} if the levels are inside the encrypted module, or a {@link + * DataPageV2} with the plaintext levels split out otherwise. */ private DataPageV2 readDataPageV2(PageHeader pageHeader, PrimitiveType type, boolean encryptedColumn) throws IOException { diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index f8048efd52..7cdf77d9fc 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -1169,10 +1169,8 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page. The repetition and definition levels and the data bytes are written - * as given: nothing is encrypted. The page of an encrypted column has its levels in the encrypted - * page body, so the data bytes of such a page must not be encrypted by the caller; use the overload - * that takes a page block encryptor. + * Writes a single v2 data page. The levels and data bytes are written as given, without encryption. + * For encrypted columns, use the overload that takes a page block encryptor. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1262,10 +1260,8 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page. The repetition and definition levels and the data bytes are written - * as given: nothing is encrypted. The page of an encrypted column has its levels in the encrypted - * page body, so the data bytes of such a page must not be encrypted by the caller; use the overload - * that takes a page block encryptor. + * Writes a single v2 data page. The levels and data bytes are written as given, without encryption. + * For encrypted columns, use the overload that takes a page block encryptor. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1361,10 +1357,8 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page. The repetition and definition levels are written as given, in - * front of the data bytes, and the data bytes are written as given too: they are not encrypted. - * To write the page of an encrypted column, whose levels are part of the encrypted page body, use - * the overload that takes a page block encryptor. + * Writes a single v2 data page. The levels and data bytes are written as given, without encryption. + * For encrypted columns, use the overload that takes a page block encryptor. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1413,14 +1407,8 @@ public void writeDataPageV2( } /** - * Writes a single v2 data page of a column that is encrypted, if a page block encryptor is given. - * - *

With Parquet Modular Encryption the data page module of a v2 page is the whole page body: the - * repetition levels, the definition levels and the (possibly compressed) data are encrypted - * together, as one module. Without a page block encryptor this method behaves like {@link - * #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, - * Statistics, BlockCipher.Encryptor, byte[], SizeStatistics)}, which writes the levels as given and - * does not encrypt the data. + * Writes a single v2 data page. If {@code pageBlockEncryptor} is not null, the repetition levels, + * definition levels and data bytes are encrypted together as one module. * * @param rowCount count of rows * @param nullCount count of nulls @@ -1428,7 +1416,7 @@ public void writeDataPageV2( * @param repetitionLevels repetition level bytes * @param definitionLevels definition level bytes * @param dataEncoding encoding for data - * @param bytes data bytes, not encrypted + * @param bytes data bytes, not yet encrypted * @param compressed whether the data bytes is compressed * @param uncompressedDataSize the size of uncompressed data * @param statistics the statistics of the page @@ -1436,9 +1424,8 @@ public void writeDataPageV2( * @param pageHeaderAAD pageHeader AAD * @param sizeStatistics size statistics for the page * @param pageBlockEncryptor encryptor for the page body, or null if the column is not encrypted - * @param dataPageAAD AAD of the data page module of this page, that is with the ordinal of the page, - * see {@link AesCipher#quickUpdatePageAAD}; it is required with a page block - * encryptor, and ignored without one + * @param dataPageAAD data page AAD, updated for this page's ordinal; required if + * {@code pageBlockEncryptor} is not null * @throws IOException if any I/O error occurs during writing the file */ public void writeDataPageV2( @@ -1482,9 +1469,7 @@ public void writeDataPageV2( sizeStatistics); } - /** - * @param pageBody the bytes that follow the page header, in the order they are written - */ + /** @param pageBody the bytes written after the page header */ private void writeDataPageV2Body( int rowCount, int nullCount, diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java index 212bd75254..9a868e3124 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java @@ -738,8 +738,7 @@ private void processChunk( BytesInput dlLevels = readBlockAllocate(dlLength, reader); int payLoadLength = pageHeader.getCompressed_page_size() - rlLength - dlLength; int rawDataLength = pageHeader.getUncompressed_page_size() - rlLength - dlLength; - // The payload is not encrypted here: for an encrypted column the writer encrypts it together - // with the levels. + // Not encrypted here: the writer encrypts the levels and the payload together pageLoad = processPageLoad( reader, headerV2.is_compressed, diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java index 3764607a38..bfacf3f2d1 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java @@ -38,22 +38,15 @@ import org.apache.parquet.schema.MessageTypeParser; /** - * Deterministic data, schema and keys shared by the DataPageV2 encryption tests and by the scripts - * that generated the fixture files under {@code src/test/resources/encryption/datapagev2}. - * - *

The schema deliberately mixes a required column (no repetition / definition levels), optional - * columns (definition levels only) and a nested list (repetition and definition levels), because - * the layout of an encrypted DataPageV2 only differs between implementations when a page has - * non-empty levels. - * - *

The row generator must stay in sync with {@code generate_arrow_fixtures.py} in the fixtures - * directory. + * Schema, rows and keys of the DataPageV2 encryption tests and fixtures. {@code seq} is required (no + * levels), {@code id} and {@code name} are optional (definition levels), {@code tags} is a list + * (repetition and definition levels). */ final class DataPageV2EncryptionTestData { public static final int NUM_ROWS = 500; - /** The footer key; with uniform encryption it is also used for all of the columns. */ + /** Footer key, also the key of all columns with uniform encryption. */ public static final byte[] FOOTER_KEY = "0123456789abcdef".getBytes(StandardCharsets.UTF_8); public static final byte[] FOOTER_KEY_METADATA = "footkey".getBytes(StandardCharsets.UTF_8); @@ -102,12 +95,11 @@ public static List expectedRows() { return rows; } - /** Key of the columns that are encrypted with their own key, in the tests that do that. */ public static final byte[] COLUMN_KEY = "fedcba9876543210".getBytes(StandardCharsets.UTF_8); public static final byte[] COLUMN_KEY_METADATA = "colkey".getBytes(StandardCharsets.UTF_8); - /** Decryption properties for the files that were written with {@link #FOOTER_KEY} and {@link #COLUMN_KEY}. */ + /** Resolves the key metadata of {@link #FOOTER_KEY} and {@link #COLUMN_KEY}. */ public static FileDecryptionProperties decryptionProperties() { return FileDecryptionProperties.builder() .withKeyRetriever(keyMetadata -> { @@ -125,13 +117,11 @@ public static FileDecryptionProperties decryptionProperties() { } /** - * Reads all of the rows of a file, rendered as strings, with the verification of the page checksums - * switched on. + * Reads all rows as strings. Page checksums are verified unless {@code conf} disables it. * - * @param conf may hold the configuration of a decryption properties factory, and whether to verify the - * page checksums, which is the default; it is not modified - * @param decryption the decryption properties, or null to take them from the configuration - * @param directMemory whether to read the file into direct buffers and decrypt it off-heap + * @param conf not modified + * @param decryption the decryption properties, or null to use the ones configured in {@code conf} + * @param directMemory whether to read into direct buffers and decrypt off heap */ public static List readRows( Path file, Configuration conf, FileDecryptionProperties decryption, boolean directMemory) @@ -139,11 +129,7 @@ public static List readRows( return readRows(file, conf, decryption, directMemory, FilterCompat.NOOP); } - /** - * Like {@link #readRows(Path, Configuration, FileDecryptionProperties, boolean)}, for the rows that - * pass the filter. With a filter, the pages that cannot hold matching rows are skipped by way of the - * offset index. - */ + /** Like {@link #readRows(Path, Configuration, FileDecryptionProperties, boolean)}, filtered by page. */ public static List readRows( Path file, Configuration conf, @@ -157,12 +143,11 @@ public static List readRows( ParquetInputFormat.PAGE_VERIFY_CHECKSUM_ENABLED, conf.getBoolean(ParquetInputFormat.PAGE_VERIFY_CHECKSUM_ENABLED, true)); if (filter != FilterCompat.NOOP) { - // Without this the rows of the pages that are read would be filtered too, which hides whether - // the pages that cannot hold matching rows were skipped + // Return every row of the pages that are read, so the test can see which pages were skipped readConf.setBoolean(ParquetInputFormat.RECORD_FILTERING_ENABLED, false); } List rows = new ArrayList<>(); - // Closing the allocator fails the test if the reader leaks a buffer + // close() fails on leaked buffers try (TrackingByteBufferAllocator allocator = TrackingByteBufferAllocator.wrap( directMemory ? new DirectByteBufferAllocator() : new HeapByteBufferAllocator())) { ParquetReader.Builder builder = ParquetReader.builder(new GroupReadSupport(), file) diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java index 276e67eaa8..d5e82f28e4 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java @@ -93,35 +93,22 @@ import org.junit.jupiter.params.provider.MethodSource; /** - * Interoperability of the encryption of DataPageV2 pages that have repetition or definition levels. + * Encryption of DataPageV2 pages with repetition or definition levels (apache/parquet-java#3934). * - *

The page body of such a page, the levels included, is a single encrypted "data page" module. - * Arrow C++ and arrow-rs write it that way, and so does parquet-java. Earlier parquet-java versions - * left the levels in plaintext and encrypted only the values (apache/parquet-java#3934), so files - * in both layouts exist. These tests make sure that + *

Fixtures in {@code src/test/resources/encryption/datapagev2}: * *

    - *
  • parquet-java writes only the single module layout, which an independent decoder written - * from the specification confirms, also when it rewrites files; - *
  • parquet-java reads the files that Arrow wrote, which the {@code arrow-v2-*} fixtures are; - *
  • parquet-java still reads the files that it wrote in the old layout, which the {@code - * java-legacy-v2-*} fixtures are. They were written by parquet-java 1.19.0-SNAPSHOT at commit - * e265ac7f5, just before the layout changed, with the footer key as the key of all columns, - * the same rows as {@link DataPageV2EncryptionTestData#row(int)}, 100 rows per page and - * {@code ExampleParquetWriter} with {@code WriterVersion.PARQUET_2_0}. To regenerate them, - * check out that commit, and write the rows with the {@code writeWithParquetJava} method of - * this class for each cipher and codec. The Arrow files come from {@code - * generate_arrow_fixtures.py}, which sits next to them. + *
  • {@code arrow-v2-*}: written by {@code generate_arrow_fixtures.py} (pyarrow). + *
  • {@code java-legacy-v2-*}: written with {@link #writeWithParquetJava} at commit e265ac7f5, + * before the levels were encrypted. To regenerate, check out that commit and run it for each + * cipher and codec. *
*/ public class TestDataPageV2EncryptionInterop { private static final String FIXTURES = "/encryption/datapagev2/"; - /** - * Matches the second half of the rows. With 100 rows per page, the first two pages cannot hold a - * match and are skipped, so the rows that are read start at the third page. - */ + /** {@code seq >= 250} lets the first two pages (rows 0-199) be skipped. */ private static final int FIRST_ROW_OF_PAGE_WITH_SEQ_250 = 200; private static final FilterCompat.Filter SEQ_AT_LEAST_250 = @@ -133,8 +120,7 @@ public class TestDataPageV2EncryptionInterop { @BeforeEach @AfterEach public void clearKmsClients() { - // The KMS clients are cached by instance id, across the tests of a JVM, and the Arrow fixtures use - // the default one + // KMS clients are cached per JVM by instance id, and the Arrow fixtures use the default id KeyToolkit.removeCacheEntriesForAllTokens(); } @@ -183,7 +169,6 @@ static Stream ciphersAndCodecs() { @ParameterizedTest(name = "{0}, directMemory={1}") @MethodSource("arrowFixtures") public void readsFilesWrittenByArrow(String fixture, boolean directMemory) throws Exception { - // The keys of the Arrow files are wrapped by the KMS client that generate_arrow_fixtures.py mirrors Configuration conf = new Configuration(); conf.set( EncryptionPropertiesFactory.CRYPTO_FACTORY_CLASS_PROPERTY_NAME, @@ -203,7 +188,6 @@ public void readsFilesWrittenByEarlierParquetJavaVersions(String fixture, boolea @ParameterizedTest(name = "{0}") @MethodSource("legacyFixtureNames") public void legacyFixturesHaveThePlaintextLevelsLayout(String fixture) throws Exception { - // The test above only proves the legacy code path is covered if the fixtures really are legacy files List pages = decodeDataPagesV2(fixture(fixture), FOOTER_KEY, false); assertThat(pages).isNotEmpty(); @@ -247,7 +231,7 @@ public void readsWhatItWrites(ParquetCipher cipher, CompressionCodecName codec) @Test public void detectsTamperingWithTheLevels() throws Exception { Path file = writeFileWithTamperedLevels(); - // The page checksum would notice the change first, which is not what this test is about + // Otherwise the page checksum fails first Configuration withoutChecksums = new Configuration(); withoutChecksums.setBoolean(ParquetInputFormat.PAGE_VERIFY_CHECKSUM_ENABLED, false); @@ -271,15 +255,12 @@ public void pageChecksumDetectsTamperingWithTheLevels() throws Exception { .hasMessage("could not verify page integrity, CRC checksum verification failed"); } - /** - * Writes a file of which the first byte of the levels of the first DataPageV2 page was changed. When - * the levels were stored in plaintext, changing them went unnoticed by the decryption. - */ + /** Writes a file with one bit of the levels of the first DataPageV2 page flipped. */ private Path writeFileWithTamperedLevels() throws Exception { Path file = writeWithParquetJava(ParquetCipher.AES_GCM_V1, CompressionCodecName.UNCOMPRESSED); DecodedPage page = decodeDataPagesV2(file, FOOTER_KEY, false).get(0); - // Flip a bit of the first byte of the ciphertext, which is the first byte of the levels + // First ciphertext byte, which is the first byte of the levels File source = new File(file.toUri()); try (RandomAccessFile raf = new RandomAccessFile(source, "rw")) { long position = page.bodyOffset + Integer.BYTES + AesCipher.NONCE_LENGTH; @@ -288,7 +269,7 @@ private Path writeFileWithTamperedLevels() throws Exception { raf.seek(position); raf.write(original ^ 0x01); } - // Drop the local filesystem CRC side-file so the tampered bytes reach the Parquet reader. + // Otherwise the local filesystem rejects the file Files.deleteIfExists(new File(source.getParent(), "." + source.getName() + ".crc").toPath()); return file; } @@ -376,19 +357,13 @@ public void skipsPagesOfFilesWrittenByEarlierParquetJavaVersions(String fixture, .containsExactlyElementsOf(expectedRows().subList(FIRST_ROW_OF_PAGE_WITH_SEQ_250, NUM_ROWS)); } - /** - * A page of the legacy layout whose plaintext levels happen to start with what the length of a - * module that covers the page body would be, so that the layout cannot be told from the page body - * alone. With AES_GCM_V1 the tag check settles it, AES_GCM_CTR_V1 is not authenticated, and the - * legacy layout is read. - */ + /** A legacy page whose first 4 level bytes equal {@code bodySize - 4}, so it is {@code AMBIGUOUS}. */ @ParameterizedTest(name = "{0}, directMemory={1}") @MethodSource("ciphersAndMemoryModes") public void readsALegacyPageThatLooksLikeASingleModule(ParquetCipher cipher, boolean directMemory) throws Exception { MessageType schema = MessageTypeParser.parseMessageType("message schema { optional int32 v; }"); - // Definition levels: a bit-packed run of one group of 8 values of which the first is defined, and - // two more bytes, which the reader does not look at. Read as an integer they are 0x0103 = 259 + // Bit-packed run of 8 levels, only the first defined, plus 2 unread bytes; as an LE int 0x0103 = 259 byte[] levels = {0x03, 0x01, 0x00, 0x00}; int bodySize = 259 + Integer.BYTES; int moduleSize = bodySize - levels.length; @@ -416,7 +391,7 @@ public void readsALegacyPageThatLooksLikeASingleModule(ParquetCipher cipher, boo writer.start(); writer.startBlock(8); writer.startColumn(column, 8, CompressionCodecName.UNCOMPRESSED); - // What parquet-java used to write: the levels as they are, in front of the encrypted values + // Legacy layout: plaintext levels, then the encrypted values writer.writeDataPageV2( 8, 7, @@ -443,7 +418,7 @@ public void readsALegacyPageThatLooksLikeASingleModule(ParquetCipher cipher, boo assertThat(rows.subList(1, 8)).allSatisfy(row -> assertThat(row).doesNotContain("v:")); } - /** Writes a DataPageV2 file with 100 rows per page, encrypted with the footer key if there is a cipher. */ + /** Writes 100 rows per page; encrypts all columns with the footer key if {@code cipher} is set. */ private Path writeWithParquetJava(ParquetCipher cipher, CompressionCodecName codec) throws IOException { return writeWithParquetJava(cipher, codec, ParquetWriter.DEFAULT_BLOCK_SIZE); } @@ -478,7 +453,7 @@ private static Path fixture(String name) throws URISyntaxException { .toURI()); } - /** What the reference decoder found out about one DataPageV2 page that has levels. */ + /** A DataPageV2 with levels, as found by {@link #decodeDataPagesV2}. */ private static final class DecodedPage { private final String column; private final int repetitionLevelsLength; @@ -517,20 +492,13 @@ public String toString() { } /** - * A decoder of the encrypted DataPageV2 pages of a file, written from the Parquet encryption - * specification and sharing nothing with the reader of parquet-java except the cipher - * implementations: it parses and decrypts the page headers itself, takes the data page module - * from the page body, and decrypts it with the AAD of the specification. - * - *

As the specification has the page body start with the length of the encrypted module, the body - * is a single module when the first four bytes hold the body length minus four. Otherwise the - * levels are in plaintext and the module of the values follows them. That is enough for the files - * of these tests, which have no page for which both could be true. + * Decodes the DataPageV2 pages of a file without the parquet-java page reader: parses and decrypts + * the page headers, then decrypts the page body as one module if its first 4 bytes are {@code + * size - 4}, otherwise the bytes after the levels. * * @param file file with an encrypted footer, whose encrypted columns all use {@code columnKey} - * @param onlyEncryptedColumns true to skip the columns that are not encrypted, for a file where - * only some of the columns are; false if all of the columns are encrypted - * @return the pages that have repetition or definition levels, in file order + * @param onlyEncryptedColumns whether to skip columns that are not encrypted + * @return the pages that have levels, in file order */ private static List decodeDataPagesV2(Path file, byte[] columnKey, boolean onlyEncryptedColumns) throws IOException { @@ -596,7 +564,6 @@ private static List decodeDataPagesV2(Path file, byte[] columnKey, boolean singleModule = BytesUtils.readIntLittleEndian(body, 0) == body.length - Integer.BYTES; byte[] module = singleModule ? body : Arrays.copyOfRange(body, levels, body.length); - // Either way, what is decrypted starts with its own length assertThat(BytesUtils.readIntLittleEndian(module, 0)) .isEqualTo(module.length - Integer.BYTES); byte[] pageAAD = AesCipher.createModuleAAD( diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java index c84f5c7e37..908ba7614f 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java @@ -24,10 +24,7 @@ import org.apache.parquet.crypto.KeyAccessDeniedException; import org.apache.parquet.crypto.keytools.KmsClient; -/** - * A KMS client that "wraps" a key by base64 encoding it. It is not secure, it only exists to read - * the fixtures written by {@code generate_arrow_fixtures.py}, whose key wrapping it mirrors. - */ +/** Wraps keys by base64 encoding them, like the KMS client of {@code generate_arrow_fixtures.py}. */ public class Base64KmsClient implements KmsClient { @Override diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java index 7095163df0..1cbe9c0870 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestFullBodyEncryptedDataPageV2.java @@ -30,10 +30,10 @@ public class TestFullBodyEncryptedDataPageV2 { - /** RLE encoded levels: a bit-packed run of one group, 0xff, followed by an RLE run of 8 values of 1. */ + /** A bit-packed run of 8 levels, then an RLE run of 8 levels. */ private static final byte[] LEVELS = {0x03, (byte) 0xff, 0x10, 0x01}; - /** The serialized form of an encrypted module: its length, then that many bytes. */ + /** {@code contentLength} as a 4 byte LE int, followed by that many bytes. */ private static byte[] module(int contentLength) { ByteBuffer module = ByteBuffer.allocate(Integer.BYTES + contentLength).order(ByteOrder.LITTLE_ENDIAN); module.putInt(contentLength); @@ -77,7 +77,7 @@ public void bodyThatIsOneModuleIsRecognizedWhenItIsNotInOneBuffer() throws IOExc @Test public void bodyWithPlaintextLevelsIsRecognizedWhenTheModuleLengthIsNotInOneBuffer() throws IOException { byte[] body = concat(LEVELS, module(300)); - // The split is inside the length of the module + // Split inside the module length ByteBuffer first = ByteBuffer.wrap(body, 0, LEVELS.length + 2).slice(); ByteBuffer second = ByteBuffer.wrap(body, LEVELS.length + 2, body.length - LEVELS.length - 2) .slice(); @@ -88,7 +88,7 @@ public void bodyWithPlaintextLevelsIsRecognizedWhenTheModuleLengthIsNotInOneBuff @Test public void bodyThatFitsBothLayoutsIsAmbiguous() throws IOException { - // Plaintext levels that start like the length of a module that spans the whole body + // Levels whose first 4 bytes are bodySize - 4 byte[] module = module(50); int bodySize = LEVELS.length + module.length; byte[] levels = ByteBuffer.allocate(LEVELS.length) @@ -116,7 +116,7 @@ public void bodyThatIsTooShortToHoldAModuleLengthIsNotAnythingElseThanASingleMod .isEqualTo(Layout.SINGLE_MODULE); assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(new byte[] {1, 2, 3}), 1)) .isEqualTo(Layout.SINGLE_MODULE); - // The levels take all of the body + // Body is only levels assertThat(FullBodyEncryptedDataPageV2.detectLayout(BytesInput.from(LEVELS), LEVELS.length)) .isEqualTo(Layout.SINGLE_MODULE); } From e12a5cb48ef51292f48a3a4f99fd2d1196ea2b23 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 5 Nov 2026 17:07:49 -0600 Subject: [PATCH 4/6] Remove the ParquetFileWriter overloads that wrote encrypted DataPageV2 levels in plaintext The writeDataPageV2 overloads that took a page header encryptor but no page block encryptor let callers encrypt only the values, which is the layout of apache/parquet-java#3934. They are removed (with japicmp excludes); the remaining overload rejects a header encryptor without a page block encryptor. The hand-built ambiguous legacy pages are now fixture files. --- .../parquet/hadoop/ParquetFileWriter.java | 204 +----------------- .../hadoop/rewrite/ParquetRewriter.java | 2 +- .../TestDataPageV2EncryptionInterop.java | 107 +++++---- ...legacy-v2-ambiguous-aes-gcm-ctr-v1.parquet | Bin 0 -> 796 bytes .../legacy-v2-ambiguous-aes-gcm-v1.parquet | Bin 0 -> 796 bytes pom.xml | 6 + 6 files changed, 64 insertions(+), 255 deletions(-) create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/legacy-v2-ambiguous-aes-gcm-ctr-v1.parquet create mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/legacy-v2-ambiguous-aes-gcm-v1.parquet diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index 7cdf77d9fc..c2019ff6f2 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -1165,6 +1165,8 @@ public void writeDataPageV2( statistics, null, null, + null, + null, null); } @@ -1209,203 +1211,11 @@ public void writeDataPageV2( statistics, null, null, + null, + null, null); } - /** - * Writes a single v2 data page - * - * @param rowCount count of rows - * @param nullCount count of nulls - * @param valueCount count of values - * @param repetitionLevels repetition level bytes - * @param definitionLevels definition level bytes - * @param dataEncoding encoding for data - * @param compressedData compressed data bytes - * @param uncompressedDataSize the size of uncompressed data - * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for the page header - * @param pageHeaderAAD pageHeader AAD - * @throws IOException if any I/O error occurs during writing the file - * @deprecated will be removed in 2.0.0. Use {@link ParquetFileWriter#writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[])} instead - */ - @Deprecated - public void writeDataPageV2( - int rowCount, - int nullCount, - int valueCount, - BytesInput repetitionLevels, - BytesInput definitionLevels, - Encoding dataEncoding, - BytesInput compressedData, - int uncompressedDataSize, - Statistics statistics, - BlockCipher.Encryptor metadataBlockEncryptor, - byte[] pageHeaderAAD) - throws IOException { - writeDataPageV2( - rowCount, - nullCount, - valueCount, - repetitionLevels, - definitionLevels, - dataEncoding, - compressedData, - true, /* compressed by default */ - uncompressedDataSize, - statistics, - metadataBlockEncryptor, - pageHeaderAAD, - null); - } - - /** - * Writes a single v2 data page. The levels and data bytes are written as given, without encryption. - * For encrypted columns, use the overload that takes a page block encryptor. - * - * @param rowCount count of rows - * @param nullCount count of nulls - * @param valueCount count of values - * @param repetitionLevels repetition level bytes - * @param definitionLevels definition level bytes - * @param dataEncoding encoding for data - * @param bytes data bytes - * @param compressed whether the data bytes is compressed - * @param uncompressedDataSize the size of uncompressed data - * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for the page header - * @param pageHeaderAAD pageHeader AAD - * @throws IOException if any I/O error occurs during writing the file - */ - public void writeDataPageV2( - int rowCount, - int nullCount, - int valueCount, - BytesInput repetitionLevels, - BytesInput definitionLevels, - Encoding dataEncoding, - BytesInput bytes, - boolean compressed, - int uncompressedDataSize, - Statistics statistics, - BlockCipher.Encryptor metadataBlockEncryptor, - byte[] pageHeaderAAD) - throws IOException { - writeDataPageV2( - rowCount, - nullCount, - valueCount, - repetitionLevels, - definitionLevels, - dataEncoding, - bytes, - compressed, - uncompressedDataSize, - statistics, - metadataBlockEncryptor, - pageHeaderAAD, - null); - } - - /** - * Writes a single v2 data page - * - * @param rowCount count of rows - * @param nullCount count of nulls - * @param valueCount count of values - * @param repetitionLevels repetition level bytes - * @param definitionLevels definition level bytes - * @param dataEncoding encoding for data - * @param compressedData compressed data bytes - * @param uncompressedDataSize the size of uncompressed data - * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for the page header - * @param pageHeaderAAD pageHeader AAD - * @param sizeStatistics size statistics for the page - * @throws IOException if any I/O error occurs during writing the file - * @deprecated will be removed in 2.0.0. Use {@link ParquetFileWriter#writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[], SizeStatistics)} instead - */ - @Deprecated - public void writeDataPageV2( - int rowCount, - int nullCount, - int valueCount, - BytesInput repetitionLevels, - BytesInput definitionLevels, - Encoding dataEncoding, - BytesInput compressedData, - int uncompressedDataSize, - Statistics statistics, - BlockCipher.Encryptor metadataBlockEncryptor, - byte[] pageHeaderAAD, - SizeStatistics sizeStatistics) - throws IOException { - writeDataPageV2( - rowCount, - nullCount, - valueCount, - repetitionLevels, - definitionLevels, - dataEncoding, - compressedData, - true, /* compressed by default */ - uncompressedDataSize, - statistics, - metadataBlockEncryptor, - pageHeaderAAD, - sizeStatistics); - } - - /** - * Writes a single v2 data page. The levels and data bytes are written as given, without encryption. - * For encrypted columns, use the overload that takes a page block encryptor. - * - * @param rowCount count of rows - * @param nullCount count of nulls - * @param valueCount count of values - * @param repetitionLevels repetition level bytes - * @param definitionLevels definition level bytes - * @param dataEncoding encoding for data - * @param bytes data bytes - * @param compressed whether the data bytes is compressed - * @param uncompressedDataSize the size of uncompressed data - * @param statistics the statistics of the page - * @param metadataBlockEncryptor encryptor for the page header - * @param pageHeaderAAD pageHeader AAD - * @param sizeStatistics size statistics for the page - * @throws IOException if any I/O error occurs during writing the file - */ - public void writeDataPageV2( - int rowCount, - int nullCount, - int valueCount, - BytesInput repetitionLevels, - BytesInput definitionLevels, - Encoding dataEncoding, - BytesInput bytes, - boolean compressed, - int uncompressedDataSize, - Statistics statistics, - BlockCipher.Encryptor metadataBlockEncryptor, - byte[] pageHeaderAAD, - SizeStatistics sizeStatistics) - throws IOException { - writeDataPageV2Body( - rowCount, - nullCount, - valueCount, - toIntWithCheck(repetitionLevels.size(), "page repetition levels"), - toIntWithCheck(definitionLevels.size(), "page definition levels"), - dataEncoding, - Arrays.asList(repetitionLevels, definitionLevels, bytes), - compressed, - uncompressedDataSize, - statistics, - metadataBlockEncryptor, - pageHeaderAAD, - sizeStatistics); - } - /** * Writes a single v2 data page. If {@code pageBlockEncryptor} is not null, the repetition levels, * definition levels and data bytes are encrypted together as one module. @@ -1423,7 +1233,8 @@ public void writeDataPageV2( * @param metadataBlockEncryptor encryptor for the page header * @param pageHeaderAAD pageHeader AAD * @param sizeStatistics size statistics for the page - * @param pageBlockEncryptor encryptor for the page body, or null if the column is not encrypted + * @param pageBlockEncryptor encryptor for the page body, or null if the column is not encrypted; null + * if and only if {@code metadataBlockEncryptor} is null * @param dataPageAAD data page AAD, updated for this page's ordinal; required if * {@code pageBlockEncryptor} is not null * @throws IOException if any I/O error occurs during writing the file @@ -1445,6 +1256,9 @@ public void writeDataPageV2( BlockCipher.Encryptor pageBlockEncryptor, byte[] dataPageAAD) throws IOException { + Preconditions.checkArgument( + (metadataBlockEncryptor == null) == (pageBlockEncryptor == null), + "Both the page header and the page body of an encrypted column must be encrypted"); List pageBody; if (pageBlockEncryptor == null) { pageBody = Arrays.asList(repetitionLevels, definitionLevels, bytes); diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java index 9a868e3124..851b84646c 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java @@ -780,7 +780,7 @@ private void processChunk( metaEncryptor, dataPageHeaderAAD, null, - encryptColumn ? dataEncryptor : null, + dataEncryptor, dataPageAAD); pageOrdinal++; break; diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java index d5e82f28e4..2e16f03789 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java @@ -40,7 +40,6 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,7 +50,6 @@ import org.apache.parquet.ParquetReadOptions; import org.apache.parquet.bytes.BytesInput; import org.apache.parquet.bytes.BytesUtils; -import org.apache.parquet.column.ColumnDescriptor; import org.apache.parquet.column.Encoding; import org.apache.parquet.column.ParquetProperties; import org.apache.parquet.column.ParquetProperties.WriterVersion; @@ -82,8 +80,6 @@ import org.apache.parquet.hadoop.util.HadoopInputFile; import org.apache.parquet.hadoop.util.HadoopOutputFile; import org.apache.parquet.io.ParquetDecodingException; -import org.apache.parquet.schema.MessageType; -import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -102,6 +98,9 @@ *

  • {@code java-legacy-v2-*}: written with {@link #writeWithParquetJava} at commit e265ac7f5, * before the levels were encrypted. To regenerate, check out that commit and run it for each * cipher and codec. + *
  • {@code legacy-v2-ambiguous-*}: one hand-built legacy page, see {@link + * #readsALegacyPageThatLooksLikeASingleModule}. Written with a {@code ParquetFileWriter} overload + * that has since been removed. * */ public class TestDataPageV2EncryptionInterop { @@ -150,9 +149,10 @@ static Stream legacyFixtureNames() { .map(codec -> "java-legacy-v2-" + cipher + "-" + codec + ".parquet")); } - static Stream ciphersAndMemoryModes() { - return Stream.of(ParquetCipher.values()) - .flatMap(cipher -> Stream.of(Arguments.of(cipher, false), Arguments.of(cipher, true))); + static Stream ambiguousFixtures() { + return Stream.of("aes-gcm-v1", "aes-gcm-ctr-v1") + .flatMap(cipher -> Stream.of(false, true) + .map(directMemory -> Arguments.of("legacy-v2-ambiguous-" + cipher + ".parquet", directMemory))); } static Stream ciphersAndCodecs() { @@ -357,65 +357,54 @@ public void skipsPagesOfFilesWrittenByEarlierParquetJavaVersions(String fixture, .containsExactlyElementsOf(expectedRows().subList(FIRST_ROW_OF_PAGE_WITH_SEQ_250, NUM_ROWS)); } - /** A legacy page whose first 4 level bytes equal {@code bodySize - 4}, so it is {@code AMBIGUOUS}. */ + /** + * {@code legacy-v2-ambiguous-*}: one legacy page of 8 optional int32 values, only the first (12345) + * defined. Its 4 bytes of definition levels, {@code 03 01 00 00}, are a bit-packed run of 8 levels + * plus 2 unread bytes, and read as an LE int they equal {@code bodySize - 4}, so the layout is {@code + * AMBIGUOUS}. + */ @ParameterizedTest(name = "{0}, directMemory={1}") - @MethodSource("ciphersAndMemoryModes") - public void readsALegacyPageThatLooksLikeASingleModule(ParquetCipher cipher, boolean directMemory) - throws Exception { - MessageType schema = MessageTypeParser.parseMessageType("message schema { optional int32 v; }"); - // Bit-packed run of 8 levels, only the first defined, plus 2 unread bytes; as an LE int 0x0103 = 259 - byte[] levels = {0x03, 0x01, 0x00, 0x00}; - int bodySize = 259 + Integer.BYTES; - int moduleSize = bodySize - levels.length; - int overhead = Integer.BYTES - + AesCipher.NONCE_LENGTH - + (cipher == ParquetCipher.AES_GCM_V1 ? AesCipher.GCM_TAG_LENGTH : 0); - byte[] values = new byte[moduleSize - overhead]; - System.arraycopy(BytesUtils.intToBytes(12345), 0, values, 0, Integer.BYTES); + @MethodSource("ambiguousFixtures") + public void readsALegacyPageThatLooksLikeASingleModule(String fixture, boolean directMemory) throws Exception { + List rows = readRows(fixture(fixture), new Configuration(), decryptionProperties(), directMemory); + assertThat(rows).hasSize(8); + assertThat(rows.get(0)).contains("v: 12345"); + assertThat(rows.subList(1, 8)).allSatisfy(row -> assertThat(row).doesNotContain("v:")); + } + + @Test + public void writerRejectsAnEncryptedPageHeaderWithoutAnEncryptedPageBody() throws Exception { Path file = new Path(tempDir.resolve(UUID.randomUUID().toString()).toUri()); - ParquetFileWriter writer = new ParquetFileWriter( + try (ParquetFileWriter writer = new ParquetFileWriter( HadoopOutputFile.fromPath(file, new Configuration()), - schema, + SCHEMA, ParquetFileWriter.Mode.CREATE, ParquetWriter.DEFAULT_BLOCK_SIZE, ParquetWriter.MAX_PADDING_SIZE_DEFAULT, - FileEncryptionProperties.builder(FOOTER_KEY) - .withFooterKeyMetadata(FOOTER_KEY_METADATA) - .withAlgorithm(cipher) - .build(), - ParquetProperties.builder().build()); - ColumnDescriptor column = schema.getColumns().get(0); - InternalColumnEncryptionSetup setup = writer.getEncryptor().getColumnSetup(ColumnPath.get("v"), true, 0); - byte[] fileAAD = writer.getEncryptor().getFileAAD(); - writer.start(); - writer.startBlock(8); - writer.startColumn(column, 8, CompressionCodecName.UNCOMPRESSED); - // Legacy layout: plaintext levels, then the encrypted values - writer.writeDataPageV2( - 8, - 7, - 8, - BytesInput.empty(), - BytesInput.from(levels), - Encoding.PLAIN, - BytesInput.from(setup.getDataEncryptor() - .encrypt(values, AesCipher.createModuleAAD(fileAAD, ModuleType.DataPage, 0, 0, 0))), - false, - values.length, - Statistics.createStats(column.getPrimitiveType()), - setup.getMetaDataEncryptor(), - AesCipher.createModuleAAD(fileAAD, ModuleType.DataPageHeader, 0, 0, 0), - null); - writer.endColumn(); - writer.endBlock(); - writer.end(Collections.emptyMap()); - - List rows = readRows(file, new Configuration(), decryptionProperties(), directMemory); - - assertThat(rows).hasSize(8); - assertThat(rows.get(0)).contains("v: 12345"); - assertThat(rows.subList(1, 8)).allSatisfy(row -> assertThat(row).doesNotContain("v:")); + null, + ParquetProperties.builder().build())) { + BlockCipher.Encryptor headerEncryptor = ModuleCipherFactory.getEncryptor(AesMode.GCM, FOOTER_KEY); + + assertThatThrownBy(() -> writer.writeDataPageV2( + 1, + 0, + 1, + BytesInput.empty(), + BytesInput.from(new byte[] {0x02, 0x01}), + Encoding.PLAIN, + BytesInput.from(new byte[4]), + false, + 4, + Statistics.createStats(SCHEMA.getType("id").asPrimitiveType()), + headerEncryptor, + new byte[0], + null, + null, + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Both the page header and the page body of an encrypted column must be encrypted"); + } } /** Writes 100 rows per page; encrypts all columns with the footer key if {@code cipher} is set. */ diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/legacy-v2-ambiguous-aes-gcm-ctr-v1.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/legacy-v2-ambiguous-aes-gcm-ctr-v1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3a7363dc887a000c1a6dc06c3913a327171c3f02 GIT binary patch literal 796 zcmV+%1LOQqK~hD40000g@4`9(0h;STZ65eEt${q=z{gWRX7&( zPat}aIVmI&We8IWr8v9On|66^L!hubq_JH8l+>2Ny3FhBfi4Uo`S{n1CH75+{Q%;J z-4i-mZwATr5vw4|zacvyrE{eEbd}`5EC#+XyE)`n1LUytRKb1&0RRC10001yMMWNM zFTL72pqB6th32&ANw!*xV-tfQ-_DqkwQca+xiE99ct4~CRcYZMXzo?LaK3VkOYN4+ z>OUP;D)uNXQ3=`9Gy&-*ICQ)m#1NJKmG{B(WCL3K%9^bhK5@`RZy0_8H;Uj;+NmSk zyApHz5XW;zYE@m@bEK2UU!n+ zoeLZm`n#CtuAN6Xo_s8&e7+B+q2t!SGNjOCZ)U*>SsMPqx=y%Wr)Piw006eOaQg^I zR;l2@6TBj_?L*MKgvU2B9|ICU8g0VDb?3jLhW+!cP**{m=7p?tsJqvgZ?upGKs`@) zAUdAfOy|!Q!dBtlld9LCYPR)dQ~$?ICA;2TB1fw@e0nu7;HLC+Z=K26+M#)&Xfq)# zIsVqv%?)gAS9sYMtPcn8#5o)+Cg5d+Cu#!mhPPdW<+4T@}OEKiTPf zO!v*Pb96i~g}xy}_s5RyzUcZr9y;5i7wTKx?&5Dy;dsl`PUkdzSY*4FC*}Jse^zHn z&vk2=yD6H?p@gv`F^4dZ>BLx>N{LK$1-=_Vf41%mCSqGfKzHbp$~({{3r)kd0UNtt zuv>`U=dn*h$%p(VlW&y$z{`_F$a0=E1cu67RMk~UtJ}evMrz~WoR*o)q{c8bJ*C0PKK@~(KxzE}0RR9{K~hEby@|{K literal 0 HcmV?d00001 diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/legacy-v2-ambiguous-aes-gcm-v1.parquet b/parquet-hadoop/src/test/resources/encryption/datapagev2/legacy-v2-ambiguous-aes-gcm-v1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..39b7292b8313f2770437dcf02fd3882ccd5a5a0a GIT binary patch literal 796 zcmV+%1LOQqK~hD40000Z4;|ezI2}d`CLD9|$Lgs^wLD@21xg)?iF{u^`2{nBL0=946)T_JFdu4%M^ z4u^(?TMbJPr+Z#k(I-X^55oX;p55H42lb|Ay6(_RKb{}>XCu=B+`o^O`ROnB(9oD)63(t>s!nyGvTyNW0u zt!vND;u)v|zJVyLuD1ygCYx=ZfI>(Yvz?|rzJ(w21uZ3tr>(xC9s~8Sq`a}~uqA*1000)DHyQHw z8&f$xWQC6G%`){^RrjQ`bjm{UC=F|%rbQbB?+^k6Nd}~hy_N-D2TVbrR&1t;C+KT@ z`qXU``t?F~*)u`T1M-O#O3m10!z)S>RiRLsn6M%O=?f3G^wdHxVswHV(~MfXS~%C+ zpFY;qkRY}D%ncXCuD3Ifc#cw+go9M!k)O>F>bbALU%K>QWxXr!pfVCo6!MnRvw?sBjc4JU+c z(RE{0wmLOph~qW8`uA%lHNMoxz=a+|DRR04(9xA}#Q3o1?e5m&K5I9gAc<&rFYDA5 zQQQ|KSBa5ukHYDRjJM1zLJk~Vt7HxJBy+HG0I<&&f-p)s)%bX1P1^LhrNF~DG4)YH z0!&jQO>BSz9m14<;NWj6yMJP$;_*cwg2l@0C}4=rR-`&Zem`T1j7`ABkOV}P6Z_zP a>=7gB?BluXyJ(w*X@I5y0RR9{K~hDy7IbU? literal 0 HcmV?d00001 diff --git a/pom.xml b/pom.xml index 4e4afde61c..c04eb94878 100644 --- a/pom.xml +++ b/pom.xml @@ -670,6 +670,12 @@ org.apache.parquet.internal.filter2.columnindex.RowRanges org.apache.parquet.internal.filter2.columnindex.RowRanges$Range org.apache.parquet.internal.filter2.columnindex.ColumnIndexFilter + + org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[]) + org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,boolean,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[]) + org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[],org.apache.parquet.column.statistics.SizeStatistics) + org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,boolean,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[],org.apache.parquet.column.statistics.SizeStatistics) From 8c94e1353510485faaf806e40a234080443bf7b7 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 5 Nov 2026 19:30:11 -0600 Subject: [PATCH 5/6] GH-3934: Remove the Arrow fixture generator script --- .../crypto/DataPageV2EncryptionTestData.java | 2 +- .../TestDataPageV2EncryptionInterop.java | 2 +- .../keytools/mocks/Base64KmsClient.java | 2 +- .../datapagev2/generate_arrow_fixtures.py | 105 ------------------ 4 files changed, 3 insertions(+), 108 deletions(-) delete mode 100644 parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java index bfacf3f2d1..7c7d1deaba 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/DataPageV2EncryptionTestData.java @@ -64,7 +64,7 @@ final class DataPageV2EncryptionTestData { private DataPageV2EncryptionTestData() {} - /** Builds row {@code i}. Must match {@code row(i)} in {@code generate_arrow_fixtures.py}. */ + /** Builds row {@code i}, as stored in the {@code arrow-v2-*} fixtures. */ public static Group row(int i) { Group group = new SimpleGroupFactory(SCHEMA).newGroup(); group.add("seq", (long) i); diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java index 2e16f03789..6a0c33f267 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java @@ -94,7 +94,7 @@ *

    Fixtures in {@code src/test/resources/encryption/datapagev2}: * *

      - *
    • {@code arrow-v2-*}: written by {@code generate_arrow_fixtures.py} (pyarrow). + *
    • {@code arrow-v2-*}: written by Arrow C++ (pyarrow 24.0.0). *
    • {@code java-legacy-v2-*}: written with {@link #writeWithParquetJava} at commit e265ac7f5, * before the levels were encrypted. To regenerate, check out that commit and run it for each * cipher and codec. diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java index 908ba7614f..7b0cb0974a 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/keytools/mocks/Base64KmsClient.java @@ -24,7 +24,7 @@ import org.apache.parquet.crypto.KeyAccessDeniedException; import org.apache.parquet.crypto.keytools.KmsClient; -/** Wraps keys by base64 encoding them, like the KMS client of {@code generate_arrow_fixtures.py}. */ +/** Wraps keys by base64 encoding them, like the KMS client that wrote the {@code arrow-v2-*} fixtures. */ public class Base64KmsClient implements KmsClient { @Override diff --git a/parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py b/parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py deleted file mode 100644 index 5db7f091b9..0000000000 --- a/parquet-hadoop/src/test/resources/encryption/datapagev2/generate_arrow_fixtures.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Generates the arrow-v2-*.parquet fixtures used by TestDataPageV2EncryptionInterop. - -The files are written by Arrow C++ (through pyarrow) with Parquet Modular Encryption and DataPageV2 -pages. Arrow encrypts the whole page body of a DataPageV2 page, repetition and definition levels -included, which is the layout the parquet-java reader has to understand. - -The rows must match DataPageV2EncryptionTestData.row(i) in the Java test sources. The keys are -wrapped by a toy KMS client (base64 only) that is mirrored by Base64KmsClient in the Java tests. - -Usage: python3 generate_arrow_fixtures.py (generated with pyarrow 24.0.0) -""" - -import base64 -import sys -from datetime import timedelta -from pathlib import Path - -import pyarrow as pa -import pyarrow.parquet as pq -import pyarrow.parquet.encryption as pe - -NUM_ROWS = 500 -FOOTER_KEY_ID = "footkey" - - -class Base64KmsClient(pe.KmsClient): - """Not a secure KMS: wrapping a key is just base64 encoding it. For test fixtures only.""" - - def __init__(self, kms_connection_configuration): - pe.KmsClient.__init__(self) - - def wrap_key(self, key_bytes, master_key_identifier): - return base64.b64encode(key_bytes).decode("ascii") - - def unwrap_key(self, wrapped_key, master_key_identifier): - return base64.b64decode(wrapped_key) - - -def table(): - seq, ids, names, tags = [], [], [], [] - for i in range(NUM_ROWS): - seq.append(i) - ids.append(None if i % 7 == 0 else i) - names.append(None if i % 5 == 0 else "name-%d" % i) - if i % 4 == 0: - tags.append(None) - else: - tags.append([None if (i + j) % 3 == 0 else "tag-%d-%d" % (i, j) for j in range(i % 4)]) - schema = pa.schema([ - pa.field("seq", pa.int64(), nullable=False), - pa.field("id", pa.int32()), - pa.field("name", pa.string()), - pa.field("tags", pa.list_(pa.field("element", pa.string()))), - ]) - return pa.table( - {"seq": seq, "id": pa.array(ids, pa.int32()), "name": names, "tags": tags}, schema=schema) - - -def main(out_dir): - out = Path(out_dir) - out.mkdir(parents=True, exist_ok=True) - factory = pe.CryptoFactory(lambda config: Base64KmsClient(config)) - kms_config = pe.KmsConnectionConfig() - for algorithm, algorithm_name in (("AES_GCM_V1", "aes-gcm-v1"), ("AES_GCM_CTR_V1", "aes-gcm-ctr-v1")): - for codec in ("none", "snappy"): - encryption = pe.EncryptionConfiguration( - footer_key=FOOTER_KEY_ID, - uniform_encryption=True, - encryption_algorithm=algorithm, - double_wrapping=False, - internal_key_material=True, - cache_lifetime=timedelta(minutes=1), - ) - properties = factory.file_encryption_properties(kms_config, encryption) - path = out / ("arrow-v2-%s-%s.parquet" % (algorithm_name, "uncompressed" if codec == "none" else codec)) - with pq.ParquetWriter( - str(path), table().schema, encryption_properties=properties, - data_page_version="2.0", compression=codec, write_batch_size=100, - data_page_size=1, use_dictionary=False) as writer: - writer.write_table(table()) - print("wrote", path, path.stat().st_size) - - -if __name__ == "__main__": - main(sys.argv[1] if len(sys.argv) > 1 else ".") From 1df11ea8d4ee004fea5cb5938d86ebceb2e29e6a Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Thu, 5 Nov 2026 20:02:12 -0600 Subject: [PATCH 6/6] GH-3934: Deprecate the plaintext-levels writeDataPageV2 overloads instead of removing them Restores the four writeDataPageV2 overloads that take only a page header encryptor, deprecated with a warning that they leave the levels of an encrypted column in plaintext, and drops their japicmp excludes. Reads DataPageV2 pages with a switch on the detected layout, notes that the plaintext-levels read path exists for backward compatibility only, and passes the CRC to the fallback page in FullBodyEncryptedDataPageV2. --- .../hadoop/ColumnChunkPageReadStore.java | 4 + .../hadoop/FullBodyEncryptedDataPageV2.java | 8 + .../parquet/hadoop/ParquetFileReader.java | 100 ++++---- .../parquet/hadoop/ParquetFileWriter.java | 216 ++++++++++++++++++ .../TestDataPageV2EncryptionInterop.java | 4 +- pom.xml | 6 - 6 files changed, 291 insertions(+), 47 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java index 72e0916fd8..fda3e9a3ad 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageReadStore.java @@ -232,6 +232,10 @@ public DataPage visit(DataPageV2 dataPageV2) { /** * Reads a page whose levels are not encrypted. If the column is encrypted, only the values are an * encrypted module. + * + *

      An encrypted page with plaintext levels does not follow the Parquet encryption spec, which + * encrypts the levels and the values as one module. Such pages are written only by parquet-java + * 1.18.0 and earlier, and are read here for backward compatibility only. */ private DataPageV2 readPageV2WithPlaintextLevels(DataPageV2 page, int currentPageIndex) { if (!page.isCompressed() && offsetIndex == null && null == blockDecryptor) { diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java index 36fc1771ea..181e5dc92b 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/FullBodyEncryptedDataPageV2.java @@ -95,6 +95,14 @@ DataPageV2 getPlaintextLevelsLayoutPage() { return plaintextLevelsLayoutPage; } + @Override + public void setCrc(int crc) { + super.setCrc(crc); + if (plaintextLevelsLayoutPage != null) { + plaintextLevelsLayoutPage.setCrc(crc); + } + } + /** * Detects the layout of the body of a page that has levels. A module starts with the length of the * rest of the module as a 4 byte LE int, so: diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java index 0bac86af3d..c33bb800de 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java @@ -2219,13 +2219,6 @@ public ColumnChunkPageReader readAllPages( // Copy crc to new page, used for testing if (pageHeader.isSetCrc()) { dataPageV2.setCrc(pageHeader.getCrc()); - if (dataPageV2 instanceof FullBodyEncryptedDataPageV2) { - DataPageV2 plaintextLevelsPage = - ((FullBodyEncryptedDataPageV2) dataPageV2).getPlaintextLevelsLayoutPage(); - if (plaintextLevelsPage != null) { - plaintextLevelsPage.setCrc(pageHeader.getCrc()); - } - } } pagesInChunk.add(dataPageV2); valuesCountReadSoFar += dataHeaderV2.getNum_values(); @@ -2276,7 +2269,7 @@ private int getPageOrdinal(int dataPageCountReadSoFar) { /** * Reads a DataPageV2 body. For an encrypted column with levels, returns a {@link - * FullBodyEncryptedDataPageV2} if the levels are inside the encrypted module, or a {@link + * FullBodyEncryptedDataPageV2} if the levels may be inside the encrypted module, or a {@link * DataPageV2} with the plaintext levels split out otherwise. */ private DataPageV2 readDataPageV2(PageHeader pageHeader, PrimitiveType type, boolean encryptedColumn) @@ -2309,38 +2302,20 @@ private DataPageV2 readDataPageV2(PageHeader pageHeader, PrimitiveType type, boo "could not verify page integrity, CRC checksum verification failed", pageBody); } - FullBodyEncryptedDataPageV2.Layout layout = - FullBodyEncryptedDataPageV2.detectLayout(pageBody, levelsByteLength); - DataPageV2 plaintextLevelsPage = null; - if (layout != FullBodyEncryptedDataPageV2.Layout.SINGLE_MODULE) { - ByteBufferInputStream pageBodyStream = pageBody.toInputStream(); - plaintextLevelsPage = new DataPageV2( - header.getNum_rows(), - header.getNum_nulls(), - header.getNum_values(), - BytesInput.from(pageBodyStream.sliceBuffers(repetitionLevelsByteLength)), - BytesInput.from(pageBodyStream.sliceBuffers(definitionLevelsByteLength)), - encoding, - BytesInput.from(pageBodyStream.remainingBuffers()), - uncompressedPageSize, - statistics, - header.isIs_compressed()); - } - if (layout == FullBodyEncryptedDataPageV2.Layout.PLAINTEXT_LEVELS) { - return plaintextLevelsPage; + switch (FullBodyEncryptedDataPageV2.detectLayout(pageBody, levelsByteLength)) { + case PLAINTEXT_LEVELS: + return pageWithPlaintextLevels(pageHeader, pageBody, encoding, statistics); + case SINGLE_MODULE: + return fullBodyEncryptedPage(pageHeader, pageBody, encoding, statistics, null); + case AMBIGUOUS: + default: + return fullBodyEncryptedPage( + pageHeader, + pageBody, + encoding, + statistics, + pageWithPlaintextLevels(pageHeader, pageBody, encoding, statistics)); } - return new FullBodyEncryptedDataPageV2( - header.getNum_rows(), - header.getNum_nulls(), - header.getNum_values(), - encoding, - pageBody, - repetitionLevelsByteLength, - definitionLevelsByteLength, - uncompressedPageSize, - statistics, - header.isIs_compressed(), - plaintextLevelsPage); } int dataSize = compressedPageSize - repetitionLevelsByteLength - definitionLevelsByteLength; @@ -2368,6 +2343,53 @@ private DataPageV2 readDataPageV2(PageHeader pageHeader, PrimitiveType type, boo header.isIs_compressed()); } + /** Splits the levels off the front of a page body whose levels are not encrypted. */ + private DataPageV2 pageWithPlaintextLevels( + PageHeader pageHeader, + BytesInput pageBody, + org.apache.parquet.column.Encoding encoding, + org.apache.parquet.column.statistics.Statistics statistics) + throws IOException { + DataPageHeaderV2 header = pageHeader.getData_page_header_v2(); + ByteBufferInputStream pageBodyStream = pageBody.toInputStream(); + return new DataPageV2( + header.getNum_rows(), + header.getNum_nulls(), + header.getNum_values(), + BytesInput.from(pageBodyStream.sliceBuffers(header.getRepetition_levels_byte_length())), + BytesInput.from(pageBodyStream.sliceBuffers(header.getDefinition_levels_byte_length())), + encoding, + BytesInput.from(pageBodyStream.remainingBuffers()), + pageHeader.getUncompressed_page_size(), + statistics, + header.isIs_compressed()); + } + + /** + * @param plaintextLevelsPage the same page read as {@link #pageWithPlaintextLevels}, if the layout is + * ambiguous; null otherwise + */ + private FullBodyEncryptedDataPageV2 fullBodyEncryptedPage( + PageHeader pageHeader, + BytesInput pageBody, + org.apache.parquet.column.Encoding encoding, + org.apache.parquet.column.statistics.Statistics statistics, + DataPageV2 plaintextLevelsPage) { + DataPageHeaderV2 header = pageHeader.getData_page_header_v2(); + return new FullBodyEncryptedDataPageV2( + header.getNum_rows(), + header.getNum_nulls(), + header.getNum_values(), + encoding, + pageBody, + header.getRepetition_levels_byte_length(), + header.getDefinition_levels_byte_length(), + pageHeader.getUncompressed_page_size(), + statistics, + header.isIs_compressed(), + plaintextLevelsPage); + } + /** * @param size the size of the page * @return the page diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java index c2019ff6f2..f800e0c5b3 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java @@ -1216,6 +1216,222 @@ public void writeDataPageV2( null); } + /** + * Writes a single v2 data page + * + *

      Do not use for encrypted columns. If {@code metadataBlockEncryptor} is set, only the page + * header is encrypted by this method: the repetition and definition levels are written in plaintext, + * and the data bytes are written as given. Such pages do not follow the Parquet encryption spec, + * cannot be read by other Parquet implementations, and leave the levels unprotected. + * + * @param rowCount count of rows + * @param nullCount count of nulls + * @param valueCount count of values + * @param repetitionLevels repetition level bytes + * @param definitionLevels definition level bytes + * @param dataEncoding encoding for data + * @param compressedData compressed data bytes + * @param uncompressedDataSize the size of uncompressed data + * @param statistics the statistics of the page + * @param metadataBlockEncryptor encryptor for the page header + * @param pageHeaderAAD pageHeader AAD + * @throws IOException if any I/O error occurs during writing the file + * @deprecated will be removed in 2.0.0. Use {@link #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[], SizeStatistics, BlockCipher.Encryptor, byte[])} instead + */ + @Deprecated + public void writeDataPageV2( + int rowCount, + int nullCount, + int valueCount, + BytesInput repetitionLevels, + BytesInput definitionLevels, + Encoding dataEncoding, + BytesInput compressedData, + int uncompressedDataSize, + Statistics statistics, + BlockCipher.Encryptor metadataBlockEncryptor, + byte[] pageHeaderAAD) + throws IOException { + writeDataPageV2( + rowCount, + nullCount, + valueCount, + repetitionLevels, + definitionLevels, + dataEncoding, + compressedData, + true, /* compressed by default */ + uncompressedDataSize, + statistics, + metadataBlockEncryptor, + pageHeaderAAD, + null); + } + + /** + * Writes a single v2 data page + * + *

      Do not use for encrypted columns. If {@code metadataBlockEncryptor} is set, only the page + * header is encrypted by this method: the repetition and definition levels are written in plaintext, + * and the data bytes are written as given. Such pages do not follow the Parquet encryption spec, + * cannot be read by other Parquet implementations, and leave the levels unprotected. + * + * @param rowCount count of rows + * @param nullCount count of nulls + * @param valueCount count of values + * @param repetitionLevels repetition level bytes + * @param definitionLevels definition level bytes + * @param dataEncoding encoding for data + * @param bytes data bytes + * @param compressed whether the data bytes is compressed + * @param uncompressedDataSize the size of uncompressed data + * @param statistics the statistics of the page + * @param metadataBlockEncryptor encryptor for the page header + * @param pageHeaderAAD pageHeader AAD + * @throws IOException if any I/O error occurs during writing the file + * @deprecated will be removed in 2.0.0. Use {@link #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[], SizeStatistics, BlockCipher.Encryptor, byte[])} instead + */ + @Deprecated + public void writeDataPageV2( + int rowCount, + int nullCount, + int valueCount, + BytesInput repetitionLevels, + BytesInput definitionLevels, + Encoding dataEncoding, + BytesInput bytes, + boolean compressed, + int uncompressedDataSize, + Statistics statistics, + BlockCipher.Encryptor metadataBlockEncryptor, + byte[] pageHeaderAAD) + throws IOException { + writeDataPageV2( + rowCount, + nullCount, + valueCount, + repetitionLevels, + definitionLevels, + dataEncoding, + bytes, + compressed, + uncompressedDataSize, + statistics, + metadataBlockEncryptor, + pageHeaderAAD, + null); + } + + /** + * Writes a single v2 data page + * + *

      Do not use for encrypted columns. If {@code metadataBlockEncryptor} is set, only the page + * header is encrypted by this method: the repetition and definition levels are written in plaintext, + * and the data bytes are written as given. Such pages do not follow the Parquet encryption spec, + * cannot be read by other Parquet implementations, and leave the levels unprotected. + * + * @param rowCount count of rows + * @param nullCount count of nulls + * @param valueCount count of values + * @param repetitionLevels repetition level bytes + * @param definitionLevels definition level bytes + * @param dataEncoding encoding for data + * @param compressedData compressed data bytes + * @param uncompressedDataSize the size of uncompressed data + * @param statistics the statistics of the page + * @param metadataBlockEncryptor encryptor for the page header + * @param pageHeaderAAD pageHeader AAD + * @param sizeStatistics size statistics for the page + * @throws IOException if any I/O error occurs during writing the file + * @deprecated will be removed in 2.0.0. Use {@link #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[], SizeStatistics, BlockCipher.Encryptor, byte[])} instead + */ + @Deprecated + public void writeDataPageV2( + int rowCount, + int nullCount, + int valueCount, + BytesInput repetitionLevels, + BytesInput definitionLevels, + Encoding dataEncoding, + BytesInput compressedData, + int uncompressedDataSize, + Statistics statistics, + BlockCipher.Encryptor metadataBlockEncryptor, + byte[] pageHeaderAAD, + SizeStatistics sizeStatistics) + throws IOException { + writeDataPageV2( + rowCount, + nullCount, + valueCount, + repetitionLevels, + definitionLevels, + dataEncoding, + compressedData, + true, /* compressed by default */ + uncompressedDataSize, + statistics, + metadataBlockEncryptor, + pageHeaderAAD, + sizeStatistics); + } + + /** + * Writes a single v2 data page + * + *

      Do not use for encrypted columns. If {@code metadataBlockEncryptor} is set, only the page + * header is encrypted by this method: the repetition and definition levels are written in plaintext, + * and the data bytes are written as given. Such pages do not follow the Parquet encryption spec, + * cannot be read by other Parquet implementations, and leave the levels unprotected. + * + * @param rowCount count of rows + * @param nullCount count of nulls + * @param valueCount count of values + * @param repetitionLevels repetition level bytes + * @param definitionLevels definition level bytes + * @param dataEncoding encoding for data + * @param bytes data bytes + * @param compressed whether the data bytes is compressed + * @param uncompressedDataSize the size of uncompressed data + * @param statistics the statistics of the page + * @param metadataBlockEncryptor encryptor for the page header + * @param pageHeaderAAD pageHeader AAD + * @param sizeStatistics size statistics for the page + * @throws IOException if any I/O error occurs during writing the file + * @deprecated will be removed in 2.0.0. Use {@link #writeDataPageV2(int, int, int, BytesInput, BytesInput, Encoding, BytesInput, boolean, int, Statistics, BlockCipher.Encryptor, byte[], SizeStatistics, BlockCipher.Encryptor, byte[])} instead + */ + @Deprecated + public void writeDataPageV2( + int rowCount, + int nullCount, + int valueCount, + BytesInput repetitionLevels, + BytesInput definitionLevels, + Encoding dataEncoding, + BytesInput bytes, + boolean compressed, + int uncompressedDataSize, + Statistics statistics, + BlockCipher.Encryptor metadataBlockEncryptor, + byte[] pageHeaderAAD, + SizeStatistics sizeStatistics) + throws IOException { + writeDataPageV2Body( + rowCount, + nullCount, + valueCount, + toIntWithCheck(repetitionLevels.size(), "page repetition levels"), + toIntWithCheck(definitionLevels.size(), "page definition levels"), + dataEncoding, + Arrays.asList(repetitionLevels, definitionLevels, bytes), + compressed, + uncompressedDataSize, + statistics, + metadataBlockEncryptor, + pageHeaderAAD, + sizeStatistics); + } + /** * Writes a single v2 data page. If {@code pageBlockEncryptor} is not null, the repetition levels, * definition levels and data bytes are encrypted together as one module. diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java index 6a0c33f267..2226e7543f 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/crypto/TestDataPageV2EncryptionInterop.java @@ -99,8 +99,8 @@ * before the levels were encrypted. To regenerate, check out that commit and run it for each * cipher and codec. *

    • {@code legacy-v2-ambiguous-*}: one hand-built legacy page, see {@link - * #readsALegacyPageThatLooksLikeASingleModule}. Written with a {@code ParquetFileWriter} overload - * that has since been removed. + * #readsALegacyPageThatLooksLikeASingleModule}. Written with a deprecated {@code + * ParquetFileWriter.writeDataPageV2} overload that takes only a page header encryptor. *
    */ public class TestDataPageV2EncryptionInterop { diff --git a/pom.xml b/pom.xml index c04eb94878..4e4afde61c 100644 --- a/pom.xml +++ b/pom.xml @@ -670,12 +670,6 @@ org.apache.parquet.internal.filter2.columnindex.RowRanges org.apache.parquet.internal.filter2.columnindex.RowRanges$Range org.apache.parquet.internal.filter2.columnindex.ColumnIndexFilter - - org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[]) - org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,boolean,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[]) - org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[],org.apache.parquet.column.statistics.SizeStatistics) - org.apache.parquet.hadoop.ParquetFileWriter#writeDataPageV2(int,int,int,org.apache.parquet.bytes.BytesInput,org.apache.parquet.bytes.BytesInput,org.apache.parquet.column.Encoding,org.apache.parquet.bytes.BytesInput,boolean,int,org.apache.parquet.column.statistics.Statistics,org.apache.parquet.format.BlockCipher$Encryptor,byte[],org.apache.parquet.column.statistics.SizeStatistics)