Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -190,12 +190,12 @@ private IWritableMemChunkGroup createMemChunkGroupIfNotExistAndGet(

private IWritableMemChunkGroup createAlignedMemChunkGroupIfNotExistAndGet(
IDeviceID deviceId, List<IMeasurementSchema> schemaList) {
List<IMeasurementSchema> filteredSchemaList =
schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList());
IWritableMemChunkGroup memChunkGroup =
memTableMap.computeIfAbsent(
deviceId,
k -> {
List<IMeasurementSchema> filteredSchemaList =
schemaList.stream().filter(Objects::nonNull).collect(Collectors.toList());
seriesNumber += filteredSchemaList.size();
return new AlignedWritableMemChunkGroup(filteredSchemaList, encryptParameter);
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,27 @@ protected void handoverAlignedTvList() {
@Override
public void writeAlignedPoints(
long insertTime, Object[] objectValue, List<IMeasurementSchema> schemaList) {
Object[] reorderedValue =
checkAndReorderColumnValuesInInsertPlan(schemaList, objectValue, null).left;
putAlignedRow(insertTime, reorderedValue);
int columnCount = ensureColumnsAndCount(schemaList);
if (columnCount == list.getTsDataTypes().size()
&& isIdentityColumnMapping(schemaList, columnCount)) {
// The insert plan already covers every device column in device order, so the sparse mapping
// arrays would only add an O(columns) allocation per row.
list.putAlignedValue(insertTime, objectValue);
} else {
int[] destColumns = new int[columnCount];
Object[] sparseValues = new Object[columnCount];
int position = 0;
for (int i = 0; i < schemaList.size(); i++) {
IMeasurementSchema measurementSchema = schemaList.get(i);
if (measurementSchema == null) {
continue;
}
destColumns[position] = measurementIndexMap.get(measurementSchema.getMeasurementName());
sparseValues[position] = objectValue[i];
position++;
}
list.putAlignedValue(insertTime, sparseValues, destColumns);
}
if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) {
handoverAlignedTvList();
}
Expand All @@ -241,49 +259,75 @@ public void writeAlignedTablet(
int start,
int end,
TSStatus[] results) {
Pair<Object[], BitMap[]> pair =
checkAndReorderColumnValuesInInsertPlan(schemaList, valueList, bitMaps);
Object[] reorderedColumnValues = pair.left;
BitMap[] reorderedBitMaps = pair.right;
putAlignedTablet(times, reorderedColumnValues, reorderedBitMaps, start, end, results);
int columnCount = ensureColumnsAndCount(schemaList);
int[] destColumns = new int[columnCount];
Object[] sparseValues = new Object[columnCount];
BitMap[] sparseBitMaps = bitMaps == null ? null : new BitMap[columnCount];
int position = 0;
for (int i = 0; i < schemaList.size(); i++) {
IMeasurementSchema measurementSchema = schemaList.get(i);
if (measurementSchema == null) {
continue;
}
destColumns[position] = measurementIndexMap.get(measurementSchema.getMeasurementName());
sparseValues[position] = valueList[i];
if (sparseBitMaps != null) {
sparseBitMaps[position] = bitMaps[i];
}
position++;
}
list.putAlignedValues(times, sparseValues, sparseBitMaps, destColumns, start, end, results);
if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) {
handoverAlignedTvList();
}
}

/**
* Register missing measurements in insertion order, then reorder values and bitmaps into the
* memchunk's column order.
* Register missing measurements in insertion order and return the number of columns written by
* this insert plan.
*
* <p>Only the written columns are materialized by the caller, so a single-measurement write to a
* wide aligned device no longer allocates device-wide value and bitmap arrays.
*
* @param schemaListInInsertPlan Contains all existed schema in InsertPlan. If some timeseries
* have been deleted, there will be null in its slot.
* @return values and bitmaps aligned with this memchunk's schema list
*/
private Pair<Object[], BitMap[]> checkAndReorderColumnValuesInInsertPlan(
List<IMeasurementSchema> schemaListInInsertPlan, Object[] columnValues, BitMap[] bitMaps) {
Object[] reorderedColumnValues = new Object[schemaList.size()];
BitMap[] reorderedBitMaps = bitMaps == null ? null : new BitMap[schemaList.size()];
private int ensureColumnsAndCount(List<IMeasurementSchema> schemaListInInsertPlan) {
for (int i = 0; i < schemaListInInsertPlan.size(); i++) {
IMeasurementSchema measurementSchema = schemaListInInsertPlan.get(i);
if (measurementSchema != null) {
Integer index = measurementIndexMap.get(measurementSchema.getMeasurementName());
if (index == null) {
// Keep writes without new measurements on a single-pass path. Only the first missing
// measurement triggers a batch extension and one resize of the reordered arrays.
extendColumnsInInsertPlan(schemaListInInsertPlan, i);
reorderedColumnValues = Arrays.copyOf(reorderedColumnValues, schemaList.size());
if (reorderedBitMaps != null) {
reorderedBitMaps = Arrays.copyOf(reorderedBitMaps, schemaList.size());
}
index = measurementIndexMap.get(measurementSchema.getMeasurementName());
}
reorderedColumnValues[index] = columnValues[i];
if (bitMaps != null) {
reorderedBitMaps[index] = bitMaps[i];
}
if (measurementSchema != null
&& !measurementIndexMap.containsKey(measurementSchema.getMeasurementName())) {
// Only the first missing measurement triggers a batch extension.
extendColumnsInInsertPlan(schemaListInInsertPlan, i);
break;
}
}
return new Pair<>(reorderedColumnValues, reorderedBitMaps);
int columnCount = 0;
for (int i = 0; i < schemaListInInsertPlan.size(); i++) {
if (schemaListInInsertPlan.get(i) != null) {
columnCount++;
}
}
return columnCount;
}

/**
* Whether every column of the insert plan maps to the device column with the same position. Such
* a plan can be written through the dense path without building a column mapping.
*/
private boolean isIdentityColumnMapping(
List<IMeasurementSchema> schemaListInInsertPlan, int columnCount) {
if (schemaListInInsertPlan.size() != columnCount) {
return false;
}
for (int i = 0; i < schemaListInInsertPlan.size(); i++) {
IMeasurementSchema measurementSchema = schemaListInInsertPlan.get(i);
if (measurementSchema == null
|| measurementIndexMap.get(measurementSchema.getMeasurementName()) != i) {
return false;
}
}
return true;
}

private void extendColumnsInInsertPlan(
Expand Down Expand Up @@ -589,6 +633,9 @@ public void encodeWorkingAlignedTVList(
if (activeSchemaList.isEmpty() && ignoreAllNullRows) {
return;
}
// Flush reads the raw value/bitmap containers, so the null tails of the open block must be
// materialized before encoding.
((AlignedTVList) workingListForFlush).materializeOpenBlockForRead();

BitMap allValueColDeletedMap;
AlignedTVList alignedWorkingListForFlush = (AlignedTVList) workingListForFlush;
Expand Down Expand Up @@ -1000,6 +1047,9 @@ private void encodeWithComputedChunkCap(
if (activeSchemaList.isEmpty() && ignoreAllNullRows) {
return;
}
// Flush reads the raw value/bitmap containers, so the null tails of the open block must be
// materialized before encoding.
((AlignedTVList) workingListForFlush).materializeOpenBlockForRead();

AlignedChunkWriterImpl alignedChunkWriter =
new AlignedChunkWriterImpl(activeSchemaList, encryptParameter);
Expand Down
Loading
Loading