From fbbcff5b549eea3a232bd0f52bdeb6ea5d3bf457 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 9 Sep 2026 23:02:42 +0800 Subject: [PATCH 1/5] [core] Support splitting large files in data evolution compaction --- docs/docs/multimodal-table/data-evolution.mdx | 26 +++ docs/generated/core_configuration.html | 8 +- .../java/org/apache/paimon/CoreOptions.java | 18 +- .../CompactCandidateRangeCollector.java | 25 ++- .../DataEvolutionCompactCoordinator.java | 42 +++- ...volutionCompactDeletionVectorRewriter.java | 49 +++-- .../DataEvolutionCompactRangePlanner.java | 6 +- .../DataEvolutionCompactTask.java | 2 +- .../DataEvolutionNormalCompactTask.java | 66 ++++-- .../CompactCandidateRangeCollectorTest.java | 29 ++- .../DataEvolutionCompactCoordinatorTest.java | 24 ++ .../DataEvolutionCompactRangePlannerTest.java | 1 + .../DataEvolutionNormalCompactTaskTest.java | 208 +++++++++++++++++- .../DataEvolutionDeletionVectorTest.java | 69 ++++++ 14 files changed, 526 insertions(+), 47 deletions(-) diff --git a/docs/docs/multimodal-table/data-evolution.mdx b/docs/docs/multimodal-table/data-evolution.mdx index 02b1d271872b..3186523014a8 100644 --- a/docs/docs/multimodal-table/data-evolution.mdx +++ b/docs/docs/multimodal-table/data-evolution.mdx @@ -608,6 +608,32 @@ Ordinary compaction **does not physically remove rows hidden by deletion vectors**. Removing their positions would change the alignment with untouched column files. +### Splitting Large Files During Compaction + +To resize existing large normal data files, enable +`data-evolution.compaction.split-large-files` (default: `false`) and run compaction: + +```sql +ALTER TABLE my_table SET TBLPROPERTIES ( + 'target-file-size' = '128 MB', + 'data-evolution.compaction.split-large-files' = 'true' +); +CALL sys.compact('default.my_table'); +``` + +The example uses Spark SQL. With this option enabled, normal files strictly larger +than twice `target-file-size` qualify for compaction even if the file count is below +`compaction.min.file-num`. Compaction includes all column updates for the same row-ID +range and rolls normal output files near `target-file-size`. Actual sizes depend on +compression and the writer's size-check granularity; the last file may be smaller. +The write-time `target-file-row-num` limit does not apply to compaction. + +Row IDs, column updates, and logical deletions are preserved. Associated dedicated +BLOB and VECTOR files are rewritten along with the normal files to keep their row-ID +ranges aligned, which adds read and write work for those columns. Their sizes alone +do not trigger splitting. Compaction rewrites files in the current snapshot; files referenced by older snapshots or tags remain until those references +expire and snapshot expiration removes them. + ### Materialize Deletion Vectors To remove deleted positions from the latest table state, run: diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 6a4955280dfb..d6d156d885e7 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -512,6 +512,12 @@ Boolean Legacy compatibility option. Setting this option to true fails. Data-evolution compaction preserves row IDs and logical deletions. Use the 'materialize_deletion_vectors' procedure to apply deletion vectors to the latest table state and assign new row IDs. Reclaiming files retained by historical snapshots or tags requires snapshot expiration. + +
data-evolution.compaction.split-large-files
+ false + Boolean + Whether data-evolution compaction selects normal data files larger than twice target-file-size, even below compaction.min.file-num. When enabled, normal compaction output rolls at target-file-size while preserving row IDs and logical deletions. Associated dedicated files are rewritten to stay within the new normal-file boundaries. +
data-evolution.enabled
false @@ -1817,7 +1823,7 @@
target-file-row-num
9223372036854775807 Long - Target number of rows per newly written data file; a file rolls when this or target-file-size is reached, whichever comes first. Enforced at bundle granularity, so a bundled write may exceed it by up to one bundle. Only constrains files at write time: compaction is size-based and may merge into larger files, and data-evolution compaction still produces a single file. Bounds per-file rows for wide columns to avoid data-evolution OOM. PyPaimon supports this for data-evolution append tables; its primary-key, blob and vector writers still fail fast when it is enabled. Disabled by default. + Target number of rows per newly written data file; a file rolls when this or target-file-size is reached, whichever comes first. Enforced at bundle granularity, so a bundled write may exceed it by up to one bundle. Only constrains files at write time: compaction is size-based and may merge into larger files, and data-evolution compaction produces a single file unless data-evolution.compaction.split-large-files is enabled. Bounds per-file rows for wide columns to avoid data-evolution OOM. PyPaimon supports this for data-evolution append tables; its primary-key, blob and vector writers still fail fast when it is enabled. Disabled by default.
target-file-size
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 298126f3c66d..15ddf7623a6f 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -865,7 +865,8 @@ public InlineElement getDescription() { + "Enforced at bundle granularity, so a bundled write may exceed it " + "by up to one bundle. Only constrains files at write time: " + "compaction is size-based and may merge into larger files, and " - + "data-evolution compaction still produces a single file. Bounds " + + "data-evolution compaction produces a single file unless " + + "data-evolution.compaction.split-large-files is enabled. Bounds " + "per-file rows for wide columns to avoid data-evolution OOM. " + "PyPaimon supports this for data-evolution append tables; its " + "primary-key, blob and vector writers still fail fast when it " @@ -2633,6 +2634,17 @@ public String toString() { .withDescription( "Whether to persist source when process merge into action on data evolution table."); + public static final ConfigOption DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES = + key("data-evolution.compaction.split-large-files") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether data-evolution compaction selects normal data files larger than " + + "twice target-file-size, even below compaction.min.file-num. " + + "When enabled, normal compaction output rolls at target-file-size " + + "while preserving row IDs and logical deletions. Associated dedicated " + + "files are rewritten to stay within the new normal-file boundaries."); + public static final ConfigOption DATA_EVOLUTION_COMPACTION_REWRITE_ROW_IDS = key("data-evolution.compaction.rewrite-row-ids") .booleanType() @@ -4389,6 +4401,10 @@ public boolean deletionVectorBitmap64() { return options.get(DELETION_VECTOR_BITMAP64); } + public boolean dataEvolutionCompactionSplitLargeFiles() { + return options.get(DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES); + } + public boolean dataEvolutionCompactionRewriteRowIds() { return options.get(DATA_EVOLUTION_COMPACTION_REWRITE_ROW_IDS); } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java index 947f944182f1..c15c89135d21 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.PriorityQueue; +import static org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator.isLargeFile; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkState; @@ -53,6 +54,7 @@ final class CompactCandidateRangeCollector { private final long blobTargetFileSize; private final long openFileCost; private final long compactMinFileNum; + private final boolean splitLargeFiles; private final List sortedChunks = new ArrayList<>(); private long[] words; private int chunkSize; @@ -64,7 +66,8 @@ final class CompactCandidateRangeCollector { long targetFileSize, long blobTargetFileSize, long openFileCost, - long compactMinFileNum) { + long compactMinFileNum, + boolean splitLargeFiles) { checkArgument(expectedFileCount >= 0, "Expected live file count cannot be negative."); checkArgument(targetFileSize > 0, "Target file size must be positive."); checkArgument(blobTargetFileSize > 0, "Blob target file size must be positive."); @@ -75,6 +78,7 @@ final class CompactCandidateRangeCollector { this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; this.compactMinFileNum = compactMinFileNum; + this.splitLargeFiles = splitLargeFiles; int initialEntries = Math.max(16, Math.min(expectedFileCount, ENTRY_CHUNK_SIZE)); this.words = new long[Math.multiplyExact(initialEntries, ENTRY_WORDS)]; } @@ -137,6 +141,7 @@ void finish(CandidateRangeConsumer consumer) { blobTargetFileSize, openFileCost, compactMinFileNum, + splitLargeFiles, consumer); if (chunks.size() == 1) { SortedEntryChunk chunk = chunks.get(0); @@ -402,6 +407,7 @@ private static final class CandidateAccumulator { private final long blobTargetFileSize; private final long openFileCost; private final long compactMinFileNum; + private final boolean splitLargeFiles; private final CandidateRangeConsumer consumer; private final CandidateBin bin = new CandidateBin(); private final Map blobFields = new HashMap<>(); @@ -412,6 +418,7 @@ private static final class CandidateAccumulator { private long normalEnd; private long normalFileCount; private long normalWeight; + private boolean largeFile; private long vectorFileCount; private int componentFileCount; private boolean hasPreviousLogicalRange; @@ -422,11 +429,13 @@ private CandidateAccumulator( long blobTargetFileSize, long openFileCost, long compactMinFileNum, + boolean splitLargeFiles, CandidateRangeConsumer consumer) { this.targetFileSize = targetFileSize; this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; this.compactMinFileNum = compactMinFileNum; + this.splitLargeFiles = splitLargeFiles; this.consumer = consumer; } @@ -457,6 +466,7 @@ private void startComponent(long start, long end, long fileSize) { normalEnd = end; normalFileCount = 1L; normalWeight = Math.max(fileSize, openFileCost); + largeFile = splitLargeFiles && isLargeFile(fileSize, targetFileSize); vectorFileCount = 0L; componentFileCount = 1; blobFields.clear(); @@ -472,6 +482,7 @@ private void addNormalFile(long start, long end, long fileSize) { checkState( normalEnd == end, "Normal files in one overlapping row-id group must have the same row-id range."); + largeFile |= splitLargeFiles && isLargeFile(fileSize, targetFileSize); normalFileCount = Math.addExact(normalFileCount, 1L); normalWeight = Math.addExact(normalWeight, Math.max(fileSize, openFileCost)); componentFileCount = Math.addExact(componentFileCount, 1); @@ -525,7 +536,8 @@ private void finishComponent() { componentFileCount, normalFileCount, normalWeight, - dedicatedCandidate); + dedicatedCandidate, + largeFile); if (normalWeight > targetFileSize) { flushBin(); emitComponent(component); @@ -541,7 +553,9 @@ private void finishComponent() { } private void emitComponent(Component component) { - if (component.normalFileCount >= compactMinFileNum || component.dedicatedCandidate) { + if (component.normalFileCount >= compactMinFileNum + || component.dedicatedCandidate + || component.largeFile) { consumer.accept(component.start, component.end, component.fileCount); } } @@ -571,6 +585,7 @@ private static final class Component { private final long normalFileCount; private final long normalWeight; private final boolean dedicatedCandidate; + private final boolean largeFile; private Component( long start, @@ -578,13 +593,15 @@ private Component( int fileCount, long normalFileCount, long normalWeight, - boolean dedicatedCandidate) { + boolean dedicatedCandidate, + boolean largeFile) { this.start = start; this.end = end; this.fileCount = fileCount; this.normalFileCount = normalFileCount; this.normalWeight = normalWeight; this.dedicatedCandidate = dedicatedCandidate; + this.largeFile = largeFile; } } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java index 23f15772a034..e19481efa195 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java @@ -46,6 +46,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -118,6 +119,7 @@ public DataEvolutionCompactCoordinator( new DataEvolutionCompactRangePlanner.CandidateOptions( compactBlob, compactVector, + options.dataEvolutionCompactionSplitLargeFiles(), targetFileSize, options.blobTargetFileSize(), openFileCost, @@ -137,6 +139,7 @@ public DataEvolutionCompactCoordinator( new CompactPlanner( compactBlob, compactVector, + options.dataEvolutionCompactionSplitLargeFiles(), targetFileSize, options.blobTargetFileSize(), openFileCost, @@ -145,6 +148,11 @@ public DataEvolutionCompactCoordinator( currentBlobFieldIds); } + static boolean isLargeFile(long fileSize, long targetFileSize) { + // Subtraction avoids overflowing twice the target size. + return fileSize > targetFileSize && fileSize - targetFileSize > targetFileSize; + } + public static void validateOptions(CoreOptions options) { checkArgument( !options.dataEvolutionCompactionRewriteRowIds(), @@ -235,6 +243,7 @@ static class CompactPlanner { private final boolean compactBlob; private final boolean compactVector; + private final boolean splitLargeFiles; private final long targetFileSize; private final long blobTargetFileSize; private final long openFileCost; @@ -252,6 +261,7 @@ static class CompactPlanner { this( compactBlob, compactVector, + false, targetFileSize, targetFileSize, openFileCost, @@ -266,6 +276,7 @@ static class CompactPlanner { CompactPlanner( boolean compactBlob, boolean compactVector, + boolean splitLargeFiles, long targetFileSize, long blobTargetFileSize, long openFileCost, @@ -274,6 +285,7 @@ static class CompactPlanner { @Nullable Set currentBlobFieldIds) { this.compactBlob = compactBlob; this.compactVector = compactVector; + this.splitLargeFiles = splitLargeFiles; this.targetFileSize = targetFileSize; this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; @@ -314,10 +326,10 @@ List compactPlan(List input) { } } - if (compactBlob) { + if (compactBlob || splitLargeFiles) { associateDedicatedFiles(blobFiles, treeMap, dataFileToBlobFiles); } - if (compactVector) { + if (compactVector || splitLargeFiles) { associateDedicatedFiles(vectorStoreFiles, treeMap, dataFileToVectorStoreFiles); } @@ -409,8 +421,32 @@ private List triggerTask( List dataFiles = compactBin.files(); List tasks = new ArrayList<>(); - boolean triggerNormalFile = dataFiles.size() >= compactMinFileNum; + boolean triggerNormalFile = + dataFiles.size() >= compactMinFileNum + || (splitLargeFiles + && dataFiles.stream() + .anyMatch( + f -> + isLargeFile( + f.fileSize(), targetFileSize))); if (triggerNormalFile) { + if (splitLargeFiles) { + // Dedicated files must be rewritten with the normal files so that their + // row-id ranges remain within the new normal-file boundaries. + Set filesToRewrite = new LinkedHashSet<>(dataFiles); + for (DataFileMeta dataFile : dataFiles) { + filesToRewrite.addAll( + dataFileToBlobFiles.getOrDefault( + dataFile, Collections.emptyList())); + filesToRewrite.addAll( + dataFileToVectorStoreFiles.getOrDefault( + dataFile, Collections.emptyList())); + } + tasks.add( + new DataEvolutionNormalCompactTask( + partition, new ArrayList<>(filesToRewrite))); + return tasks; + } tasks.add(new DataEvolutionNormalCompactTask(partition, dataFiles)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java index cb7e36b5fed1..e6454246c7c4 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactDeletionVectorRewriter.java @@ -42,6 +42,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; @@ -136,32 +137,31 @@ private Map collectMaintainers( continue; } - checkState( - after.size() == 1, - "Only one normal file should be generated by a single compact task."); Range beforeRange = checkContiguousRowRange(before); - Range afterRange = after.get(0).nonNullRowIdRange(); + Range afterRange = checkContiguousRowRange(after); checkState( beforeRange.equals(afterRange), "Non-materialized data evolution compaction should keep the same row-id " + "range, but compact before range is %s and compact after range is %s.", beforeRange, afterRange); - DeletionVector merged = newDeletionVector(); + TreeMap outputFiles = new TreeMap<>(); + for (DataFileMeta file : after) { + outputFiles.put(file.nonNullFirstRowId(), file); + } + Map rewritten = new LinkedHashMap<>(); - // Merge all old DeletionVectors, should consider row range offset of each sub dv + // Redistribute old deletion positions across the output file boundaries. for (List previousRowGroup : rangeHelper.mergeOverlappingRanges(before)) { DataFileMeta oldAnchor = retrieveAnchorFile(previousRowGroup, file -> file); moveDeletionVector( maintainer, - merged, + outputFiles, + rewritten, oldAnchor.fileName(), - oldAnchor.nonNullRowIdRange(), - afterRange); - } - if (!merged.isEmpty()) { - maintainer.notifyNewDeletionVector(after.get(0).fileName(), merged); + oldAnchor.nonNullRowIdRange()); } + rewritten.forEach(maintainer::notifyNewDeletionVector); } return result; } @@ -185,27 +185,32 @@ private boolean isMaterialized(CompactIncrement compactIncrement) { } /** - * 'Move' the deletion vectors of old anchor files to the new anchor file. This may merge - * several deletion vectors into one if row-level compaction is triggered, + * Moves old anchor-file deletion vectors to the new files, merging or splitting them at the + * output row-id boundaries. */ private void moveDeletionVector( AppendDeleteFileMaintainer maintainer, - DeletionVector merged, + TreeMap outputFiles, + Map rewritten, String oldFileName, - Range oldRange, - Range newRange) { + Range oldRange) { DeletionVector old = maintainer.getDeletionVector(oldFileName); if (old == null || old.isEmpty()) { return; } - // Fast path for renaming deletion vectors only - if (oldRange.equals(newRange)) { - merged.merge(old); + // Fast path for renaming deletion vectors only. + DataFileMeta firstOutput = outputFiles.floorEntry(oldRange.from).getValue(); + if (oldRange.equals(firstOutput.nonNullRowIdRange())) { + rewritten + .computeIfAbsent(firstOutput.fileName(), ignored -> newDeletionVector()) + .merge(old); } else { old.forEachDeletedPosition( position -> { long absolutePosition = oldRange.from + position; + DataFileMeta output = outputFiles.floorEntry(absolutePosition).getValue(); + Range newRange = output.nonNullRowIdRange(); long newPosition = absolutePosition - newRange.from; checkState( newPosition >= 0 && newPosition < newRange.count(), @@ -213,7 +218,9 @@ private void moveDeletionVector( absolutePosition, oldRange, newRange); - merged.delete(newPosition); + rewritten + .computeIfAbsent(output.fileName(), ignored -> newDeletionVector()) + .delete(newPosition); }); } maintainer.notifyRemovedDeletionVector(oldFileName); diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java index d7abda070804..9c68c1cecb40 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java @@ -157,7 +157,8 @@ private Queue planManifestGroup(List manifestGroup candidateOptions.targetFileSize, candidateOptions.blobTargetFileSize, candidateOptions.openFileCost, - candidateOptions.compactMinFileNum); + candidateOptions.compactMinFileNum, + candidateOptions.splitLargeFiles); try { collectDeletedIdentifiers(manifestGroup, deletedIdentifiers, identifier); collectCandidateRanges(manifestGroup, deletedIdentifiers, identifier, candidateRanges); @@ -489,6 +490,7 @@ static final class CandidateOptions { private final boolean compactBlob; private final boolean compactVector; + private final boolean splitLargeFiles; private final long targetFileSize; private final long blobTargetFileSize; private final long openFileCost; @@ -499,6 +501,7 @@ static final class CandidateOptions { CandidateOptions( boolean compactBlob, boolean compactVector, + boolean splitLargeFiles, long targetFileSize, long blobTargetFileSize, long openFileCost, @@ -507,6 +510,7 @@ static final class CandidateOptions { @Nullable Set currentBlobFieldIds) { this.compactBlob = compactBlob; this.compactVector = compactVector; + this.splitLargeFiles = splitLargeFiles; this.targetFileSize = targetFileSize; this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java index 0019c4e17f92..6f24a41bcc16 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java @@ -61,7 +61,7 @@ private static Map dynamicWriteOptions() { Map options = new HashMap<>(); options.put(CoreOptions.TARGET_FILE_SIZE.key(), "99999 G"); options.put(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "99999 G"); - // Data evolution requires a single output file, so the row limit must not roll it either. + // Compaction is size-based; the write-time row limit must not roll its output. options.put(CoreOptions.TARGET_FILE_ROW_NUM.key(), String.valueOf(Long.MAX_VALUE)); return Collections.unmodifiableMap(options); } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java index 7fe573974410..0d41add7b2b7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java @@ -43,6 +43,7 @@ import javax.annotation.Nullable; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +51,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; import static org.apache.paimon.types.BlobType.fieldNamesInBlobFile; import static org.apache.paimon.types.VectorType.fieldNamesInVectorFile; import static org.apache.paimon.types.VectorType.isVectorStoreFile; @@ -86,13 +88,34 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E fieldNamesInBlobFile(table.rowType(), options.blobInlineField()), fieldNamesInVectorFile(table.rowType(), options.withVectorFormat())); - table = table.copy(DYNAMIC_WRITE_OPTIONS); + Function schemaLoader = table.schemaManager()::schema; + Set dedicatedFieldsToRewrite = + compactBefore.stream() + .filter( + file -> + isBlobFile(file.fileName()) + || isVectorStoreFile(file.fileName())) + .flatMap(file -> fileFields(schemaLoader, file).stream()) + .map(DataField::id) + .collect(Collectors.toSet()); + Map writeOptions = new HashMap<>(DYNAMIC_WRITE_OPTIONS); + if (options.dataEvolutionCompactionSplitLargeFiles()) { + writeOptions.put( + CoreOptions.TARGET_FILE_SIZE.key(), options.targetFileSize(false) + " b"); + writeOptions.put( + CoreOptions.BLOB_TARGET_FILE_SIZE.key(), options.blobTargetFileSize() + " b"); + } + table = table.copy(writeOptions); long firstRowId = compactBefore.get(0).nonNullFirstRowId(); RowType readWriteType = new RowType( table.rowType().getFields().stream() - .filter(f -> !fieldsInDedicatedFile.contains(f.name())) + .filter( + f -> + !fieldsInDedicatedFile.contains(f.name()) + || dedicatedFieldsToRewrite.contains( + f.id())) .collect(Collectors.toList())); FileStorePathFactory pathFactory = table.store().pathFactory(); AppendOnlyFileStore store = (AppendOnlyFileStore) table.store(); @@ -123,7 +146,8 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E List writeResult = writer.prepareCommit(false).newFilesIncrement().newFiles(); checkArgument( - writeResult.size() == 1, "Data evolution compaction should produce one file."); + options.dataEvolutionCompactionSplitLargeFiles() || writeResult.size() == 1, + "Data evolution compaction should produce one file unless splitting is enabled."); try { writer.close(); @@ -132,18 +156,34 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E LOG.warn("Failed to close reader and writer.", e); } - DataFileMeta dataFileMeta = writeResult.get(0).assignFirstRowId(firstRowId); - dataFileMeta = - dataFileMeta.assignSequenceNumber( - minSequenceId(compactBefore), maxSequenceId(compactBefore)); - if (options.ignoreIndexColumnUpdate()) { - long[] columnMaxSequenceNumbers = - compactedColumnMaxSequenceNumbers(table, dataFileMeta); - if (columnMaxSequenceNumbers != null) { - dataFileMeta = dataFileMeta.withColumnMaxSequenceNumbers(columnMaxSequenceNumbers); + long minSequenceNumber = minSequenceId(compactBefore); + long maxSequenceNumber = maxSequenceId(compactBefore); + Map, Long> nextRowIds = new HashMap<>(); + Map, long[]> columnSequences = new HashMap<>(); + for (DataFileMeta file : writeResult) { + List columnGroup = + isBlobFile(file.fileName()) || isVectorStoreFile(file.fileName()) + ? file.writeCols() + : Collections.emptyList(); + long fileFirstRowId = nextRowIds.getOrDefault(columnGroup, firstRowId); + DataFileMeta dataFileMeta = + file.assignFirstRowId(fileFirstRowId) + .assignSequenceNumber(minSequenceNumber, maxSequenceNumber); + if (options.ignoreIndexColumnUpdate()) { + if (!columnSequences.containsKey(columnGroup)) { + columnSequences.put( + columnGroup, compactedColumnMaxSequenceNumbers(table, dataFileMeta)); + } + long[] columnMaxSequenceNumbers = columnSequences.get(columnGroup); + if (columnMaxSequenceNumbers != null) { + dataFileMeta = + dataFileMeta.withColumnMaxSequenceNumbers(columnMaxSequenceNumbers); + } } + compactAfter.add(dataFileMeta); + nextRowIds.put(columnGroup, fileFirstRowId + dataFileMeta.rowCount()); } - compactAfter.add(dataFileMeta); + checkSameRowRange("Normal file", compactBefore, compactAfter); return commitMessage(compactBefore, compactAfter); } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java index 29c368f9f77e..ca6a640ff2c9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java @@ -45,6 +45,33 @@ void testSelectsOnlyNormalFileBinsWhichCanCompact() { assertThat(collector.retainedWordCount()).isZero(); } + @Test + void testSplitLargeFilesUsesPhysicalSizeAndIncludesColumnUpdates() { + for (boolean enabled : new boolean[] {false, true}) { + CompactCandidateRangeCollector collector = + new CompactCandidateRangeCollector(16, 100L, 100L, 1000L, 10L, enabled); + collector.add(0, NORMAL_FILE, 0L, 10L, 199L); + collector.add(0, NORMAL_FILE, 10L, 10L, 200L); + collector.add(0, NORMAL_FILE, 20L, 10L, 201L); + collector.add(0, NORMAL_FILE, 20L, 10L, 10L); + // Dedicated files never trigger normal-file splitting. + collector.add(0, 3, 0L, 10L, 1000L); + if (enabled) { + assertThat(finish(collector)).containsExactly("20-29:2"); + } else { + assertThat(finish(collector)).isEmpty(); + } + } + } + + @Test + void testSplitThresholdDoesNotOverflow() { + CompactCandidateRangeCollector collector = + new CompactCandidateRangeCollector(16, Long.MAX_VALUE, 100L, 1L, 2L, true); + collector.add(0, NORMAL_FILE, 0L, 10L, Long.MAX_VALUE); + assertThat(finish(collector)).isEmpty(); + } + @Test void testSelectsUpdatedFilesEvenWhenOneLogicalRangeExceedsTarget() { CompactCandidateRangeCollector collector = collector(100L, 100L, 1L, 2L); @@ -156,7 +183,7 @@ private CompactCandidateRangeCollector collector( long openFileCost, long compactMinFileNum) { return new CompactCandidateRangeCollector( - 16, targetFileSize, blobTargetFileSize, openFileCost, compactMinFileNum); + 16, targetFileSize, blobTargetFileSize, openFileCost, compactMinFileNum, false); } private List finish(CompactCandidateRangeCollector collector) { diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java index cd784064bbae..097529d6a7b6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java @@ -104,6 +104,29 @@ public void testCompactPlannerSingleFile() { assertThat(tasks).isEmpty(); } + @Test + public void testSplitLargeFilesUsesPhysicalSizeAndIncludesColumnUpdates() { + List entries = + Arrays.asList( + makeEntryWithSize("below.parquet", 0L, 10L, 0, 199L), + makeEntryWithSize("boundary.parquet", 10L, 10L, 0, 200L), + makeEntryWithSize("large.parquet", 20L, 10L, 0, 201L), + makeEntryWithSize("update.parquet", 20L, 10L, 1, 10L)); + for (boolean enabled : new boolean[] {false, true}) { + DataEvolutionCompactCoordinator.CompactPlanner planner = + new DataEvolutionCompactCoordinator.CompactPlanner( + false, false, enabled, 100L, 100L, 1000L, 10L, schemaId -> null, null); + List tasks = planner.compactPlan(entries); + if (enabled) { + assertThat(tasks).hasSize(1); + assertThat(tasks.get(0).compactBefore()) + .containsExactly(entries.get(2).file(), entries.get(3).file()); + } else { + assertThat(tasks).isEmpty(); + } + } + } + @Test public void testCompactPlannerContiguousFiles() { // Multiple contiguous files should be grouped together @@ -876,6 +899,7 @@ private DataEvolutionCompactCoordinator.CompactPlanner blobPlanner( return new DataEvolutionCompactCoordinator.CompactPlanner( true, false, + false, targetFileSize, targetFileSize, openFileCost, diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java index 88278ec40f54..28c3afe29e75 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java @@ -400,6 +400,7 @@ private DataEvolutionCompactRangePlanner.CandidateOptions candidateOptions( return new DataEvolutionCompactRangePlanner.CandidateOptions( compactBlob, compactVector, + false, 100L, 100L, 1L, diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java index dd91e37f3a48..a515e2c64803 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java @@ -21,10 +21,15 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.BinaryVector; +import org.apache.paimon.data.BlobData; import org.apache.paimon.data.GenericRow; +import org.apache.paimon.data.InternalRow; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.TableTestBase; @@ -33,11 +38,15 @@ import org.apache.paimon.table.sink.BatchWriteBuilder; import org.apache.paimon.table.sink.CommitMessage; import org.apache.paimon.table.sink.CommitMessageImpl; +import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.Range; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.ArrayList; import java.util.Arrays; @@ -45,12 +54,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.stream.Collectors; +import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; +import static org.apache.paimon.types.VectorType.isVectorStoreFile; import static org.apache.paimon.utils.DataEvolutionUtils.fileFields; import static org.assertj.core.api.Assertions.assertThat; -/** Tests for column sequence propagation in {@link DataEvolutionNormalCompactTask}. */ +/** + * Tests for splitting and column sequence propagation in {@link DataEvolutionNormalCompactTask}. + */ public class DataEvolutionNormalCompactTaskTest extends TableTestBase { private static final int ROW_COUNT = 100; @@ -127,6 +141,198 @@ public void testOmitColumnSequencesUnlessUpdatesAreIgnored() throws Exception { assertThat(compacted.columnMaxSequenceNumbers()).isNull(); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testSplitHistoricalLargeFile(boolean updateColumn) throws Exception { + createTableDefault(); + FileStoreTable table = getTableDefault(); + int rowCount = 12000; + Random random = new Random(42); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + for (int i = 0; i < rowCount; i++) { + StringBuilder value = new StringBuilder(); + for (int j = 0; j < 8; j++) { + value.append(Long.toHexString(random.nextLong())); + } + write.write( + GenericRow.of( + BinaryString.fromString("p0"), + i, + BinaryString.fromString(value.toString()))); + } + commit.commit(write.prepareCommit()); + } + DataFileMeta original = table.store().newScan().plan().files().get(0).file(); + if (updateColumn) { + builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = + builder.newWrite() + .withWriteType( + table.rowType().project(Arrays.asList("dt", "f0"))); + BatchTableCommit commit = builder.newCommit()) { + for (int i = 0; i < rowCount; i++) { + write.write(GenericRow.of(BinaryString.fromString("p0"), i + rowCount)); + } + List messages = write.prepareCommit(); + assignFirstRowId(messages, original.nonNullFirstRowId()); + commit.commit(messages); + } + } + List expected = read(table); + Map options = new HashMap<>(); + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "128 kb"); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "10"); + options.put(CoreOptions.GLOBAL_INDEX_COLUMN_UPDATE_ACTION.key(), "ignore"); + table = table.copy(options); + long targetSize = table.coreOptions().targetFileSize(false); + assertThat(original.fileSize()).isGreaterThan(2 * targetSize); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + assertThat(new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan()) + .isEmpty(); + + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + table = table.copy(options); + List tasks = + new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan(); + assertThat(tasks).hasSize(1); + DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer(); + DataEvolutionCompactTask task = + serializer.deserialize(serializer.getVersion(), serializer.serialize(tasks.get(0))); + List messages = new ArrayList<>(); + messages.add(task.doCompact(table, "split-large-file")); + List output = task.compactAfter(); + assertThat(output.size()).isGreaterThan(1); + long nextRowId = original.nonNullFirstRowId(); + long maxSequenceNumber = + task.compactBefore().stream() + .mapToLong(DataFileMeta::maxSequenceNumber) + .max() + .getAsLong(); + for (DataFileMeta file : output) { + assertThat(file.nonNullFirstRowId()).isEqualTo(nextRowId); + assertThat(file.fileSize()).isLessThan(2 * targetSize); + assertThat(file.minSequenceNumber()).isEqualTo(original.minSequenceNumber()); + assertThat(file.maxSequenceNumber()).isEqualTo(maxSequenceNumber); + if (updateColumn) { + assertThat(columnSequence(file, table.rowType().getField("f1").id())) + .isEqualTo(original.maxSequenceNumber()); + assertThat(columnSequence(file, table.rowType().getField("f0").id())) + .isEqualTo(maxSequenceNumber); + } + nextRowId += file.rowCount(); + } + assertThat(nextRowId).isEqualTo(original.nonNullFirstRowId() + rowCount); + messages.addAll( + new DataEvolutionCompactionCommitPreparation(table, snapshot).prepare(messages)); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(messages); + } + assertThat(read(table)).containsExactlyInAnyOrderElementsOf(expected); + assertThat( + new DataEvolutionCompactCoordinator( + table, + false, + false, + table.snapshotManager().latestSnapshot()) + .plan()) + .isEmpty(); + } + + @Test + public void testSplitAlignsMultipleBlobColumnsAndVectorFiles() throws Exception { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("b1", DataTypes.BLOB()) + .column("b2", DataTypes.BLOB()) + .column("v", DataTypes.VECTOR(2, DataTypes.FLOAT())) + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "128 kb") + .option(CoreOptions.VECTOR_TARGET_FILE_SIZE.key(), "128 kb") + .option(CoreOptions.VECTOR_FILE_FORMAT.key(), "json") + .option(CoreOptions.FILE_COMPRESSION.key(), "none") + .build(); + catalog.createTable(identifier(), schema, false); + FileStoreTable table = getTableDefault(); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + for (int i = 0; i < 2500; i++) { + write.write( + GenericRow.of( + i, + new BlobData(new byte[] {(byte) i}), + new BlobData(new byte[] {(byte) (i + 1)}), + BinaryVector.fromPrimitiveArray(new float[] {i, i + 1}))); + } + commit.commit(write.prepareCommit()); + } + catalog.alterTable( + identifier(), + Collections.singletonList(SchemaChange.renameColumn("v", "renamed_v")), + false); + table = getTableDefault(); + Map options = new HashMap<>(); + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + table = table.copy(options); + List tasks = + new DataEvolutionCompactCoordinator( + table, false, false, table.snapshotManager().latestSnapshot()) + .plan(); + assertThat(tasks).hasSize(1); + DataEvolutionCompactTask task = tasks.get(0); + assertThat(task.compactBefore()).hasSize(4); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(Collections.singletonList(task.doCompact(table, "split-dedicated"))); + } + List normalRanges = + task.compactAfter().stream() + .filter( + file -> + !isBlobFile(file.fileName()) + && !isVectorStoreFile(file.fileName())) + .map(DataFileMeta::nonNullRowIdRange) + .collect(Collectors.toList()); + assertThat(normalRanges.size()).isGreaterThan(1); + assertThat(task.compactAfter()) + .allSatisfy( + file -> + assertThat( + normalRanges.stream() + .anyMatch( + range -> + range.from + <= file + .nonNullFirstRowId() + && range.to + >= file + .nonNullRowIdRange() + .to)) + .isTrue()); + ReadBuilder readBuilder = table.newReadBuilder(); + List ids = new ArrayList<>(); + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan())) { + reader.forEachRemaining( + row -> { + int id = row.getInt(0); + ids.add(id); + assertThat(row.getBlob(1).toData()).containsExactly((byte) id); + assertThat(row.getBlob(2).toData()).containsExactly((byte) (id + 1)); + assertThat(row.getVector(3).toFloatArray()).containsExactly(id, id + 1); + }); + } + assertThat(ids) + .containsExactlyElementsOf( + java.util.stream.IntStream.range(0, 2500) + .boxed() + .collect(Collectors.toList())); + } + private void write() throws Exception { createTableDefault(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java index 384f3fe2a306..9853a5bf32fa 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java @@ -67,6 +67,8 @@ import org.apache.paimon.utils.RangeHelper; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.lang.reflect.Constructor; @@ -82,6 +84,7 @@ import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX; import static org.apache.paimon.errors.ErrorMessages.DATA_EVOLUTION_ROW_ID_CONFLICT_MESSAGE; +import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET; import static org.apache.paimon.types.VectorType.isVectorStoreFile; import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile; @@ -581,6 +584,72 @@ public void testCompactRewritesDeletionVectors() throws Exception { assertThat(liveDeletionVectorDataFileNames).doesNotContainAnyElementsOf(oldAnchorFiles); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testSplitLargeFilePreservesDeletionVectorsAndBlob(boolean bitmap64) + throws Exception { + createTableDefault(); + Map options = new HashMap<>(); + options.put(CoreOptions.DELETION_VECTOR_BITMAP64.key(), String.valueOf(bitmap64)); + options.put(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "128 mb"); + FileStoreTable table = getTableDefault().copy(options); + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + for (int rowId = 0; rowId < 2500; rowId++) { + write.write( + GenericRow.of( + rowId, + BinaryString.fromString("name-" + rowId), + BinaryString.fromString("base-" + rowId), + new BlobData(new byte[] {(byte) rowId}))); + } + commit.commit(write.prepareCommit()); + } + Range range = new Range(0, 2499); + commitDeletionVectors( + table, Collections.singletonList(new DvSpec(range, 0, 999, 2000, 2499))); + List expected = readRows(table.newReadBuilder()); + String oldAnchor = anchorFilesByRange(table).get(range); + List blobs = + currentDataFiles(table, BinaryRow.EMPTY_ROW).stream() + .filter(file -> isBlobFile(file.fileName())) + .collect(Collectors.toList()); + assertThat(blobs).hasSize(1); + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + table = table.copy(options); + compactDataEvolutionTable(table, false); + + List files = currentDataFiles(table, BinaryRow.EMPTY_ROW); + List normalFiles = + files.stream() + .filter(file -> !isBlobFile(file.fileName())) + .collect(Collectors.toList()); + assertThat(normalFiles.size()).isGreaterThan(1); + assertThat(normalFiles.stream().mapToLong(DataFileMeta::rowCount).sum()).isEqualTo(2500); + assertThat(files).doesNotContainAnyElementsOf(blobs); + assertThat(readRows(table.newReadBuilder())).containsExactlyElementsOf(expected); + List expectedAnchors = new ArrayList<>(); + for (DataFileMeta file : normalFiles) { + Range fileRange = file.nonNullRowIdRange(); + if (Arrays.stream(new long[] {0, 999, 2000, 2499}) + .anyMatch(id -> id >= fileRange.from && id <= fileRange.to)) { + expectedAnchors.add(file.fileName()); + } + } + assertThat(liveDeletionVectorDataFileNames(table)) + .containsExactlyInAnyOrderElementsOf(expectedAnchors) + .doesNotContain(oldAnchor); + ReadBuilder readBuilder = + table.newReadBuilder() + .withReadType(SpecialFields.rowTypeWithRowId(table.rowType())); + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan())) { + reader.forEachRemaining(row -> assertThat(row.getLong(4)).isEqualTo(row.getInt(0))); + } + } + @Test public void testCompactRenamesDeletionVectorForSameRowRange() throws Exception { createTableDefault(); From 1ef1522c5dca94bd684f8e4eb20cf6905c0eb40b Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Wed, 9 Sep 2026 23:45:01 +0800 Subject: [PATCH 2/5] [core] Preserve dedicated files when splitting normal files --- docs/docs/multimodal-table/data-evolution.mdx | 14 +- docs/generated/core_configuration.html | 2 +- .../java/org/apache/paimon/CoreOptions.java | 4 +- .../DataEvolutionCompactCoordinator.java | 22 +- .../DataEvolutionNormalCompactTask.java | 52 +-- .../operation/DataEvolutionFileStoreScan.java | 30 +- .../operation/DataEvolutionSplitRead.java | 368 +++++++++--------- .../DataEvolutionConflictDetection.java | 48 ++- .../apache/paimon/table/source/DataSplit.java | 21 +- .../DataEvolutionCompactCoordinatorTest.java | 21 + .../DataEvolutionNormalCompactTaskTest.java | 70 +++- .../DataEvolutionFileStoreScanTest.java | 71 ++++ .../operation/DataEvolutionReadTest.java | 154 ++++---- .../operation/DataEvolutionSplitReadTest.java | 279 +++++++++++++ .../commit/ConflictDetectionTest.java | 129 +++++- .../DataEvolutionDeletionVectorTest.java | 49 ++- .../table/source/DataSplitCompatibleTest.java | 53 +++ .../pypaimon/read/reader/field_bunch.py | 32 +- .../scanner/chunk_shuffle_split_generator.py | 21 +- paimon-python/pypaimon/read/split_read.py | 134 ++++--- .../data_evolution_spanning_dedicated_test.py | 237 +++++++++++ .../tests/write/conflict_detection_test.py | 78 +++- .../pypaimon/utils/data_evolution_utils.py | 42 +- .../write/commit/conflict_detection.py | 28 +- paimon-python/pypaimon/write/table_delete.py | 7 +- 25 files changed, 1529 insertions(+), 437 deletions(-) create mode 100644 paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py diff --git a/docs/docs/multimodal-table/data-evolution.mdx b/docs/docs/multimodal-table/data-evolution.mdx index 3186523014a8..1f3de44363e4 100644 --- a/docs/docs/multimodal-table/data-evolution.mdx +++ b/docs/docs/multimodal-table/data-evolution.mdx @@ -628,11 +628,15 @@ range and rolls normal output files near `target-file-size`. Actual sizes depend compression and the writer's size-check granularity; the last file may be smaller. The write-time `target-file-row-num` limit does not apply to compaction. -Row IDs, column updates, and logical deletions are preserved. Associated dedicated -BLOB and VECTOR files are rewritten along with the normal files to keep their row-ID -ranges aligned, which adds read and write work for those columns. Their sizes alone -do not trigger splitting. Compaction rewrites files in the current snapshot; files referenced by older snapshots or tags remain until those references -expire and snapshot expiration removes them. +Row IDs, column updates, and logical deletions are preserved. This option only rewrites +normal files: associated BLOB and VECTOR files keep their existing contents and file names, +and their sizes do not trigger splitting. Separate dedicated-file compaction options keep +their existing behavior. Files referenced by older snapshots or tags remain until those +references expire and snapshot expiration removes them. + +After splitting, one dedicated file can cover multiple normal-file row-ID ranges. +All Java and PyPaimon readers must support this layout before enabling the option. +Disabling the option later does not undo this layout change. ### Materialize Deletion Vectors diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index d6d156d885e7..57c50f641772 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -516,7 +516,7 @@
data-evolution.compaction.split-large-files
false Boolean - Whether data-evolution compaction selects normal data files larger than twice target-file-size, even below compaction.min.file-num. When enabled, normal compaction output rolls at target-file-size while preserving row IDs and logical deletions. Associated dedicated files are rewritten to stay within the new normal-file boundaries. + Whether data-evolution compaction selects normal data files larger than twice target-file-size, even below compaction.min.file-num. When enabled, normal compaction output rolls at target-file-size while preserving row IDs and logical deletions. This option does not rewrite associated BLOB or VECTOR files.
data-evolution.enabled
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 15ddf7623a6f..1b261cf24ed9 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2642,8 +2642,8 @@ public String toString() { "Whether data-evolution compaction selects normal data files larger than " + "twice target-file-size, even below compaction.min.file-num. " + "When enabled, normal compaction output rolls at target-file-size " - + "while preserving row IDs and logical deletions. Associated dedicated " - + "files are rewritten to stay within the new normal-file boundaries."); + + "while preserving row IDs and logical deletions. This option does not " + + "rewrite associated BLOB or VECTOR files."); public static final ConfigOption DATA_EVOLUTION_COMPACTION_REWRITE_ROW_IDS = key("data-evolution.compaction.rewrite-row-ids") diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java index e19481efa195..1bb11c5be697 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java @@ -46,7 +46,6 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -326,10 +325,10 @@ List compactPlan(List input) { } } - if (compactBlob || splitLargeFiles) { + if (compactBlob) { associateDedicatedFiles(blobFiles, treeMap, dataFileToBlobFiles); } - if (compactVector || splitLargeFiles) { + if (compactVector) { associateDedicatedFiles(vectorStoreFiles, treeMap, dataFileToVectorStoreFiles); } @@ -430,23 +429,6 @@ private List triggerTask( isLargeFile( f.fileSize(), targetFileSize))); if (triggerNormalFile) { - if (splitLargeFiles) { - // Dedicated files must be rewritten with the normal files so that their - // row-id ranges remain within the new normal-file boundaries. - Set filesToRewrite = new LinkedHashSet<>(dataFiles); - for (DataFileMeta dataFile : dataFiles) { - filesToRewrite.addAll( - dataFileToBlobFiles.getOrDefault( - dataFile, Collections.emptyList())); - filesToRewrite.addAll( - dataFileToVectorStoreFiles.getOrDefault( - dataFile, Collections.emptyList())); - } - tasks.add( - new DataEvolutionNormalCompactTask( - partition, new ArrayList<>(filesToRewrite))); - return tasks; - } tasks.add(new DataEvolutionNormalCompactTask(partition, dataFiles)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java index 0d41add7b2b7..62f455fb7c0e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java @@ -43,7 +43,6 @@ import javax.annotation.Nullable; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,7 +50,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; import static org.apache.paimon.types.BlobType.fieldNamesInBlobFile; import static org.apache.paimon.types.VectorType.fieldNamesInVectorFile; import static org.apache.paimon.types.VectorType.isVectorStoreFile; @@ -88,22 +86,10 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E fieldNamesInBlobFile(table.rowType(), options.blobInlineField()), fieldNamesInVectorFile(table.rowType(), options.withVectorFormat())); - Function schemaLoader = table.schemaManager()::schema; - Set dedicatedFieldsToRewrite = - compactBefore.stream() - .filter( - file -> - isBlobFile(file.fileName()) - || isVectorStoreFile(file.fileName())) - .flatMap(file -> fileFields(schemaLoader, file).stream()) - .map(DataField::id) - .collect(Collectors.toSet()); Map writeOptions = new HashMap<>(DYNAMIC_WRITE_OPTIONS); if (options.dataEvolutionCompactionSplitLargeFiles()) { writeOptions.put( CoreOptions.TARGET_FILE_SIZE.key(), options.targetFileSize(false) + " b"); - writeOptions.put( - CoreOptions.BLOB_TARGET_FILE_SIZE.key(), options.blobTargetFileSize() + " b"); } table = table.copy(writeOptions); long firstRowId = compactBefore.get(0).nonNullFirstRowId(); @@ -111,11 +97,7 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E RowType readWriteType = new RowType( table.rowType().getFields().stream() - .filter( - f -> - !fieldsInDedicatedFile.contains(f.name()) - || dedicatedFieldsToRewrite.contains( - f.id())) + .filter(f -> !fieldsInDedicatedFile.contains(f.name())) .collect(Collectors.toList())); FileStorePathFactory pathFactory = table.store().pathFactory(); AppendOnlyFileStore store = (AppendOnlyFileStore) table.store(); @@ -158,30 +140,24 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E long minSequenceNumber = minSequenceId(compactBefore); long maxSequenceNumber = maxSequenceId(compactBefore); - Map, Long> nextRowIds = new HashMap<>(); - Map, long[]> columnSequences = new HashMap<>(); + long nextRowId = firstRowId; + long[] columnMaxSequenceNumbers = + options.ignoreIndexColumnUpdate() && !writeResult.isEmpty() + ? compactedColumnMaxSequenceNumbers( + table, + writeResult + .get(0) + .assignSequenceNumber(minSequenceNumber, maxSequenceNumber)) + : null; for (DataFileMeta file : writeResult) { - List columnGroup = - isBlobFile(file.fileName()) || isVectorStoreFile(file.fileName()) - ? file.writeCols() - : Collections.emptyList(); - long fileFirstRowId = nextRowIds.getOrDefault(columnGroup, firstRowId); DataFileMeta dataFileMeta = - file.assignFirstRowId(fileFirstRowId) + file.assignFirstRowId(nextRowId) .assignSequenceNumber(minSequenceNumber, maxSequenceNumber); - if (options.ignoreIndexColumnUpdate()) { - if (!columnSequences.containsKey(columnGroup)) { - columnSequences.put( - columnGroup, compactedColumnMaxSequenceNumbers(table, dataFileMeta)); - } - long[] columnMaxSequenceNumbers = columnSequences.get(columnGroup); - if (columnMaxSequenceNumbers != null) { - dataFileMeta = - dataFileMeta.withColumnMaxSequenceNumbers(columnMaxSequenceNumbers); - } + if (columnMaxSequenceNumbers != null) { + dataFileMeta = dataFileMeta.withColumnMaxSequenceNumbers(columnMaxSequenceNumbers); } compactAfter.add(dataFileMeta); - nextRowIds.put(columnGroup, fileFirstRowId + dataFileMeta.rowCount()); + nextRowId += dataFileMeta.rowCount(); } checkSameRowRange("Normal file", compactBefore, compactAfter); diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java index ee054d445c2f..214aeb147160 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java @@ -223,15 +223,14 @@ private boolean filterByStats(List entries) { * a row-disjoint pre-ALTER group), one file is kept as a row-count representative so the reader * can emit the right number of NULL-filled rows. * - *

If Deletion-Vector is enabled, we always keep the oldest normal file for each group as the - * anchor file to lookup corresponding Deletion Files. + *

If Deletion-Vector is enabled, we always keep the oldest normal file for each normal + * row-id range as the anchor file to lookup corresponding Deletion Files. A retained dedicated + * file can span several normal ranges after normal-file compaction. */ private List pruneByReadType(List group) { if (readType == null || group.size() <= 1) { return group; } - ManifestEntry anchor = - deletionVectorsEnabled ? retrieveAnchorFile(group, ManifestEntry::file) : null; Set readFieldIds = new HashSet<>(); for (DataField f : readType.getFields()) { readFieldIds.add(f.id()); @@ -246,8 +245,27 @@ private List pruneByReadType(List group) { } } } - if (anchor != null && !kept.contains(anchor)) { - kept.add(anchor); + List normalFiles = + group.stream() + .filter(entry -> !isBlobFile(entry.file().fileName())) + .filter(entry -> !isVectorStoreFile(entry.file().fileName())) + .collect(Collectors.toList()); + RangeHelper rangeHelper = + new RangeHelper<>(entry -> entry.file().nonNullRowIdRange()); + List> normalGroups = rangeHelper.mergeOverlappingRanges(normalFiles); + Set keptFiles = new HashSet<>(kept); + for (List normalGroup : normalGroups) { + if (deletionVectorsEnabled) { + ManifestEntry anchor = retrieveAnchorFile(normalGroup, ManifestEntry::file); + if (anchor != null && keptFiles.add(anchor)) { + kept.add(anchor); + } + } else if (normalGroups.size() > 1 + && normalGroup.stream().noneMatch(keptFiles::contains)) { + // Preserve each normal range even when only a spanning dedicated column or a + // newly added column is projected. A single representative would lose rows. + kept.add(normalGroup.get(0)); + } } // Group must contribute at least one file so the reader sees rowCount and can NULL-fill // missing columns for the projection's rows. diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java index 919100ec1f8f..e8ab18f26954 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java @@ -94,7 +94,6 @@ import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile; import static org.apache.paimon.utils.ListUtils.isNullOrEmpty; import static org.apache.paimon.utils.Preconditions.checkArgument; -import static org.apache.paimon.utils.Preconditions.checkNotNull; /** * A union {@link SplitRead} to read multiple inner files to merge columns. @@ -231,6 +230,10 @@ private RecordReader createReader( List> splitByRowId = mergeRangesAndSort(files); for (List needMergeFiles : splitByRowId) { + List groupRowRanges = groupRowRanges(needMergeFiles, rowRanges); + if (groupRowRanges != null && groupRowRanges.isEmpty()) { + continue; + } if (needMergeFiles.size() == 1 || readRowType.getFields().isEmpty()) { // No need to merge fields, just create a single file reader suppliers.add( @@ -242,7 +245,7 @@ private RecordReader createReader( dataFilePathFactory, needMergeFiles.get(0), filters, - rowRanges, + groupRowRanges, readRowType, deletionVector); }); @@ -259,7 +262,7 @@ private RecordReader createReader( needMergeFiles, partition, dataFilePathFactory, - rowRanges, + groupRowRanges, readRowType, deletionVector); }); @@ -269,6 +272,37 @@ private RecordReader createReader( return ConcatRecordReader.create(suppliers); } + private static List groupRowRanges( + List files, @Nullable List rowRanges) { + DataFileMeta first = files.get(0); + if (isBlobFile(first.fileName()) || isVectorStoreFile(first.fileName())) { + return rowRanges; + } + Range normalRange = first.nonNullRowIdRange(); + boolean needsClipping = + files.stream() + .anyMatch( + file -> { + Range range = file.nonNullRowIdRange(); + return range.from < normalRange.from + || range.to > normalRange.to; + }); + if (!needsClipping) { + return rowRanges; + } + if (rowRanges == null) { + return Collections.singletonList(normalRange); + } + List intersections = new ArrayList<>(); + for (Range range : rowRanges) { + Range intersection = Range.intersection(normalRange, range); + if (intersection != null) { + intersections.add(intersection); + } + } + return intersections; + } + private RecordReader createReader(IndexedSplit indexedSplit) throws IOException { DataSplit dataSplit = indexedSplit.dataSplit(); List rowRanges = indexedSplit.rowRanges(); @@ -326,7 +360,16 @@ private DataEvolutionFileReader createUnionReader( DataFileMeta first = fieldsFiles.get(i).files().get(0); bunchDataSchemas[i] = schemaFetcher.apply(first.schemaId()).dataFileSchema(first.writeCols()); - bunchAvailTypes.add(rowTypeWithRowTracking(bunchDataSchemas[i].logicalRowType())); + RowType availableType = bunchDataSchemas[i].logicalRowType(); + if (fieldsFiles.get(i) instanceof VectorFileBunch) { + int fieldId = ((VectorFileBunch) fieldsFiles.get(i)).fieldId; + availableType = + new RowType( + availableType.getFields().stream() + .filter(field -> field.id() == fieldId) + .collect(Collectors.toList())); + } + bunchAvailTypes.add(rowTypeWithRowTracking(availableType)); } DataEvolutionReadPlanner.DataEvolutionReadPlan plan = new DataEvolutionReadPlanner(readRowType, bunchAvailTypes, nestedFieldEnabled) @@ -456,10 +499,10 @@ private RecordReader createFieldBunchReader( } else if (bunch instanceof VectorFileBunch) { // for vector bunch, sequential read all data files and concat them return sequentialReadFiles( - bunch.files(), + (VectorFileBunch) bunch, partition, dataFilePathFactory, - formatReaderMapping, + readRowType, rowRanges, deletionVector); } else if (bunch instanceof BlobFileBunch) { @@ -490,22 +533,37 @@ private RecordReader createFieldBunchReader( } private RecordReader sequentialReadFiles( - List files, + VectorFileBunch bunch, BinaryRow partition, DataFilePathFactory dataFilePathFactory, - FormatReaderMapping formatReaderMapping, + RowType readRowType, List rowRanges, @Nullable DeletionVectorWithRange deletionVector) throws IOException { List> readerSuppliers = new ArrayList<>(); - for (DataFileMeta file : files) { + for (VectorFileRange selected : bunch.selectedFiles()) { + DataFileMeta file = selected.file; + List selectedRanges = new ArrayList<>(); + if (rowRanges == null) { + selectedRanges.add(selected.range); + } else { + for (Range range : rowRanges) { + Range intersection = Range.intersection(range, selected.range); + if (intersection != null) { + selectedRanges.add(intersection); + } + } + } + if (selectedRanges.isEmpty()) { + continue; + } readerSuppliers.add( () -> createFileReader( partition, file, - formatReaderMapping, - rowRanges, + vectorReaderMapping(file, readRowType), + selectedRanges, readRowType, new FileReadTarget( DataFilePathFactory.formatIdentifier(file.fileName()), @@ -517,6 +575,20 @@ private RecordReader sequentialReadFiles( return ConcatRecordReader.create(readerSuppliers); } + private FormatReaderMapping vectorReaderMapping(DataFileMeta file, RowType readRowType) { + String formatIdentifier = DataFilePathFactory.formatIdentifier(file.fileName()); + TableSchema dataSchema = + schemaFetcher.apply(file.schemaId()).dataFileSchema(file.writeCols()); + List readFields = readRowType.getFields(); + boolean nestedFieldEnabled = nestedFieldEnabledFor(Collections.singletonList(file)); + List cacheKey = readerCacheKey(readFields, dataSchema.fields(), nestedFieldEnabled); + return formatReaderMappings.computeIfAbsent( + new FormatKey(file.schemaId(), formatIdentifier, cacheKey), + key -> + formatBuilder(readRowType, null, nestedFieldEnabled) + .build(formatIdentifier, schema, dataSchema, readFields, false)); + } + private static int findBlobFieldIndex(RowType rowType) { for (int i = 0; i < rowType.getFieldCount(); i++) { if (isBlobFileField(rowType.getTypeAt(i))) { @@ -1040,8 +1112,7 @@ public static List splitFieldBunches( boolean rowIdPushDown) { List fieldsFiles = new ArrayList<>(); Map blobBunchMap = new HashMap<>(); - Map vectorStoreBunchMap = new TreeMap<>(); - long rowCount = -1; + Map vectorStoreBunchMap = new TreeMap<>(); Range rowRange = null; for (DataFileMeta file : needMergeFiles) { if (isBlobFile(file.fileName())) { @@ -1054,20 +1125,17 @@ public static List splitFieldBunches( .add(file); } else if (isVectorStoreFile(file.fileName())) { RowType rowType = fileToRowType.apply(file); - String fileFormat = DataFilePathFactory.formatIdentifier(file.fileName()); - VectorStoreBunchKey vectorStoreKey = - new VectorStoreBunchKey( - file.schemaId(), fileFormat, file.writeCols(), rowType); - final long expectedRowCount = rowCount; - vectorStoreBunchMap - .computeIfAbsent( - vectorStoreKey, - key -> new VectorFileBunch(expectedRowCount, rowIdPushDown)) - .add(file); + final Range expectedRowRange = rowRange; + for (String column : file.writeCols()) { + int fieldId = rowType.getField(column).id(); + vectorStoreBunchMap + .computeIfAbsent( + fieldId, key -> new VectorFileBunch(fieldId, expectedRowRange)) + .add(file); + } } else { // Normal file, just add it to the current merge split fieldsFiles.add(new DataBunch(file)); - rowCount = file.rowCount(); rowRange = file.nonNullRowIdRange(); } } @@ -1088,6 +1156,9 @@ private static long bunchFirstRowId(FieldBunch bunch) { if (bunch instanceof BlobFileBunch) { return ((BlobFileBunch) bunch).logicalRange().from; } + if (bunch instanceof VectorFileBunch) { + return ((VectorFileBunch) bunch).selectedFiles().get(0).range.from; + } return bunch.files().get(0).nonNullFirstRowId(); } @@ -1151,7 +1222,10 @@ public long rowCount() { if (expectedRowRange != null) { for (Range range : merged) { Preconditions.checkState( - range.from >= expectedRowRange.from && range.to <= expectedRowRange.to, + rowIdPushdown + ? range.hasIntersection(expectedRowRange) + : range.from >= expectedRowRange.from + && range.to <= expectedRowRange.to, "Blob file range %s should be within normal file range %s.", range, expectedRowRange); @@ -1188,80 +1262,49 @@ public List files() { static class VectorFileBunch implements FieldBunch { final List files; - final long expectedRowCount; - final boolean rowIdPushDown; - - long latestFistRowId = -1; - long expectedNextFirstRowId = -1; - long latestMaxSequenceNumber = -1; - long rowCount; + @Nullable final Range expectedRowRange; + final int fieldId; - VectorFileBunch(long expectedRowCount, boolean rowIdPushDown) { + VectorFileBunch(int fieldId, @Nullable Range expectedRowRange) { this.files = new ArrayList<>(); - this.expectedRowCount = expectedRowCount; - this.rowIdPushDown = rowIdPushDown; + this.expectedRowRange = expectedRowRange; + this.fieldId = fieldId; } void add(DataFileMeta file) { - if (!isVectorStoreFile(file.fileName())) { - throw new IllegalArgumentException( - "Only vector-store file can be added to this bunch."); - } - if (file.nonNullFirstRowId() == latestFistRowId) { - if (file.maxSequenceNumber() >= latestMaxSequenceNumber) { - throw new IllegalArgumentException( - "Vector file with same first row id should have decreasing sequence number."); - } - return; - } + checkArgument( + isVectorStoreFile(file.fileName()), + "Only vector-store file can be added to this bunch."); + files.add(file); + } - if (!files.isEmpty()) { - long firstRowId = file.nonNullFirstRowId(); - if (rowIdPushDown && firstRowId < expectedNextFirstRowId) { - if (file.maxSequenceNumber() > latestMaxSequenceNumber) { - DataFileMeta lastFile = files.remove(files.size() - 1); - rowCount -= lastFile.rowCount(); - } else { - return; - } - } else if (firstRowId < expectedNextFirstRowId) { - checkArgument( - file.maxSequenceNumber() < latestMaxSequenceNumber, - "Vector file with overlapping row id should have decreasing sequence number."); - return; - } else if (!rowIdPushDown && firstRowId > expectedNextFirstRowId) { - throw new IllegalArgumentException( - "Vector file first row id should be continuous, expect " - + expectedNextFirstRowId - + " but got " - + firstRowId); + List selectedFiles() { + List selected = new ArrayList<>(); + // A retained large vector file may be partly overwritten after its normal file was + // split. Preserve older prefixes and suffixes when these normal ranges merge again. + List newestFirst = new ArrayList<>(files); + newestFirst.sort(comparingLong(DataFileMeta::maxSequenceNumber).reversed()); + List covered = new ArrayList<>(); + for (DataFileMeta file : newestFirst) { + Range range = + expectedRowRange == null + ? file.nonNullRowIdRange() + : Range.intersection(file.nonNullRowIdRange(), expectedRowRange); + if (range == null) { + continue; } - - if (!files.isEmpty()) { - checkArgument( - file.schemaId() == files.get(0).schemaId(), - "All files in this bunch should have the same schema id."); - checkArgument( - file.writeCols().equals(files.get(0).writeCols()), - "All files in this bunch should have the same write columns."); + for (Range remaining : range.exclude(covered)) { + selected.add(new VectorFileRange(file, remaining)); } + covered.add(range); } - - files.add(file); - rowCount += file.rowCount(); - if (expectedRowCount > 0) { - checkArgument( - rowCount <= expectedRowCount, - "Vector files row count exceed the expect " + expectedRowCount); - } - latestMaxSequenceNumber = file.maxSequenceNumber(); - latestFistRowId = file.nonNullFirstRowId(); - expectedNextFirstRowId = latestFistRowId + file.rowCount(); + selected.sort(comparingLong(file -> file.range.from)); + return selected; } @Override public long rowCount() { - return rowCount; + return selectedFiles().stream().mapToLong(file -> file.range.count()).sum(); } @Override @@ -1270,11 +1313,63 @@ public List files() { } } + @VisibleForTesting + static class VectorFileRange { + final DataFileMeta file; + final Range range; + + private VectorFileRange(DataFileMeta file, Range range) { + this.file = file; + this.range = range; + } + } + public static List> mergeRangesAndSort(List files) { // group by row id range ToLongFunction maxSeqF = DataFileMeta::maxSequenceNumber; RangeHelper rangeHelper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange); - List> result = rangeHelper.mergeOverlappingRanges(files); + List normalFiles = new ArrayList<>(); + List dedicatedFiles = new ArrayList<>(); + for (DataFileMeta file : files) { + if (isBlobFile(file.fileName()) || isVectorStoreFile(file.fileName())) { + dedicatedFiles.add(file); + } else { + normalFiles.add(file); + } + } + List> result = rangeHelper.mergeOverlappingRanges(normalFiles); + TreeMap> normalGroups = new TreeMap<>(); + for (List group : result) { + checkArgument( + rangeHelper.areAllRangesSame(group), + "Data files %s should be all row id ranges same.", + group); + normalGroups.put(group.get(0).nonNullFirstRowId(), group); + } + List unanchoredFiles = new ArrayList<>(); + for (DataFileMeta file : dedicatedFiles) { + Range range = file.nonNullRowIdRange(); + Map.Entry> entry = normalGroups.floorEntry(range.from); + if (entry == null) { + entry = normalGroups.ceilingEntry(range.from); + } + boolean attached = false; + while (entry != null && entry.getKey() <= range.to) { + List group = entry.getValue(); + if (group.get(0).nonNullRowIdRange().hasIntersection(range)) { + // Retain the physical range: readers need its original first row id to locate + // the selected rows in dedicated files shared by multiple normal groups. + group.add(file); + attached = true; + } + entry = normalGroups.higherEntry(entry.getKey()); + } + if (!attached) { + unanchoredFiles.add(file); + } + } + result.addAll(rangeHelper.mergeOverlappingRanges(unanchoredFiles)); + result.sort(comparingLong(group -> group.get(0).nonNullFirstRowId())); // in group, sort by blob/vector-store file and max_seq for (List group : result) { @@ -1305,8 +1400,12 @@ public static List> mergeRangesAndSort(List fil .thenComparing(reverseOrder(comparingLong(maxSeqF)))); // vector-store files sort by first row id then by reversed max sequence number + long normalFirstRowId = + dataFiles.isEmpty() ? Long.MIN_VALUE : dataFiles.get(0).nonNullFirstRowId(); vectorStoreFiles.sort( - comparingLong(DataFileMeta::nonNullFirstRowId) + comparingLong( + (DataFileMeta file) -> + Math.max(normalFirstRowId, file.nonNullFirstRowId())) .thenComparing(reverseOrder(comparingLong(maxSeqF)))); // concat data files, blob files, vector-store files @@ -1318,97 +1417,4 @@ public static List> mergeRangesAndSort(List fil return result; } - - static final class VectorStoreBunchKey implements Comparable { - public final long schemaId; - public final String formatIdentifier; - public final List writeCols; - - public VectorStoreBunchKey( - long schemaId, - String formatIdentifier, - List writeCols, - RowType preferredColOrder) { - this.schemaId = schemaId; - this.formatIdentifier = checkNotNull(formatIdentifier, "formatIdentifier"); - this.writeCols = normalizeWriteCols(writeCols, preferredColOrder); - } - - @Override - public int compareTo(VectorStoreBunchKey o) { - int c = Long.compare(this.schemaId, o.schemaId); - if (c != 0) { - return c; - } - - c = this.formatIdentifier.compareTo(o.formatIdentifier); - if (c != 0) { - return c; - } - - int n = Math.min(this.writeCols.size(), o.writeCols.size()); - for (int i = 0; i < n; i++) { - c = this.writeCols.get(i).compareTo(o.writeCols.get(i)); - if (c != 0) { - return c; - } - } - return Integer.compare(this.writeCols.size(), o.writeCols.size()); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof VectorStoreBunchKey)) { - return false; - } - VectorStoreBunchKey that = (VectorStoreBunchKey) o; - return schemaId == that.schemaId - && formatIdentifier.equals(that.formatIdentifier) - && writeCols.equals(that.writeCols); - } - - @Override - public int hashCode() { - return Objects.hash(schemaId, formatIdentifier, writeCols); - } - - @Override - public String toString() { - return "VectorStoreBunchKey{schemaId=" - + schemaId - + ", format=" - + formatIdentifier - + ", writeCols=" - + writeCols - + "}"; - } - - private static List normalizeWriteCols(List writeCols, RowType rowType) { - if (writeCols == null || writeCols.isEmpty()) { - return Collections.emptyList(); - } - - Map colPosMap = new HashMap<>(); - List namesInRowType = rowType.getFieldNames(); - for (int i = 0; i < namesInRowType.size(); i++) { - colPosMap.putIfAbsent(namesInRowType.get(i), i); - } - - ArrayList sorted = new ArrayList<>(writeCols); - sorted.sort( - (a, b) -> { - int ia = colPosMap.getOrDefault(a, Integer.MAX_VALUE); - int ib = colPosMap.getOrDefault(b, Integer.MAX_VALUE); - if (ia != ib) { - return Integer.compare(ia, ib); - } - return a.compareTo(b); - }); - - return sorted; - } - } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java index 056c6b8dca86..6434b78cbb11 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java @@ -249,7 +249,7 @@ protected Optional checkTableSpecificConflicts( return exception; } - exception = checkRowIdRangeConflicts(commitKind, mergedEntries); + exception = checkRowIdRangeConflicts(commitKind, baseEntries, deltaEntries, mergedEntries); if (exception.isPresent()) { return exception; } @@ -264,7 +264,10 @@ protected Optional checkTableSpecificConflicts( } private Optional checkRowIdRangeConflicts( - CommitKind commitKind, Collection mergedEntries) { + CommitKind commitKind, + List baseEntries, + List deltaEntries, + Collection mergedEntries) { if (rowIdCheckFromSnapshot == null && commitKind != CommitKind.COMPACT) { return Optional.empty(); } @@ -291,7 +294,8 @@ private Optional checkRowIdRangeConflicts( entries.stream() .filter(file -> dedicatedStorageFile(file.fileName())) .collect(Collectors.toList()); - return checkDedicatedFileRowIdRangeConflicts(dataFiles, dedicatedFiles); + return checkDedicatedFileRowIdRangeConflicts( + commitKind, baseEntries, deltaEntries, dataFiles, dedicatedFiles); } private Optional checkDataFileRowIdRangeConflicts( @@ -310,18 +314,54 @@ private Optional checkDataFileRowIdRangeConflicts( } private Optional checkDedicatedFileRowIdRangeConflicts( - List dataFiles, List dedicatedFiles) { + CommitKind commitKind, + List baseEntries, + List deltaEntries, + List dataFiles, + List dedicatedFiles) { if (dedicatedFiles.isEmpty()) { return Optional.empty(); } RowRangeIndex dataFileRowRangeIndex = rowRangeIndex(dataFiles, false); + RowRangeIndex contiguousDataFileRowRangeIndex = rowRangeIndex(dataFiles, true); + RowRangeIndex baseDataFileRowRangeIndex = + rowRangeIndex( + baseEntries.stream() + .filter(file -> file.firstRowId() != null) + .filter(file -> !dedicatedStorageFile(file.fileName())) + .collect(Collectors.toList()), + false); + Set addedFiles = + deltaEntries.stream() + .filter(file -> file.kind() == FileKind.ADD) + .map(FileEntry::identifier) + .collect(Collectors.toSet()); for (SimpleFileEntry dedicatedFile : dedicatedFiles) { Range dedicatedRange = dedicatedFile.nonNullRowIdRange(); if (dataFileRowRangeIndex.contains(dedicatedRange)) { continue; } + if (!addedFiles.contains(dedicatedFile.identifier())) { + // Normal-file compaction can change boundaries without rewriting dedicated files. + // A row-range scan may contain only part of an existing dedicated file, so check + // that all of its previously visible normal-file coverage remains present. + List previouslyCoveredRanges = + baseDataFileRowRangeIndex.intersectedRanges( + dedicatedRange.from, dedicatedRange.to); + if (!previouslyCoveredRanges.isEmpty() + && previouslyCoveredRanges.stream() + .allMatch(contiguousDataFileRowRangeIndex::contains)) { + continue; + } + } else if (commitKind == CommitKind.COMPACT + && contiguousDataFileRowRangeIndex.contains(dedicatedRange)) { + // Dedicated compaction may merge files across the new normal-file boundaries. + // New DML files must still fit one range to reject stale MERGE INTO writers. + continue; + } + List intersectingRanges = dataFileRowRangeIndex.intersectedRanges(dedicatedRange.from, dedicatedRange.to); List intersectingDataFiles = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java index 88bf60f019c8..3adc53ffc1fa 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java @@ -40,6 +40,7 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.FunctionWithIOException; import org.apache.paimon.utils.InternalRowUtils; +import org.apache.paimon.utils.Range; import org.apache.paimon.utils.RangeHelper; import org.apache.paimon.utils.SerializationUtils; @@ -56,7 +57,9 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; import static org.apache.paimon.io.DataFilePathFactory.INDEX_PATH_SUFFIX; +import static org.apache.paimon.types.VectorType.isVectorStoreFile; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Input splits. Needed by most batch computation engines. */ @@ -188,13 +191,19 @@ private boolean dataEvolutionRowCountAvailable() { private long dataEvolutionMergedRowCount() { long sum = 0L; RangeHelper rangeHelper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange); - List> ranges = rangeHelper.mergeOverlappingRanges(dataFiles); - for (List group : ranges) { - long maxCount = 0; - for (DataFileMeta file : group) { - maxCount = Math.max(maxCount, file.rowCount()); + for (List group : rangeHelper.mergeOverlappingRanges(dataFiles)) { + List ranges = + group.stream() + .filter( + f -> + !isBlobFile(f.fileName()) + && !isVectorStoreFile(f.fileName())) + .map(DataFileMeta::nonNullRowIdRange) + .collect(Collectors.toList()); + if (ranges.isEmpty()) { + group.stream().map(DataFileMeta::nonNullRowIdRange).forEach(ranges::add); } - sum += maxCount; + sum += Range.sortAndMergeOverlap(ranges, false).stream().mapToLong(Range::count).sum(); } if (dataDeletionFiles != null) { for (DeletionFile deletionFile : dataDeletionFiles) { diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java index 097529d6a7b6..18d8fd385e12 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java @@ -127,6 +127,27 @@ public void testSplitLargeFilesUsesPhysicalSizeAndIncludesColumnUpdates() { } } + @Test + public void testSplitLargeFilesAndMergeSmallFilesKeepDedicatedFiles() { + List entries = + Arrays.asList( + makeEntryWithSize("large.parquet", 0L, 10L, 0, 201L), + makeEntryWithSize("small1.parquet", 10L, 10L, 0, 20L), + makeEntryWithSize("small2.parquet", 20L, 10L, 0, 20L), + makeBlobEntry("original.blob", 0L, 30L, 1000L), + makeVectorStoreEntry("original.vector.lance", 0L, 30L, 1000L)); + DataEvolutionCompactCoordinator.CompactPlanner planner = + new DataEvolutionCompactCoordinator.CompactPlanner( + false, false, true, 100L, 100L, 1L, 2L, schemaId -> null, null); + + List tasks = planner.compactPlan(entries); + + assertThat(tasks).hasSize(2); + assertThat(tasks.get(0).compactBefore()).containsExactly(entries.get(0).file()); + assertThat(tasks.get(1).compactBefore()) + .containsExactly(entries.get(1).file(), entries.get(2).file()); + } + @Test public void testCompactPlannerContiguousFiles() { // Multiple contiguous files should be grouped together diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java index a515e2c64803..d6231f79da4e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java @@ -241,7 +241,7 @@ public void testSplitHistoricalLargeFile(boolean updateColumn) throws Exception } @Test - public void testSplitAlignsMultipleBlobColumnsAndVectorFiles() throws Exception { + public void testSplitRetainsMultipleBlobColumnsAndVectorFiles() throws Exception { Schema schema = Schema.newBuilder() .column("id", DataTypes.INT()) @@ -275,6 +275,15 @@ public void testSplitAlignsMultipleBlobColumnsAndVectorFiles() throws Exception Collections.singletonList(SchemaChange.renameColumn("v", "renamed_v")), false); table = getTableDefault(); + List dedicatedFiles = + table.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .filter( + file -> + isBlobFile(file.fileName()) + || isVectorStoreFile(file.fileName())) + .collect(Collectors.toList()); + assertThat(dedicatedFiles).hasSize(3); Map options = new HashMap<>(); options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); @@ -285,7 +294,7 @@ public void testSplitAlignsMultipleBlobColumnsAndVectorFiles() throws Exception .plan(); assertThat(tasks).hasSize(1); DataEvolutionCompactTask task = tasks.get(0); - assertThat(task.compactBefore()).hasSize(4); + assertThat(task.compactBefore()).hasSize(1); try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { commit.commit(Collections.singletonList(task.doCompact(table, "split-dedicated"))); } @@ -302,17 +311,52 @@ public void testSplitAlignsMultipleBlobColumnsAndVectorFiles() throws Exception .allSatisfy( file -> assertThat( - normalRanges.stream() - .anyMatch( - range -> - range.from - <= file - .nonNullFirstRowId() - && range.to - >= file - .nonNullRowIdRange() - .to)) - .isTrue()); + isBlobFile(file.fileName()) + || isVectorStoreFile(file.fileName())) + .isFalse()); + assertThat( + table.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .filter( + file -> + isBlobFile(file.fileName()) + || isVectorStoreFile(file.fileName())) + .collect(Collectors.toList())) + .containsExactlyInAnyOrderElementsOf(dedicatedFiles); + + // Reading the new layout does not depend on the compaction option remaining enabled. + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "false"); + table = table.copy(options); + assertDedicatedValues(table); + + // Merging split normal files must also leave spanning dedicated files untouched. + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "128 mb"); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + table = table.copy(options); + tasks = + new DataEvolutionCompactCoordinator( + table, false, false, table.snapshotManager().latestSnapshot()) + .plan(); + assertThat(tasks).hasSize(1); + task = tasks.get(0); + assertThat(task.compactBefore()).hasSize(normalRanges.size()); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(Collections.singletonList(task.doCompact(table, "merge-split-normal"))); + } + assertThat(task.compactAfter()).hasSize(1); + assertThat( + table.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .filter( + file -> + isBlobFile(file.fileName()) + || isVectorStoreFile(file.fileName())) + .collect(Collectors.toList())) + .containsExactlyInAnyOrderElementsOf(dedicatedFiles); + assertDedicatedValues(table); + } + + private void assertDedicatedValues(FileStoreTable table) throws Exception { ReadBuilder readBuilder = table.newReadBuilder(); List ids = new ArrayList<>(); try (RecordReader reader = diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java index 1d2c9b654c48..f7ca2e207940 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java @@ -663,6 +663,77 @@ public void testIntersectsRowRanges() { assertThat(index.intersects(100, 200)).isFalse(); } + @Test + public void testDedicatedProjectionKeepsEveryNormalDeletionVectorAnchor() { + DataEvolutionFileStoreScan scan = pruningScan(true, "f1"); + ManifestEntry first = pruningEntry("first.parquet", "f0", 0, 5, 0); + ManifestEntry second = pruningEntry("second.parquet", "f0", 5, 5, 0); + ManifestEntry update = pruningEntry("update.parquet", "f0", 0, 5, 1); + ManifestEntry blob = pruningEntry("spanning.blob", "f1", 0, 10, 0); + + assertThat(scan.postFilterManifestEntries(Arrays.asList(first, second, update, blob))) + .containsExactlyInAnyOrder(first, second, blob); + } + + @Test + public void testMissingColumnProjectionKeepsEverySplitNormalRange() { + DataEvolutionFileStoreScan scan = pruningScan(false, "f2"); + ManifestEntry first = pruningEntry("first.parquet", "f0", 0, 5, 0); + ManifestEntry second = pruningEntry("second.parquet", "f0", 5, 5, 0); + ManifestEntry blob = pruningEntry("spanning.blob", "f1", 0, 10, 0); + + assertThat(scan.postFilterManifestEntries(Arrays.asList(first, second, blob))) + .containsExactlyInAnyOrder(first, second); + } + + @Test + public void testPartialColumnProjectionKeepsMissingSplitNormalRange() { + DataEvolutionFileStoreScan scan = pruningScan(false, "f2"); + ManifestEntry first = pruningEntry("first.parquet", "f0", 0, 5, 0); + ManifestEntry second = pruningEntry("second.parquet", "f0", 5, 5, 0); + ManifestEntry update = pruningEntry("update.parquet", "f2", 0, 5, 1); + ManifestEntry vector = pruningEntry("spanning.vector.json", "f1", 0, 10, 0); + + assertThat(scan.postFilterManifestEntries(Arrays.asList(first, second, update, vector))) + .containsExactlyInAnyOrder(update, second); + } + + private DataEvolutionFileStoreScan pruningScan(boolean deletionVectorsEnabled, String field) { + TableSchema schema = TableSchema.create(0L, createSchema("f0", "f1", "f2")); + DataEvolutionFileStoreScan scan = + new DataEvolutionFileStoreScan( + null, null, null, null, schema, null, null, deletionVectorsEnabled); + scan.withReadType(schema.logicalRowType().project(Collections.singletonList(field))); + return scan; + } + + private ManifestEntry pruningEntry( + String fileName, String column, long firstRowId, long rowCount, long sequence) { + return ManifestEntry.create( + FileKind.ADD, + createBinaryRow(0), + 0, + 0, + DataFileMeta.create( + fileName, + 100L, + rowCount, + BinaryRow.EMPTY_ROW, + BinaryRow.EMPTY_ROW, + SimpleStats.EMPTY_STATS, + SimpleStats.EMPTY_STATS, + sequence, + sequence, + 0L, + 0, + null, + null, + FileSource.APPEND, + null, + firstRowId, + Collections.singletonList(column))); + } + private Schema createSchema(String... fieldNames) { Schema.Builder builder = Schema.newBuilder(); for (int i = 0; i < fieldNames.length; i++) { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java index 748dbe7bdfd6..ce8d7aa6bcc6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java @@ -51,7 +51,7 @@ public class DataEvolutionReadTest { @BeforeEach public void setUp() { - vectorBunch = new VectorFileBunch(Long.MAX_VALUE, false); + vectorBunch = new VectorFileBunch(0, null); } @Test @@ -94,81 +94,69 @@ public void testAddNonVectorFileThrowsException() { } @Test - public void testAddVectorFileWithSameFirstRowId() { - DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); - DataFileMeta vectorEntry2 = createVectorFile("vector2", 0, 50, 2); + public void testNewVectorVersionPreservesOldTail() { + DataFileMeta oldFile = createVectorFile("old", 0, 100, 1); + DataFileMeta newFile = createVectorFile("new", 0, 50, 2); + vectorBunch.add(oldFile); + vectorBunch.add(newFile); - vectorBunch.add(vectorEntry1); - // Adding file with same firstRowId but higher sequence number should throw exception - assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Vector file with same first row id should have decreasing sequence number."); + assertVectorSelection(vectorBunch, newFile, new Range(0, 49), oldFile, new Range(50, 99)); } @Test - public void testAddVectorFileWithSameFirstRowIdAndLowerSequenceNumber() { - DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 2); - DataFileMeta vectorEntry2 = createVectorFile("vector2", 0, 50, 1); - - vectorBunch.add(vectorEntry1); - // Adding file with same firstRowId and lower sequence number should be ignored - vectorBunch.add(vectorEntry2); + public void testNewVectorVersionReplacesCoveredOldRows() { + DataFileMeta newFile = createVectorFile("new", 0, 100, 2); + DataFileMeta oldFile = createVectorFile("old", 0, 50, 1); + vectorBunch.add(newFile); + vectorBunch.add(oldFile); - assertThat(vectorBunch.files).hasSize(1); - assertThat(vectorBunch.files.get(0)).isEqualTo(vectorEntry1); + assertThat(vectorBunch.rowCount()).isEqualTo(100); + assertThat(vectorBunch.selectedFiles()).hasSize(1); + assertThat(vectorBunch.selectedFiles().get(0).file).isEqualTo(newFile); + assertThat(vectorBunch.selectedFiles().get(0).range).isEqualTo(new Range(0, 99)); } @Test - public void testAddVectorFileWithOverlappingRowId() { - DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 2); - DataFileMeta vectorEntry2 = createVectorFile("vector2", 50, 150, 1); + public void testOlderOverlappingVectorFileSuppliesUncoveredTail() { + DataFileMeta newFile = createVectorFile("new", 0, 100, 2); + DataFileMeta oldFile = createVectorFile("old", 50, 150, 1); + vectorBunch.add(newFile); + vectorBunch.add(oldFile); - vectorBunch.add(vectorEntry1); - // Adding file with overlapping row id and lower sequence number should be ignored - vectorBunch.add(vectorEntry2); - - assertThat(vectorBunch.files).hasSize(1); - assertThat(vectorBunch.files.get(0)).isEqualTo(vectorEntry1); + assertVectorSelection(vectorBunch, newFile, new Range(0, 99), oldFile, new Range(100, 199)); } @Test - public void testAddVectorFileWithOverlappingRowIdAndHigherSequenceNumber() { - DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); - DataFileMeta vectorEntry2 = createVectorFile("vector2", 50, 150, 2); + public void testNewOverlappingVectorFilePreservesOldPrefix() { + DataFileMeta oldFile = createVectorFile("old", 0, 100, 1); + DataFileMeta newFile = createVectorFile("new", 50, 150, 2); + vectorBunch.add(oldFile); + vectorBunch.add(newFile); - vectorBunch.add(vectorEntry1); - // Adding file with overlapping row id and higher sequence number should throw exception - assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Vector file with overlapping row id should have decreasing sequence number."); + assertVectorSelection(vectorBunch, oldFile, new Range(0, 49), newFile, new Range(50, 199)); } @Test - public void testAddVectorFileWithNonContinuousRowId() { - DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); - DataFileMeta vectorEntry2 = createVectorFile("vector2", 200, 300, 1); + public void testVectorSelectionRetainsGaps() { + DataFileMeta first = createVectorFile("first", 0, 100, 1); + DataFileMeta second = createVectorFile("second", 200, 300, 1); + vectorBunch.add(first); + vectorBunch.add(second); - vectorBunch.add(vectorEntry1); - // Adding file with non-continuous row id should throw exception - assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Vector file first row id should be continuous, expect 100 but got 200"); + assertVectorSelection(vectorBunch, first, new Range(0, 99), second, new Range(200, 499)); } @Test - public void testAddVectorFileWithDifferentWriteCols() { - DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); - DataFileMeta vectorEntry2 = - createVectorFileWithCols("vector2", 100, 200, 1, Arrays.asList("different_col")); - - vectorBunch.add(vectorEntry1); - // Adding file with different write columns should throw exception - assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("All files in this bunch should have the same write columns."); + public void testVectorSelectionRetainsPhysicalColumnNames() { + DataFileMeta oldFile = createVectorFile("old", 0, 100, 1); + DataFileMeta renamedFile = + createVectorFileWithCols( + "renamed", 100, 200, 2, Collections.singletonList("renamed_vector")); + vectorBunch.add(oldFile); + vectorBunch.add(renamedFile); + + assertVectorSelection( + vectorBunch, oldFile, new Range(0, 99), renamedFile, new Range(100, 299)); } @Test @@ -453,14 +441,13 @@ private DataFileMeta createFile( } @Test - void testAddVectorFilesWithDifferentSchemaId() { - DataFileMeta vectorEntry1 = createVectorFileWithSchema("vector1", 0, 100, 1, 0L); - DataFileMeta vectorEntry2 = createVectorFileWithSchema("vector2", 100, 200, 1, 1L); + void testVectorSelectionAcrossSchemas() { + DataFileMeta oldFile = createVectorFileWithSchema("old", 0, 100, 1, 0L); + DataFileMeta newFile = createVectorFileWithSchema("new", 100, 200, 2, 1L); + vectorBunch.add(oldFile); + vectorBunch.add(newFile); - vectorBunch.add(vectorEntry1); - assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("All files in this bunch should have the same schema id."); + assertVectorSelection(vectorBunch, oldFile, new Range(0, 99), newFile, new Range(100, 299)); } @Test @@ -498,25 +485,30 @@ void testBlobBunchRejectsRangeOutsideNormalFile() { } @Test - public void testRowIdPushDown() { - VectorFileBunch vectorBunch = new VectorFileBunch(Long.MAX_VALUE, true); - DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); - DataFileMeta vectorEntry2 = createVectorFile("vector2", 200, 300, 1); - vectorBunch.add(vectorEntry1); - VectorFileBunch finalVectorBunch = vectorBunch; - DataFileMeta finalVectorEntry = vectorEntry2; - assertThatCode(() -> finalVectorBunch.add(finalVectorEntry)).doesNotThrowAnyException(); - - vectorBunch = new VectorFileBunch(Long.MAX_VALUE, true); - vectorEntry1 = createVectorFile("vector1", 0, 100, 1); - vectorEntry2 = createVectorFile("vector2", 50, 200, 2); - vectorBunch.add(vectorEntry1); - vectorBunch.add(vectorEntry2); - assertThat(vectorBunch.files).containsExactlyInAnyOrder(vectorEntry2); + public void testVectorSelectionClipsToNormalRange() { + VectorFileBunch clipped = new VectorFileBunch(0, new Range(50, 149)); + DataFileMeta oldFile = createVectorFile("old", 0, 200, 1); + DataFileMeta newFile = createVectorFile("new", 100, 100, 2); + clipped.add(oldFile); + clipped.add(newFile); + + assertVectorSelection(clipped, oldFile, new Range(50, 99), newFile, new Range(100, 149)); + assertThat(oldFile.nonNullRowIdRange()).isEqualTo(new Range(0, 199)); + assertThat(newFile.nonNullRowIdRange()).isEqualTo(new Range(100, 199)); + } - VectorFileBunch finalVectorBunch2 = vectorBunch; - DataFileMeta vectorEntry3 = createVectorFile("vector2", 250, 100, 2); - assertThatCode(() -> finalVectorBunch2.add(vectorEntry3)).doesNotThrowAnyException(); + private static void assertVectorSelection( + VectorFileBunch bunch, + DataFileMeta first, + Range firstRange, + DataFileMeta second, + Range secondRange) { + assertThat(bunch.selectedFiles()).hasSize(2); + assertThat(bunch.selectedFiles().get(0).file).isEqualTo(first); + assertThat(bunch.selectedFiles().get(0).range).isEqualTo(firstRange); + assertThat(bunch.selectedFiles().get(1).file).isEqualTo(second); + assertThat(bunch.selectedFiles().get(1).range).isEqualTo(secondRange); + assertThat(bunch.rowCount()).isEqualTo(firstRange.count() + secondRange.count()); } /** Creates a normal (non-blob) file for testing. */ diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java index cd60d981643a..b12b6058146f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java @@ -20,6 +20,9 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryString; +import org.apache.paimon.data.BinaryVector; +import org.apache.paimon.data.BlobData; +import org.apache.paimon.data.BlobPlaceholder; import org.apache.paimon.data.GenericArray; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; @@ -35,9 +38,15 @@ import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; +import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.source.DataSplit; +import org.apache.paimon.table.source.Split; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; @@ -45,12 +54,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.IntFunction; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -163,6 +177,271 @@ public void testSplitWithMultipleVectorStoreFilesPerGroup() { assertEquals(Arrays.asList(file4, file5, file6), result.get(1)); } + @Test + public void testSplitWithDedicatedFilesSpanningNormalGroups() { + DataFileMeta first = createFile("first.parquet", 0, 4, 3); + DataFileMeta middle = createFile("middle.parquet", 4, 4, 3); + DataFileMeta last = createFile("last.parquet", 8, 4, 3); + DataFileMeta blob = createFile("blob.blob", 0, 12, 1); + DataFileMeta vector = createFile("vector.vector.json", 0, 12, 1); + DataFileMeta vectorUpdate = createFile("update.vector.json", 4, 4, 2); + + List> groups = + DataEvolutionSplitRead.mergeRangesAndSort( + Arrays.asList(blob, vector, middle, last, first, vectorUpdate)); + + assertEquals( + Arrays.asList( + Arrays.asList(first, blob, vector), + Arrays.asList(middle, blob, vectorUpdate, vector), + Arrays.asList(last, blob, vector)), + groups); + // Associations must retain physical file offsets for readers of the second and third group. + assertEquals(new Range(0, 11), groups.get(2).get(1).nonNullRowIdRange()); + } + + @ParameterizedTest + @CsvSource({ + "false,4,false", + "true,4,false", + "false,12,false", + "true,12,false", + "false,4,true", + "true,4,true", + "false,12,true", + "true,12,true" + }) + public void testReadSpanningDedicatedFiles( + boolean indexed, int normalRowCount, boolean renameBeforeUpdate) throws Exception { + LocalFileIO fileIO = new LocalFileIO(); + Path tablePath = new Path(tempDir.resolve("spanning").toUri()); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); + TableSchema schema = + schemaManager.createTable( + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("blob", DataTypes.BLOB()) + .column("vector", DataTypes.VECTOR(2, DataTypes.FLOAT())) + .column("vector2", DataTypes.VECTOR(2, DataTypes.FLOAT())) + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .build()); + FileStoreTable table = FileStoreTableFactory.create(fileIO, tablePath, schema); + FileStorePathFactory pathFactory = table.store().pathFactory(); + Path bucketPath = pathFactory.bucketPath(EMPTY_ROW, 0); + fileIO.mkdirs(bucketPath); + RowType rowType = schema.logicalRowType(); + List files = new ArrayList<>(); + for (int from = 0; from < 12; from += normalRowCount) { + files.add( + writeProjectedFile( + fileIO, + bucketPath, + "normal-" + from + ".parquet", + "parquet", + rowType.project("id"), + from, + normalRowCount, + 3, + GenericRow::of)); + } + files.add( + writeProjectedFile( + fileIO, + bucketPath, + "base.blob", + "blob", + rowType.project("blob"), + 0, + 12, + 1, + i -> GenericRow.of(new BlobData(new byte[] {(byte) i})))); + files.add( + writeProjectedFile( + fileIO, + bucketPath, + "update.blob", + "blob", + rowType.project("blob"), + 2, + 8, + 2, + i -> + GenericRow.of( + i % 2 == 0 + ? BlobPlaceholder.INSTANCE + : new BlobData(new byte[] {(byte) (i + 20)})))); + files.add( + writeProjectedFile( + fileIO, + bucketPath, + "base.vector.json", + "json", + rowType.project("vector", "vector2"), + 0, + 12, + 1, + i -> + GenericRow.of( + BinaryVector.fromPrimitiveArray(new float[] {i, i + 1}), + BinaryVector.fromPrimitiveArray( + new float[] {i + 100, i + 101})))); + if (renameBeforeUpdate) { + schema = + schemaManager.commitChanges( + SchemaChange.renameColumn("vector", "renamed_vector")); + } + files.add( + writeProjectedFile( + fileIO, + bucketPath, + "update.vector.json", + "json", + schema.logicalRowType() + .project(renameBeforeUpdate ? "renamed_vector" : "vector"), + 4, + 4, + 2, + i -> + GenericRow.of( + BinaryVector.fromPrimitiveArray( + new float[] {i + 20, i + 21})), + schema.id())); + if (!renameBeforeUpdate) { + schema = + schemaManager.commitChanges( + SchemaChange.renameColumn("vector", "renamed_vector")); + } + RowType allReadType = SpecialFields.rowTypeWithRowId(schema.logicalRowType()); + DataSplit dataSplit = + DataSplit.builder() + .withPartition(EMPTY_ROW) + .withBucket(0) + .withBucketPath(bucketPath.toString()) + .withDataFiles(files) + .rawConvertible(false) + .build(); + for (boolean vectorOnly : new boolean[] {false, true}) { + RowType readType = + vectorOnly + ? allReadType.project( + "renamed_vector", "vector2", SpecialFields.ROW_ID.name()) + : allReadType; + DataSplit projectedSplit = + vectorOnly + ? dataSplit + .filterDataFile( + file -> + org.apache.paimon.types.VectorType + .isVectorStoreFile(file.fileName())) + .get() + : dataSplit; + Split split = + indexed + ? new IndexedSplit( + projectedSplit, + Arrays.asList( + new Range(1, 2), new Range(5, 5), new Range(8, 9)), + null) + : projectedSplit; + DataEvolutionSplitRead splitRead = + new DataEvolutionSplitRead( + fileIO, + schemaManager, + schema, + readType, + table.coreOptions(), + pathFactory); + List ids = new ArrayList<>(); + try (RecordReader reader = splitRead.createReader(split)) { + reader.forEachRemaining( + row -> { + int id = (int) row.getLong(vectorOnly ? 2 : 4); + ids.add(id); + if (!vectorOnly) { + assertEquals(id, row.getInt(0)); + int expectedBlob = id >= 2 && id < 10 && id % 2 != 0 ? id + 20 : id; + assertEquals((byte) expectedBlob, row.getBlob(1).toData()[0]); + } + int expectedVector = id >= 4 && id < 8 ? id + 20 : id; + org.assertj.core.api.Assertions.assertThat( + row.getVector(vectorOnly ? 0 : 2).toFloatArray()) + .containsExactly(expectedVector, expectedVector + 1); + org.assertj.core.api.Assertions.assertThat( + row.getVector(vectorOnly ? 1 : 3).toFloatArray()) + .containsExactly(id + 100, id + 101); + }); + } + assertEquals( + indexed + ? Arrays.asList(1, 2, 5, 8, 9) + : IntStream.range(0, 12).boxed().collect(Collectors.toList()), + ids); + } + } + + private static DataFileMeta writeProjectedFile( + LocalFileIO fileIO, + Path bucketPath, + String name, + String formatIdentifier, + RowType writeType, + int firstRowId, + int rowCount, + int sequence, + IntFunction rowFactory) + throws IOException { + return writeProjectedFile( + fileIO, + bucketPath, + name, + formatIdentifier, + writeType, + firstRowId, + rowCount, + sequence, + rowFactory, + 0); + } + + private static DataFileMeta writeProjectedFile( + LocalFileIO fileIO, + Path bucketPath, + String name, + String formatIdentifier, + RowType writeType, + int firstRowId, + int rowCount, + int sequence, + IntFunction rowFactory, + long schemaId) + throws IOException { + Path filePath = new Path(bucketPath, name); + FileFormat format = FileFormat.fromIdentifier(formatIdentifier, new Options()); + try (PositionOutputStream output = fileIO.newOutputStream(filePath, false)) { + FormatWriter writer = format.createWriterFactory(writeType).create(output, "none"); + for (int i = firstRowId; i < firstRowId + rowCount; i++) { + writer.addElement(rowFactory.apply(i)); + } + writer.close(); + } + return DataFileMeta.forAppend( + name, + fileIO.getFileStatus(filePath).getLen(), + rowCount, + SimpleStats.EMPTY_STATS, + sequence, + sequence, + schemaId, + Collections.emptyList(), + null, + FileSource.APPEND, + null, + null, + (long) firstRowId, + writeType.getFieldNames()); + } + @Test public void testRowSidecarFileName() { DataFileMeta file = diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 787d0761ea3e..8cba65317de3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -38,6 +38,8 @@ import org.apache.paimon.utils.SnapshotManager; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nullable; @@ -1361,6 +1363,7 @@ void testCheckRowIdRangeConflictsUsesRetryableExceptionForDataFiles() { @Test void testCheckRowIdRangeConflictsReportsDedicatedFileSpanningDataFiles() { DataEvolutionConflictDetection detection = createConflictDetection(); + detection.setRowIdCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -1371,7 +1374,7 @@ void testCheckRowIdRangeConflictsReportsDedicatedFileSpanningDataFiles() { Collections.singletonList(createFileEntryWithRowId("p1.blob", ADD, 0L, 4L)), Collections.emptyList(), null, - Snapshot.CommitKind.COMPACT); + Snapshot.CommitKind.APPEND); assertThat(exception).isPresent(); assertThat(exception.get()) @@ -1383,6 +1386,130 @@ void testCheckRowIdRangeConflictsReportsDedicatedFileSpanningDataFiles() { .hasMessageContaining("f2"); } + @ParameterizedTest + @ValueSource(strings = {"retained.blob", "retained.vector.json"}) + void testNormalSplitRetainsDedicatedFileAcrossAdjacentRanges(String dedicatedFile) { + DataEvolutionConflictDetection detection = createConflictDetection(); + + assertThat( + detection.checkConflicts( + snapshot(1), + Arrays.asList( + createFileEntryWithRowId("normal", ADD, 0L, 4L), + createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), + Arrays.asList( + createFileEntryWithRowId("normal", DELETE, 0L, 4L), + createFileEntryWithRowId("split-1", ADD, 0L, 2L), + createFileEntryWithRowId("split-2", ADD, 2L, 2L)), + Collections.emptyList(), + null, + Snapshot.CommitKind.COMPACT)) + .isEmpty(); + } + + @ParameterizedTest + @ValueSource(strings = {"retained.blob", "retained.vector.json"}) + void testNormalCompactionRetainsDedicatedFileOutsideScannedRange(String dedicatedFile) { + DataEvolutionConflictDetection detection = createConflictDetection(); + + // A later compaction scans only one of the normal ranges inside the retained file. + assertThat( + detection.checkConflicts( + snapshot(1), + Arrays.asList( + createFileEntryWithRowId("normal", ADD, 2L, 2L), + createFileEntryWithRowId(dedicatedFile, ADD, 0L, 6L)), + Arrays.asList( + createFileEntryWithRowId("normal", DELETE, 2L, 2L), + createFileEntryWithRowId("split-1", ADD, 2L, 1L), + createFileEntryWithRowId("split-2", ADD, 3L, 1L)), + Collections.emptyList(), + null, + Snapshot.CommitKind.COMPACT)) + .isEmpty(); + } + + @ParameterizedTest + @ValueSource(strings = {"retained.blob", "retained.vector.json"}) + void testNormalSplitRejectsGapUnderRetainedDedicatedFile(String dedicatedFile) { + DataEvolutionConflictDetection detection = createConflictDetection(); + + Optional exception = + detection.checkConflicts( + snapshot(1), + Arrays.asList( + createFileEntryWithRowId("normal", ADD, 0L, 4L), + createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), + Arrays.asList( + createFileEntryWithRowId("normal", DELETE, 0L, 4L), + createFileEntryWithRowId("split-1", ADD, 0L, 1L), + createFileEntryWithRowId("split-2", ADD, 2L, 2L)), + Collections.emptyList(), + null, + Snapshot.CommitKind.COMPACT); + + assertThat(exception).isPresent(); + assertThat(exception.get()).hasMessageContaining("dedicated file"); + } + + @ParameterizedTest + @ValueSource(strings = {"retained.blob", "retained.vector.json"}) + void testNormalCompactionRejectsOrphanedDedicatedFile(String dedicatedFile) { + DataEvolutionConflictDetection detection = createConflictDetection(); + + Optional exception = + detection.checkConflicts( + snapshot(1), + Arrays.asList( + createFileEntryWithRowId("normal", ADD, 0L, 4L), + createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), + Collections.singletonList( + createFileEntryWithRowId("normal", DELETE, 0L, 4L)), + Collections.emptyList(), + null, + Snapshot.CommitKind.COMPACT); + + assertThat(exception).isPresent(); + assertThat(exception.get()).hasMessageContaining("dedicated file"); + } + + @ParameterizedTest + @ValueSource(strings = {"compacted.blob", "compacted.vector.json"}) + void testDedicatedCompactionMaySpanSplitNormalFiles(String dedicatedFile) { + DataEvolutionConflictDetection detection = createConflictDetection(); + + assertThat( + detection.checkConflicts( + snapshot(1), + Arrays.asList( + createFileEntryWithRowId("split-1", ADD, 0L, 2L), + createFileEntryWithRowId("split-2", ADD, 2L, 2L)), + Collections.singletonList( + createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), + Collections.emptyList(), + null, + Snapshot.CommitKind.COMPACT)) + .isEmpty(); + } + + @ParameterizedTest + @ValueSource(strings = {"merge-normal", "merge.blob", "merge.vector.json"}) + void testMergeRejectsStaleFileAcrossSplitNormalRanges(String mergeFile) { + DataEvolutionConflictDetection detection = createConflictDetection(); + + Optional exception = + detection.checkRowIdExistence( + Arrays.asList( + createFileEntryWithRowId("split-1", ADD, 0L, 2L), + createFileEntryWithRowId("split-2", ADD, 2L, 2L)), + Collections.singletonList(createFileEntryWithRowId(mergeFile, ADD, 0L, 4L)), + 4L, + Snapshot.CommitKind.APPEND); + + assertThat(exception).isPresent(); + assertThat(exception.get()).isInstanceOf(RowIdExistenceConflictException.class); + } + @Test void testCheckRowIdRangeConflictsRejectsOverlappingNormalFiles() { DataEvolutionConflictDetection detection = createConflictDetection(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java index 9853a5bf32fa..a5ad4f1a9ffc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java @@ -628,7 +628,8 @@ public void testSplitLargeFilePreservesDeletionVectorsAndBlob(boolean bitmap64) .collect(Collectors.toList()); assertThat(normalFiles.size()).isGreaterThan(1); assertThat(normalFiles.stream().mapToLong(DataFileMeta::rowCount).sum()).isEqualTo(2500); - assertThat(files).doesNotContainAnyElementsOf(blobs); + assertThat(files.stream().filter(file -> isBlobFile(file.fileName()))) + .containsExactlyInAnyOrderElementsOf(blobs); assertThat(readRows(table.newReadBuilder())).containsExactlyElementsOf(expected); List expectedAnchors = new ArrayList<>(); for (DataFileMeta file : normalFiles) { @@ -648,6 +649,52 @@ public void testSplitLargeFilePreservesDeletionVectorsAndBlob(boolean bitmap64) readBuilder.newRead().createReader(readBuilder.newScan().plan())) { reader.forEachRemaining(row -> assertThat(row.getLong(4)).isEqualTo(row.getInt(0))); } + ReadBuilder blobRead = table.newReadBuilder().withReadType(table.rowType().project("f3")); + List actualBlobs = new ArrayList<>(); + try (RecordReader reader = + blobRead.newRead().createReader(blobRead.newScan().plan())) { + reader.forEachRemaining(row -> actualBlobs.add(row.getBlob(0).toData()[0])); + } + List expectedBlobs = new ArrayList<>(); + for (int rowId = 0; rowId < 2500; rowId++) { + if (rowId != 0 && rowId != 999 && rowId != 2000 && rowId != 2499) { + expectedBlobs.add((byte) rowId); + } + } + assertThat(actualBlobs).containsExactlyElementsOf(expectedBlobs); + } + + @Test + public void testNormalCompactRetainsSpanningBlobFile() throws Exception { + createTableDefault(); + FileStoreTable table = getTableDefault(); + writeBaseRows(table); + writeBlobRange(table, 5L, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114); + List blobs = + currentDataFiles(table, BinaryRow.EMPTY_ROW).stream() + .filter(file -> isBlobFile(file.fileName())) + .collect(Collectors.toList()); + Map options = new HashMap<>(); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + compactDataEvolutionTable(table.copy(options), false); + + assertThat( + currentDataFiles(table, BinaryRow.EMPTY_ROW).stream() + .filter(file -> isBlobFile(file.fileName()))) + .containsExactlyInAnyOrderElementsOf(blobs); + List expectedRows = new ArrayList<>(); + for (int rowId = 0; rowId < 15; rowId++) { + expectedRows.add( + rowId + + "|name-" + + rowId + + "|base-" + + rowId + + "|" + + (rowId < 5 ? rowId : rowId + 100)); + } + assertThat(readRows(table.newReadBuilder())).containsExactlyElementsOf(expectedRows); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java index e3f5403c68be..6519d1ed18d7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java @@ -93,6 +93,59 @@ public void testSplitMergedRowCount() { assertThat(split.mergedRowCount()).hasValue(5700L); } + @Test + public void testMergedRowCountWithSpanningDedicatedFiles() { + List files = + Arrays.asList( + newRowTrackedDataFile("first.parquet", 0, 5), + newRowTrackedDataFile("update.parquet", 0, 5), + newRowTrackedDataFile("second.parquet", 5, 5), + newRowTrackedDataFile("first.blob", 0, 7), + newRowTrackedDataFile("second.blob", 7, 3)); + DataSplit split = newDataSplit(false, files, null); + assertThat(split.mergedRowCount()).hasValue(10L); + + split = + newDataSplit( + false, + files, + Arrays.asList( + new DeletionFile("dv", 0, 1, 1L), + null, + new DeletionFile("dv", 1, 1, 2L), + null, + null)); + assertThat(split.mergedRowCount()).hasValue(7L); + + // A row-range scan may retain a dedicated file extending outside its normal file. + split = newDataSplit(false, Arrays.asList(files.get(2), files.get(3), files.get(4)), null); + assertThat(split.mergedRowCount()).hasValue(5L); + + // Projection can prune the normal file from another component packed in this split. + List projectedFiles = new ArrayList<>(files); + projectedFiles.add(newRowTrackedDataFile("projected.blob", 10, 10)); + assertThat(newDataSplit(false, projectedFiles, null).mergedRowCount()).hasValue(20L); + } + + private DataFileMeta newRowTrackedDataFile(String name, long firstRowId, long rowCount) { + return DataFileMeta.forAppend( + name, + 1024, + rowCount, + SimpleStats.EMPTY_STATS, + 0L, + rowCount - 1, + 1, + Collections.emptyList(), + null, + null, + null, + null, + null, + null) + .assignFirstRowId(firstRowId); + } + @Test public void testDeletionFilesSerialize() throws Exception { List dataFiles = diff --git a/paimon-python/pypaimon/read/reader/field_bunch.py b/paimon-python/pypaimon/read/reader/field_bunch.py index 65ab1af046ed..bbc60efe6e10 100644 --- a/paimon-python/pypaimon/read/reader/field_bunch.py +++ b/paimon-python/pypaimon/read/reader/field_bunch.py @@ -171,8 +171,9 @@ def finish(self) -> None: physical_row_count = sum(row_range.count() for row_range in merged) if self.expected_row_range is not None: for row_range in merged: - if (row_range.from_ < self.expected_row_range.from_ - or row_range.to > self.expected_row_range.to): + if (not self.row_id_push_down + and (row_range.from_ < self.expected_row_range.from_ + or row_range.to > self.expected_row_range.to)): raise ValueError( f"Blob file range {row_range} should be within normal " f"file range {self.expected_row_range}." @@ -236,6 +237,33 @@ def _file_type_label(self) -> str: class VectorBunch(_SpecialFieldBunch): """Files for partial field (vector files).""" + def __init__(self, expected_row_count: int, row_id_push_down: bool = False, + expected_row_range: Optional[Range] = None, field_id: Optional[int] = None): + super().__init__(expected_row_count, row_id_push_down) + self.expected_row_range = expected_row_range + self.field_id = field_id + + def add(self, file: DataFileMeta) -> None: + if not self._is_special_file(file.file_name): + raise ValueError("Only vector file can be added to a vector bunch.") + self._files.append(file) + + def segments(self): + """Select the newest physical file for each part of the normal range.""" + covered = [] + segments = [] + for file in sorted(self._files, + key=lambda f: (-f.max_sequence_number, f.file_name)): + selected = ([file.row_id_range()] if self.expected_row_range is None + else Range.and_([file.row_id_range()], [self.expected_row_range])) + for row_range in selected: + segments.extend((visible, file) for visible in row_range.exclude(covered)) + covered.append(row_range) + return sorted(segments, key=lambda segment: segment[0].from_) + + def row_count(self) -> int: + return sum(row_range.count() for row_range, _ in self.segments()) + def _is_special_file(self, file_name: str) -> bool: return DataFileMeta.is_vector_file(file_name) diff --git a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py b/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py index 89216322cfe1..8da60a6bdc4e 100644 --- a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py +++ b/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py @@ -30,9 +30,8 @@ from pypaimon.read.split import DataSplit, Split from pypaimon.table.row.generic_row import GenericRow from pypaimon.table.source.deletion_file import DeletionFile -from pypaimon.utils.data_evolution_utils import retrieve_anchor_file +from pypaimon.utils.data_evolution_utils import retrieve_anchor_file, split_normal_file_groups from pypaimon.utils.range import Range -from pypaimon.utils.range_helper import RangeHelper def _null_safe_partition_key(partition_values) -> tuple: @@ -577,6 +576,9 @@ def _chunk_to_split(self, chunk: _Chunk) -> Split: row_ranges.append(seg.row_range) row_ranges.sort(key=lambda r: r.from_) + # A retained dedicated file can belong to several normal ranges in one chunk. + all_files = list({file.file_name: file for file in all_files}.values()) + data_deletion_files = self._get_deletion_files_for_split( all_files, chunk.partition, @@ -602,23 +604,20 @@ def _chunk_to_split(self, chunk: _Chunk) -> Split: def _split_by_row_id_with_range( files: List[DataFileMeta], ) -> List[Tuple[Range, List[DataFileMeta]]]: - """Group files by overlapping row_id range, returning (range, files) - pairs sorted by ``range.from_``. - - Mirrors :meth:`DataEvolutionSplitGenerator._split_by_row_id` but - also returns the merged row_id range per group, which the chunk - slicer needs to drive row-count accumulation. - """ + """Group by normal ranges so each chunk uses the correct deletion-vector anchor.""" for f in files: if f.row_id_range() is None: raise ValueError( "chunk_shuffle for data evolution tables requires row tracking; " f"file {f.file_name} is missing first_row_id" ) - groups = RangeHelper(lambda f: f.row_id_range()).merge_overlapping_ranges(files) + groups = split_normal_file_groups(files) result = [] for group in groups: - ranges = [f.row_id_range() for f in group] + normal_files = [f for f in group + if not DataFileMeta.is_blob_file(f.file_name) + and not DataFileMeta.is_vector_file(f.file_name)] + ranges = [f.row_id_range() for f in normal_files or group] merged = Range(min(r.from_ for r in ranges), max(r.to for r in ranges)) result.append((merged, group)) return sorted(result, key=lambda kv: kv[0].from_) diff --git a/paimon-python/pypaimon/read/split_read.py b/paimon-python/pypaimon/read/split_read.py index f00976f1f0fc..feca4f649c50 100644 --- a/paimon-python/pypaimon/read/split_read.py +++ b/paimon-python/pypaimon/read/split_read.py @@ -17,6 +17,7 @@ import os from abc import ABC, abstractmethod +from copy import copy from functools import partial from typing import Callable, Dict, List, Optional, Tuple @@ -80,7 +81,7 @@ from pypaimon.schema.data_types import DataField, PyarrowFieldParser from pypaimon.table.special_fields import SpecialFields from pypaimon.globalindex.indexed_split import IndexedSplit -from pypaimon.utils.data_evolution_utils import retrieve_anchor_file +from pypaimon.utils.data_evolution_utils import retrieve_anchor_file, split_normal_file_groups KEY_PREFIX = "_KEY_" KEY_FIELD_ID_START = 1000000 @@ -1157,16 +1158,20 @@ def _create_raw_reader(self) -> RecordReader: split_by_row_id = self._split_by_row_id(files) for need_merge_files in split_by_row_id: + group_read = self._read_for_normal_range(need_merge_files) + if group_read.row_ranges == []: + continue deletion_vector = self._read_deletion_vector(need_merge_files) if len(need_merge_files) == 1 or not self.read_fields: # No need to merge fields, just create a single file reader suppliers.append( - lambda f=need_merge_files[0], dv=deletion_vector: self._create_file_reader( - f, self._get_final_read_data_fields(), dv) + lambda f=need_merge_files[0], dv=deletion_vector, read=group_read: read._create_file_reader( + f, read._get_final_read_data_fields(), dv) ) else: suppliers.append( - lambda files=need_merge_files, dv=deletion_vector: self._create_union_reader(files, dv) + lambda files=need_merge_files, dv=deletion_vector, read=group_read: + read._create_union_reader(files, dv) ) merge_reader = ConcatBatchReader( @@ -1224,7 +1229,10 @@ def _apply_deletion_vector(self, reader, reader_range: Range, deletion_vector): if dv.is_empty(): return reader - if dv_range.from_ > reader_range.from_ or dv_range.to < reader_range.to: + selected_ranges = ([reader_range] if self.row_ranges is None + else Range.and_([reader_range], self.row_ranges)) + if any(dv_range.from_ > selected.from_ or dv_range.to < selected.to + for selected in selected_ranges): raise ValueError( f"Deletion vector range {dv_range} should contain reader range {reader_range}." ) @@ -1274,7 +1282,7 @@ def _create_prescan_reader(self, field_names): return prescan_read._create_raw_reader() def _split_by_row_id(self, files: List[DataFileMeta]) -> List[List[DataFileMeta]]: - """Split files by firstRowId for data evolution.""" + """Split by normal row ranges, retaining spanning dedicated files in each group.""" # Sort files by firstRowId and then by maxSequenceNumber def sort_key(file: DataFileMeta) -> tuple: @@ -1282,41 +1290,39 @@ def sort_key(file: DataFileMeta) -> tuple: is_special = 1 if (DataFileMeta.is_blob_file(file.file_name) or DataFileMeta.is_vector_file(file.file_name)) else 0 max_seq = file.max_sequence_number - return (first_row_id, is_special, -max_seq) - - sorted_files = sorted(files, key=sort_key) - - # Split files by firstRowId - split_by_row_id = [] - last_row_id = -1 - check_row_id_start = 0 - current_split = [] - - for file in sorted_files: - first_row_id = file.first_row_id - if first_row_id is None: - split_by_row_id.append([file]) - continue - - if (not DataFileMeta.is_blob_file(file.file_name) - and not DataFileMeta.is_vector_file(file.file_name) - and first_row_id != last_row_id): - if current_split: - split_by_row_id.append(current_split) - if first_row_id < check_row_id_start: - raise ValueError( - f"There are overlapping files in the split: {files}, " - f"the wrong file is: {file}" - ) - current_split = [] - last_row_id = first_row_id - check_row_id_start = first_row_id + file.row_count - current_split.append(file) - - if current_split: - split_by_row_id.append(current_split) - - return split_by_row_id + return (is_special, first_row_id, -max_seq) + + tracked_files = [file for file in files if file.first_row_id is not None] + groups = split_normal_file_groups(tracked_files) + for group in groups: + normal_ranges = { + (file.first_row_id, file.row_count) for file in group + if not DataFileMeta.is_blob_file(file.file_name) + and not DataFileMeta.is_vector_file(file.file_name) + } + if len(normal_ranges) > 1: + raise ValueError(f"There are overlapping files in the split: {group}") + group.sort(key=sort_key) + return [[file] for file in files if file.first_row_id is None] + groups + + def _read_for_normal_range(self, files: List[DataFileMeta]): + normal_files = [file for file in files + if not DataFileMeta.is_blob_file(file.file_name) + and not DataFileMeta.is_vector_file(file.file_name)] + if not normal_files or normal_files[0].first_row_id is None: + return self + normal_range = normal_files[0].row_id_range() + if all(file.row_id_range().from_ >= normal_range.from_ + and file.row_id_range().to <= normal_range.to for file in files): + return self + + # Suppliers are lazy: keep the clipping range on a separate read instance. + group_read = copy(self) + group_read.row_ranges = ( + [normal_range] if self.row_ranges is None + else Range.and_([normal_range], self.row_ranges) + ) + return group_read def _create_union_reader(self, need_merge_files: List[DataFileMeta], deletion_vector=None) -> RecordReader: """Create a DataEvolutionFileReader for merging multiple files.""" @@ -1355,9 +1361,10 @@ def _create_union_reader(self, need_merge_files: List[DataFileMeta], deletion_ve if DataFileMeta.is_blob_file(first_file.file_name): field_ids = [self._get_field_id_from_write_cols(first_file)] elif DataFileMeta.is_vector_file(first_file.file_name): - field_ids = self._get_field_ids_from_write_cols(first_file.write_cols) + field_ids = [bunch.field_id, SpecialFields.ROW_ID.id, + SpecialFields.SEQUENCE_NUMBER.id] elif first_file.write_cols: - field_ids = self._get_field_ids_from_write_cols(first_file.write_cols) + field_ids = self._get_field_ids_from_write_cols(first_file) else: # For regular files without write_cols, derive field IDs from # the file's schema version, not the current table schema. @@ -1424,6 +1431,14 @@ def _create_union_reader(self, need_merge_files: List[DataFileMeta], deletion_ve blob_parallelism=self._blob_parallelism, logical_ranges=[bunch.logical_range()], ) + elif isinstance(bunch, VectorBunch): + vector_read = copy(self) + suppliers = [ + partial(vector_read._create_vector_segment_reader, + file, read_field_names, row_range, deletion_vector) + for row_range, file in bunch.segments() + ] + file_record_readers[i] = MergeAllBatchReader(suppliers, batch_size=batch_size) elif len(bunch.files()) == 1: suppliers = [lambda r=self._create_file_reader( bunch.files()[0], read_field_names, deletion_vector @@ -1486,6 +1501,16 @@ def _create_raw_blob_file_reader( file_size=file.file_size, ) + def _create_vector_segment_reader(self, file, read_fields, row_range, deletion_vector): + segment_read = copy(self) + segment_read.row_ranges = ( + [row_range] if self.row_ranges is None + else Range.and_([row_range], self.row_ranges) + ) + if not segment_read.row_ranges: + return EmptyRecordBatchReader() + return segment_read._create_file_reader(file, read_fields, deletion_vector) + def _split_field_bunches(self, need_merge_files: List[DataFileMeta]) -> List[FieldBunch]: """Split files into field bunches.""" @@ -1504,10 +1529,13 @@ def _split_field_bunches(self, need_merge_files: List[DataFileMeta]) -> List[Fie row_count, row_id_push_down, row_range) blob_bunch_map[field_id].add(file) elif DataFileMeta.is_vector_file(file.file_name): - field_id = self._get_field_id_from_write_cols(file) - if field_id not in vector_bunch_map: - vector_bunch_map[field_id] = VectorBunch(row_count, row_id_push_down) - vector_bunch_map[field_id].add(file) + for field_id in self._get_field_ids_from_write_cols(file): + if field_id in (SpecialFields.ROW_ID.id, SpecialFields.SEQUENCE_NUMBER.id): + continue + if field_id not in vector_bunch_map: + vector_bunch_map[field_id] = VectorBunch( + row_count, row_id_push_down, row_range, field_id) + vector_bunch_map[field_id].add(file) else: fields_files.append(DataBunch(file)) row_count = file.row_count @@ -1523,6 +1551,8 @@ def _split_field_bunches(self, need_merge_files: List[DataFileMeta]) -> List[Fie def _bunch_first_row_id(bunch: FieldBunch) -> int: if isinstance(bunch, BlobBunch): return bunch.logical_range().from_ + if isinstance(bunch, VectorBunch): + return bunch.segments()[0][0].from_ return bunch.files()[0].first_row_id def _get_field_id_from_write_cols(self, file: DataFileMeta) -> int: @@ -1530,17 +1560,17 @@ def _get_field_id_from_write_cols(self, file: DataFileMeta) -> int: if not file.write_cols or len(file.write_cols) == 0: raise ValueError("Blob/vector file must have write columns") - # Find the field by name in the table schema + # write_cols names belong to the file's historical schema. field_name = file.write_cols[0] - for field in self.table.fields: + for field in self._resolve_schema(file.schema_id).fields: if field.name == field_name: return field.id raise ValueError(f"Field {field_name} not found in table schema") - def _get_field_ids_from_write_cols(self, write_cols: List[str]) -> List[int]: + def _get_field_ids_from_write_cols(self, file: DataFileMeta) -> List[int]: field_ids = [] - for field_name in write_cols: - for field in self.table.fields: + for field_name in file.write_cols: + for field in self._resolve_schema(file.schema_id).fields: if field.name == field_name: field_ids.append(field.id) field_ids.append(SpecialFields.ROW_ID.id) diff --git a/paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py b/paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py new file mode 100644 index 000000000000..da9d47094f99 --- /dev/null +++ b/paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py @@ -0,0 +1,237 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import tempfile +import unittest + +import pyarrow as pa + +from pypaimon import CatalogFactory, Schema +from pypaimon.manifest.schema.data_file_meta import DataFileMeta +from pypaimon.read.split import DataSplit +from pypaimon.schema.schema_change import SchemaChange +from pypaimon.utils.range import Range + + +class DataEvolutionSpanningDedicatedTest(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.catalog = CatalogFactory.create({ + 'warehouse': os.path.join(self.tempdir.name, 'warehouse')}) + catalog = self.catalog + catalog.create_database('default', True) + schema = pa.schema([ + ('id', pa.int32()), + ('payload', pa.large_binary()), + ('embedding', pa.list_(pa.float32(), 2)), + ]) + catalog.create_table('default.spanning', Schema.from_pyarrow_schema( + schema, options={ + 'row-tracking.enabled': 'true', + 'data-evolution.enabled': 'true', + 'deletion-vectors.enabled': 'true', + 'vector.file.format': 'parquet', + 'read.batch-size': '2', + }), False) + self.table = catalog.get_table('default.spanning') + self.data = pa.Table.from_pydict({ + 'id': list(range(10)), + 'payload': [f'blob-{i}'.encode() for i in range(10)], + 'embedding': [[float(i), float(i + 1)] for i in range(10)], + }, schema=schema) + + # Reproduce the persistent layout after splitting only normal files. + # Readers must support it even with split-large-files absent or disabled. + self._write(self.table.copy({'target-file-row-num': '3'}), ['id'], self.data) + self._write(self.table, ['payload'], self.data.slice(0, 7), 0) + self._write(self.table, ['payload'], self.data.slice(7, 3), 7) + self._write(self.table, ['embedding'], self.data, 0) + + def tearDown(self): + self.tempdir.cleanup() + + @staticmethod + def _write(table, columns, data, first_row_id=None): + builder = table.new_batch_write_builder() + writer = builder.new_write().with_write_type(columns) + commit = builder.new_commit() + try: + writer.write_arrow(data.select(columns)) + messages = writer.prepare_commit() + if first_row_id is not None: + for message in messages: + for file in message.new_files: + file.first_row_id = first_row_id + commit.commit(messages) + finally: + writer.close() + commit.close() + + def _assert_rows(self, actual, expected_ids): + actual = actual.sort_by([('_ROW_ID', 'ascending')]) + self.assertEqual(actual['_ROW_ID'].to_pylist(), expected_ids) + self.assertEqual(actual['payload'].to_pylist(), + [f'blob-{i}'.encode() for i in expected_ids]) + self.assertEqual(actual['embedding'].to_pylist(), + [[float(i), float(i + 1)] for i in expected_ids]) + + def test_full_scan_projection_and_row_ranges(self): + builder = self.table.new_read_builder().with_projection( + ['payload', '_ROW_ID', 'embedding']) + splits = builder.new_scan().plan().splits() + self.assertEqual(sum(split.merged_row_count() for split in splits), 10) + self._assert_rows(builder.new_read().to_arrow(splits), list(range(10))) + + splits = builder.new_scan().with_row_ranges([Range(2, 7)]).plan().splits() + self._assert_rows(builder.new_read().to_arrow(splits), list(range(2, 8))) + + def test_deletions_and_chunk_shuffle_use_each_normal_anchor(self): + builder = self.table.new_batch_write_builder() + messages = builder.new_update().delete_by_row_id([2, 4, 8]) + commit = builder.new_commit() + try: + commit.commit(messages) + finally: + commit.close() + + expected = [0, 1, 3, 5, 6, 7, 9] + read_builder = self.table.new_read_builder().with_projection( + ['payload', '_ROW_ID', 'embedding']) + splits = read_builder.new_scan().plan().splits() + self.assertEqual(sum(split.merged_row_count() for split in splits), len(expected)) + self._assert_rows(read_builder.new_read().to_arrow(splits), expected) + + splits = read_builder.new_scan().with_row_ranges([Range(2, 7)]).plan().splits() + self._assert_rows(read_builder.new_read().to_arrow(splits), [3, 5, 6, 7]) + + chunks = read_builder.new_scan().with_chunk_shuffle( + seed=17, chunk_size=4).plan().splits() + self.assertEqual(sum(split.merged_row_count() for split in chunks), len(expected)) + for split in chunks: + names = [file.file_name for file in split.files] + self.assertEqual(len(names), len(set(names))) + self._assert_rows(read_builder.new_read().to_arrow(chunks), expected) + + def test_vector_middle_update_survives_normal_merge_and_rename(self): + self.catalog.alter_table('default.spanning', [ + SchemaChange.rename_column('embedding', 'renamed')], False) + self.table = self.catalog.get_table('default.spanning') + updated = self.data.slice(3, 3).set_column( + 2, self.data.schema.field('embedding'), + pa.array([[103.0, 104.0], [104.0, 105.0], [105.0, 106.0]], + type=self.data.schema.field('embedding').type)) + updated = updated.rename_columns(['id', 'payload', 'renamed']) + self._write(self.table, ['renamed'], updated, 3) + expected = [[float(i), float(i + 1)] for i in range(10)] + expected[3:6] = [[103.0, 104.0], [104.0, 105.0], [105.0, 106.0]] + read_builder = self.table.new_read_builder() + actual = read_builder.new_read().to_arrow( + read_builder.new_scan().plan().splits()).sort_by([('id', 'ascending')]) + self.assertEqual(actual['renamed'].to_pylist(), expected) + + self._merge_normal_files(self.table, self.data) + builder = self.table.new_read_builder().with_projection(['_ROW_ID', 'renamed']) + for selected in [None, [Range(1, 7)]]: + scan = builder.new_scan() + if selected is not None: + scan.with_row_ranges(selected) + actual = builder.new_read().to_arrow(scan.plan().splits()).sort_by( + [('_ROW_ID', 'ascending')]) + ids = list(range(10)) if selected is None else list(range(1, 8)) + self.assertEqual(actual['_ROW_ID'].to_pylist(), ids) + self.assertEqual(actual['renamed'].to_pylist(), [expected[i] for i in ids]) + + @staticmethod + def _merge_normal_files(table, data): + # Re-merge normal ranges while keeping every dedicated version unchanged. + old_files = [file for split in table.new_read_builder().new_scan().plan().splits() + for file in split.files + if not DataFileMeta.is_blob_file(file.file_name) + and not DataFileMeta.is_vector_file(file.file_name)] + builder = table.new_batch_write_builder() + writer = builder.new_write().with_write_type(['id']) + commit = builder.new_commit() + try: + writer.write_arrow(data.select(['id'])) + messages = writer.prepare_commit() + messages[0].deleted_files = old_files + for file in messages[0].new_files: + file.first_row_id = 0 + commit.commit(messages) + finally: + writer.close() + commit.close() + + def test_multi_vector_file_keeps_untouched_column_during_partial_update(self): + vector_type = pa.list_(pa.float32(), 2) + schema = pa.schema([('id', pa.int32()), ('left', vector_type), ('right', vector_type)]) + data = pa.Table.from_pydict({ + 'id': list(range(10)), + 'left': [[float(i), float(i + 1)] for i in range(10)], + 'right': [[float(i + 20), float(i + 21)] for i in range(10)], + }, schema=schema) + self.catalog.create_table('default.multi_vector', Schema.from_pyarrow_schema( + schema, options={ + 'row-tracking.enabled': 'true', + 'data-evolution.enabled': 'true', + 'vector.file.format': 'parquet', + }), False) + table = self.catalog.get_table('default.multi_vector') + self._write(table.copy({'target-file-row-num': '3'}), ['id'], data) + self._write(table, ['left', 'right'], data, 0) + vector_files = [file for split in table.new_read_builder().new_scan().plan().splits() + for file in split.files if DataFileMeta.is_vector_file(file.file_name)] + self.assertEqual([file.write_cols for file in vector_files], [['left', 'right']]) + + self.catalog.alter_table('default.multi_vector', [ + SchemaChange.rename_column('left', 'renamed')], False) + table = self.catalog.get_table('default.multi_vector') + updates = [[103.0, 104.0], [104.0, 105.0], [105.0, 106.0]] + self._write(table, ['renamed'], pa.Table.from_pydict( + {'renamed': updates}, schema=pa.schema([('renamed', vector_type)])), 3) + expected = data['left'].to_pylist() + expected[3:6] = updates + + for merge_normal_files in [False, True]: + if merge_normal_files: + self._merge_normal_files(table, data) + builder = table.new_read_builder().with_projection(['_ROW_ID', 'renamed', 'right']) + for selected in [None, [Range(1, 7)]]: + scan = builder.new_scan() + if selected is not None: + scan.with_row_ranges(selected) + actual = builder.new_read().to_arrow(scan.plan().splits()).sort_by( + [('_ROW_ID', 'ascending')]) + ids = list(range(10)) if selected is None else list(range(1, 8)) + self.assertEqual(actual['_ROW_ID'].to_pylist(), ids) + self.assertEqual(actual['renamed'].to_pylist(), [expected[i] for i in ids]) + self.assertEqual(actual['right'].to_pylist(), + [[float(i + 20), float(i + 21)] for i in ids]) + + # With DVs disabled, a vector-only projection may prune every normal file. + vector_builder = table.new_read_builder().with_projection(['renamed', 'right']) + splits = vector_builder.new_scan().plan().splits() + projected_splits = [DataSplit( + files=[file for file in split.files if DataFileMeta.is_vector_file(file.file_name)], + partition=split.partition, bucket=split.bucket, raw_convertible=False) + for split in splits] + for input_splits in [splits, projected_splits]: + actual = vector_builder.new_read().to_arrow(input_splits) + rows = sorted(actual.to_pylist(), key=lambda row: row['right'][0]) + self.assertEqual([row['renamed'] for row in rows], expected) + self.assertEqual([row['right'] for row in rows], data['right'].to_pylist()) diff --git a/paimon-python/pypaimon/tests/write/conflict_detection_test.py b/paimon-python/pypaimon/tests/write/conflict_detection_test.py index a82d9b0ee3a7..b6ec24f8572a 100644 --- a/paimon-python/pypaimon/tests/write/conflict_detection_test.py +++ b/paimon-python/pypaimon/tests/write/conflict_detection_test.py @@ -229,13 +229,15 @@ def _make_detection(self): def test_reports_dedicated_file_spanning_data_files(self): detection = self._make_detection() + detection._row_id_check_from_snapshot = 1 entries = [ _make_entry("f1", kind=0, first_row_id=0, row_count=2), _make_entry("f2", kind=0, first_row_id=2, row_count=2), _make_entry("p1.blob", kind=0, first_row_id=0, row_count=4), ] - result = detection.check_row_id_range_conflicts("COMPACT", entries) + result = detection.check_row_id_range_conflicts( + "APPEND", entries, entries[:2], entries[2:]) self.assertIsNotNone(result) self.assertIn("dedicated file", str(result)) @@ -251,7 +253,7 @@ def test_allows_adjacent_data_files(self): _make_entry("f2", kind=0, first_row_id=2, row_count=2), ] - result = detection.check_row_id_range_conflicts("COMPACT", entries) + result = detection.check_row_id_range_conflicts("COMPACT", entries, entries, []) self.assertIsNone(result) @@ -262,10 +264,80 @@ def test_allows_dedicated_file_covered_by_one_data_file(self): _make_entry("p1.blob", kind=0, first_row_id=1, row_count=2), ] - result = detection.check_row_id_range_conflicts("COMPACT", entries) + result = detection.check_row_id_range_conflicts( + "COMPACT", entries, entries[:1], entries[1:]) self.assertIsNone(result) + def test_normal_split_retains_dedicated_file(self): + for dedicated_file in ("retained.blob", "retained.vector.json"): + with self.subTest(dedicated_file=dedicated_file): + base = [ + _make_entry("normal", first_row_id=0, row_count=4), + _make_entry(dedicated_file, first_row_id=0, row_count=4), + ] + delta = [ + _make_entry("normal", kind=1, first_row_id=0, row_count=4), + _make_entry("split-1", first_row_id=0, row_count=2), + _make_entry("split-2", first_row_id=2, row_count=2), + ] + self.assertIsNone(self._make_detection().check_conflicts( + None, base, delta, "COMPACT")) + + def test_normal_compaction_retains_dedicated_file_outside_scanned_range(self): + for dedicated_file in ("retained.blob", "retained.vector.json"): + with self.subTest(dedicated_file=dedicated_file): + base = [ + _make_entry("normal", first_row_id=2, row_count=2), + _make_entry(dedicated_file, first_row_id=0, row_count=6), + ] + delta = [ + _make_entry("normal", kind=1, first_row_id=2, row_count=2), + _make_entry("split-1", first_row_id=2, row_count=1), + _make_entry("split-2", first_row_id=3, row_count=1), + ] + self.assertIsNone(self._make_detection().check_conflicts( + None, base, delta, "COMPACT")) + + def test_normal_compaction_rejects_gap_or_orphan_under_retained_dedicated_file(self): + for dedicated_file in ("retained.blob", "retained.vector.json"): + for remaining in ( + [], + [_make_entry("split-1", first_row_id=0, row_count=1), + _make_entry("split-2", first_row_id=2, row_count=2)]): + with self.subTest(dedicated_file=dedicated_file, remaining=remaining): + base = [ + _make_entry("normal", first_row_id=0, row_count=4), + _make_entry(dedicated_file, first_row_id=0, row_count=4), + ] + delta = [_make_entry("normal", kind=1, first_row_id=0, row_count=4)] + remaining + result = self._make_detection().check_conflicts(None, base, delta, "COMPACT") + self.assertIsNotNone(result) + self.assertIn("dedicated file", str(result)) + + def test_dedicated_compaction_may_span_split_normal_files(self): + for dedicated_file in ("compacted.blob", "compacted.vector.json"): + with self.subTest(dedicated_file=dedicated_file): + base = [ + _make_entry("split-1", first_row_id=0, row_count=2), + _make_entry("split-2", first_row_id=2, row_count=2), + ] + delta = [_make_entry(dedicated_file, first_row_id=0, row_count=4)] + self.assertIsNone(self._make_detection().check_conflicts( + None, base, delta, "COMPACT")) + + def test_merge_rejects_stale_file_across_split_normal_ranges(self): + for merge_file in ("merge-normal", "merge.blob", "merge.vector.json"): + with self.subTest(merge_file=merge_file): + detection = self._make_detection() + detection._row_id_check_from_snapshot = 1 + base = [ + _make_entry("split-1", first_row_id=0, row_count=2), + _make_entry("split-2", first_row_id=2, row_count=2), + ] + delta = [_make_entry(merge_file, first_row_id=0, row_count=4)] + self.assertIsNotNone(detection.check_conflicts(None, base, delta, "APPEND")) + class TestOverwriteConflictDetection(unittest.TestCase): diff --git a/paimon-python/pypaimon/utils/data_evolution_utils.py b/paimon-python/pypaimon/utils/data_evolution_utils.py index 06c5d3626563..1dcecd4e8007 100644 --- a/paimon-python/pypaimon/utils/data_evolution_utils.py +++ b/paimon-python/pypaimon/utils/data_evolution_utils.py @@ -17,13 +17,53 @@ """Utilities for data-evolution tables.""" -from typing import Callable, Iterable, TypeVar +from bisect import bisect_left +from typing import Callable, Iterable, List, TypeVar from pypaimon.manifest.schema.data_file_meta import DataFileMeta +from pypaimon.utils.range_helper import RangeHelper T = TypeVar("T") +def split_normal_file_groups(files: List[DataFileMeta]) -> List[List[DataFileMeta]]: + """Keep normal ranges separate and attach every intersecting dedicated file.""" + normal_files = [] + dedicated_files = [] + for file in files: + if DataFileMeta.is_blob_file(file.file_name) or DataFileMeta.is_vector_file(file.file_name): + dedicated_files.append(file) + else: + normal_files.append(file) + + helper = RangeHelper(lambda file: file.row_id_range()) + groups = helper.merge_overlapping_ranges(normal_files) + starts = [min(file.row_id_range().from_ for file in group) for group in groups] + ends = [max(file.row_id_range().to for file in group) for group in groups] + unassociated = [] + for file in dedicated_files: + file_range = file.row_id_range() + index = bisect_left(ends, file_range.from_) + associated = False + while index < len(groups) and starts[index] <= file_range.to: + groups[index].append(file) + associated = True + index += 1 + if not associated: + unassociated.append(file) + + groups.extend(helper.merge_overlapping_ranges(unassociated)) + + def group_start(group): + normal_starts = [file.row_id_range().from_ for file in group + if not DataFileMeta.is_blob_file(file.file_name) + and not DataFileMeta.is_vector_file(file.file_name)] + return min(normal_starts or [file.row_id_range().from_ for file in group]) + + groups.sort(key=group_start) + return groups + + def retrieve_anchor_file( entries: Iterable[T], file_meta_func: Callable[[T], DataFileMeta] = lambda entry: entry, diff --git a/paimon-python/pypaimon/write/commit/conflict_detection.py b/paimon-python/pypaimon/write/commit/conflict_detection.py index 2792c84fca36..ec839e3b2230 100644 --- a/paimon-python/pypaimon/write/commit/conflict_detection.py +++ b/paimon-python/pypaimon/write/commit/conflict_detection.py @@ -270,7 +270,8 @@ def check_conflicts( if conflict is not None: return conflict - conflict = self.check_row_id_range_conflicts(commit_kind, merged_entries) + conflict = self.check_row_id_range_conflicts( + commit_kind, merged_entries, base_entries, delta_entries) if conflict is not None: return conflict @@ -567,7 +568,8 @@ def check_row_id_existence(self, base_entries, delta_entries, next_row_id=None): return None - def check_row_id_range_conflicts(self, commit_kind, commit_entries): + def check_row_id_range_conflicts(self, commit_kind, commit_entries, + base_entries, delta_entries): if not self.data_evolution_enabled: return None if self._row_id_check_from_snapshot is None and commit_kind != "COMPACT": @@ -597,7 +599,7 @@ def check_row_id_range_conflicts(self, commit_kind, commit_entries): if self._is_dedicated_file(entry.file.file_name) ] conflict = self._check_dedicated_file_row_id_range_conflicts( - data_files, dedicated_files) + commit_kind, data_files, dedicated_files, base_entries, delta_entries) if conflict is not None: return conflict @@ -616,17 +618,35 @@ def _check_data_file_row_id_range_conflicts(self, range_helper, data_files): return None def _check_dedicated_file_row_id_range_conflicts( - self, data_files, dedicated_files): + self, commit_kind, data_files, dedicated_files, base_entries, delta_entries): if not dedicated_files: return None data_ranges = self._data_file_row_ranges(data_files) + base_ranges = self._data_file_row_ranges([ + entry for entry in base_entries + if entry.file.first_row_id is not None + and not self._is_dedicated_file(entry.file.file_name) + ]) + added_files = {entry.identifier() for entry in delta_entries if entry.kind == 0} for dedicated_file in dedicated_files: dedicated_range = dedicated_file.file.row_id_range() if any(self._contains(row_range, dedicated_range) for row_range in data_ranges): continue + if dedicated_file.identifier() not in added_files: + # A normal-only compaction retains dedicated files even when their ranges span + # multiple output files. Partial scans only expose part of the base coverage. + previously_covered = Range.and_([dedicated_range], base_ranges) + if previously_covered and all( + not row_range.exclude(data_ranges) for row_range in previously_covered): + continue + elif commit_kind == "COMPACT" and not dedicated_range.exclude(data_ranges): + # Dedicated compaction may merge across normal-file boundaries, but newly + # written DML files still need to fit one range to reject stale writers. + continue + intersecting_ranges = [ row_range for row_range in data_ranges if row_range.overlaps(dedicated_range) diff --git a/paimon-python/pypaimon/write/table_delete.py b/paimon-python/pypaimon/write/table_delete.py index 5c475c31b1f6..6e0f4a7617ee 100644 --- a/paimon-python/pypaimon/write/table_delete.py +++ b/paimon-python/pypaimon/write/table_delete.py @@ -27,12 +27,9 @@ from pypaimon.manifest.index_manifest_entry import IndexManifestEntry from pypaimon.manifest.index_manifest_file import IndexManifestFile from pypaimon.manifest.schema.data_file_meta import DataFileMeta -from pypaimon.read.scanner.data_evolution_split_generator import ( - DataEvolutionSplitGenerator, -) from pypaimon.table.row.generic_row import GenericRow from pypaimon.table.source.deletion_file import DeletionFile -from pypaimon.utils.data_evolution_utils import retrieve_anchor_file +from pypaimon.utils.data_evolution_utils import retrieve_anchor_file, split_normal_file_groups from pypaimon.utils.file_store_path_factory import FileStorePathFactory from pypaimon.utils.range import Range from pypaimon.write.commit_message import CommitMessage @@ -144,7 +141,7 @@ def _scan_anchor_ranges(self) -> Tuple[int, List[_AnchorRange]]: for file in split.files if file.row_id_range() is not None ] - for group in DataEvolutionSplitGenerator._split_by_row_id(files): + for group in split_normal_file_groups(files): anchor = retrieve_anchor_file(group) anchors.append( _AnchorRange( From 4e1ee8e9d25b5560547e420e105179b6aee54d51 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 10 Sep 2026 08:34:54 +0800 Subject: [PATCH 3/5] [core] Configure data evolution large-file size ratio --- docs/docs/multimodal-table/data-evolution.mdx | 17 ++-- docs/generated/core_configuration.html | 8 +- .../java/org/apache/paimon/CoreOptions.java | 23 ++++- .../CompactCandidateRangeCollector.java | 19 ++-- .../DataEvolutionCompactCoordinator.java | 34 ++++--- .../DataEvolutionCompactRangePlanner.java | 8 +- .../CompactCandidateRangeCollectorTest.java | 34 ++++++- .../DataEvolutionCompactCoordinatorTest.java | 94 ++++++++++++++++++- .../DataEvolutionCompactRangePlannerTest.java | 2 +- .../DataEvolutionNormalCompactTaskTest.java | 7 +- 10 files changed, 201 insertions(+), 45 deletions(-) diff --git a/docs/docs/multimodal-table/data-evolution.mdx b/docs/docs/multimodal-table/data-evolution.mdx index 1f3de44363e4..310627a05033 100644 --- a/docs/docs/multimodal-table/data-evolution.mdx +++ b/docs/docs/multimodal-table/data-evolution.mdx @@ -616,16 +616,21 @@ To resize existing large normal data files, enable ```sql ALTER TABLE my_table SET TBLPROPERTIES ( 'target-file-size' = '128 MB', - 'data-evolution.compaction.split-large-files' = 'true' + 'data-evolution.compaction.split-large-files' = 'true', + 'data-evolution.compaction.large-file-ratio' = '3.0' ); CALL sys.compact('default.my_table'); ``` -The example uses Spark SQL. With this option enabled, normal files strictly larger -than twice `target-file-size` qualify for compaction even if the file count is below -`compaction.min.file-num`. Compaction includes all column updates for the same row-ID -range and rolls normal output files near `target-file-size`. Actual sizes depend on -compression and the writer's size-check granularity; the last file may be smaller. +The example uses Spark SQL and selects normal files strictly larger than 384 MB. +`data-evolution.compaction.large-file-ratio` controls the multiplier relative to +`target-file-size`; it defaults to `2.0` and accepts finite values of at least `1.0`, +including fractional values such as `1.5`. Files strictly exceeding the threshold +qualify for compaction even if the file count is below `compaction.min.file-num`. +Changing the ratio does not change the output target size. Compaction includes all +column updates for the same row-ID range and rolls normal output files near +`target-file-size`. Actual sizes depend on compression and the writer's size-check +granularity; the last file may be smaller. The write-time `target-file-row-num` limit does not apply to compaction. Row IDs, column updates, and logical deletions are preserved. This option only rewrites diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 57c50f641772..fb0d358870b9 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -506,6 +506,12 @@ Duration The TTL in local index for cross partition upsert (primary keys not contain all partition fields), this can avoid maintaining too many indexes and lead to worse and worse performance, but please note that this may also cause data duplication. + +

data-evolution.compaction.large-file-ratio
+ 2.0 + Double + Size multiplier relative to target-file-size for selecting large normal files when data-evolution.compaction.split-large-files is enabled. An individual file must strictly exceed this threshold. The value must be finite and at least 1.0. This does not change the target size of compacted output files. +
data-evolution.compaction.rewrite-row-ids
false @@ -516,7 +522,7 @@
data-evolution.compaction.split-large-files
false Boolean - Whether data-evolution compaction selects normal data files larger than twice target-file-size, even below compaction.min.file-num. When enabled, normal compaction output rolls at target-file-size while preserving row IDs and logical deletions. This option does not rewrite associated BLOB or VECTOR files. + Whether data-evolution compaction selects normal data files larger than data-evolution.compaction.large-file-ratio times target-file-size, even below compaction.min.file-num. When enabled, normal compaction output rolls at target-file-size while preserving row IDs and logical deletions. This option does not rewrite associated BLOB or VECTOR files.
data-evolution.enabled
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 1b261cf24ed9..f33e450a11f8 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2640,11 +2640,23 @@ public String toString() { .defaultValue(false) .withDescription( "Whether data-evolution compaction selects normal data files larger than " - + "twice target-file-size, even below compaction.min.file-num. " + + "data-evolution.compaction.large-file-ratio times target-file-size, " + + "even below compaction.min.file-num. " + "When enabled, normal compaction output rolls at target-file-size " + "while preserving row IDs and logical deletions. This option does not " + "rewrite associated BLOB or VECTOR files."); + public static final ConfigOption DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO = + key("data-evolution.compaction.large-file-ratio") + .doubleType() + .defaultValue(2.0d) + .withDescription( + "Size multiplier relative to target-file-size for selecting large normal " + + "files when data-evolution.compaction.split-large-files is enabled. " + + "An individual file must strictly exceed this threshold. The value " + + "must be finite and at least 1.0. This does not change the target " + + "size of compacted output files."); + public static final ConfigOption DATA_EVOLUTION_COMPACTION_REWRITE_ROW_IDS = key("data-evolution.compaction.rewrite-row-ids") .booleanType() @@ -4405,6 +4417,15 @@ public boolean dataEvolutionCompactionSplitLargeFiles() { return options.get(DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES); } + public double dataEvolutionCompactionLargeFileRatio() { + double ratio = options.get(DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO); + checkArgument( + Double.isFinite(ratio) && ratio >= 1.0d, + "The option %s must be finite and at least 1.0.", + DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO.key()); + return ratio; + } + public boolean dataEvolutionCompactionRewriteRowIds() { return options.get(DATA_EVOLUTION_COMPACTION_REWRITE_ROW_IDS); } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java index c15c89135d21..e991892fd4c2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollector.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.PriorityQueue; -import static org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator.isLargeFile; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.paimon.utils.Preconditions.checkState; @@ -54,7 +53,7 @@ final class CompactCandidateRangeCollector { private final long blobTargetFileSize; private final long openFileCost; private final long compactMinFileNum; - private final boolean splitLargeFiles; + private final long largeFileThreshold; private final List sortedChunks = new ArrayList<>(); private long[] words; private int chunkSize; @@ -67,7 +66,7 @@ final class CompactCandidateRangeCollector { long blobTargetFileSize, long openFileCost, long compactMinFileNum, - boolean splitLargeFiles) { + long largeFileThreshold) { checkArgument(expectedFileCount >= 0, "Expected live file count cannot be negative."); checkArgument(targetFileSize > 0, "Target file size must be positive."); checkArgument(blobTargetFileSize > 0, "Blob target file size must be positive."); @@ -78,7 +77,7 @@ final class CompactCandidateRangeCollector { this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; this.compactMinFileNum = compactMinFileNum; - this.splitLargeFiles = splitLargeFiles; + this.largeFileThreshold = largeFileThreshold; int initialEntries = Math.max(16, Math.min(expectedFileCount, ENTRY_CHUNK_SIZE)); this.words = new long[Math.multiplyExact(initialEntries, ENTRY_WORDS)]; } @@ -141,7 +140,7 @@ void finish(CandidateRangeConsumer consumer) { blobTargetFileSize, openFileCost, compactMinFileNum, - splitLargeFiles, + largeFileThreshold, consumer); if (chunks.size() == 1) { SortedEntryChunk chunk = chunks.get(0); @@ -407,7 +406,7 @@ private static final class CandidateAccumulator { private final long blobTargetFileSize; private final long openFileCost; private final long compactMinFileNum; - private final boolean splitLargeFiles; + private final long largeFileThreshold; private final CandidateRangeConsumer consumer; private final CandidateBin bin = new CandidateBin(); private final Map blobFields = new HashMap<>(); @@ -429,13 +428,13 @@ private CandidateAccumulator( long blobTargetFileSize, long openFileCost, long compactMinFileNum, - boolean splitLargeFiles, + long largeFileThreshold, CandidateRangeConsumer consumer) { this.targetFileSize = targetFileSize; this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; this.compactMinFileNum = compactMinFileNum; - this.splitLargeFiles = splitLargeFiles; + this.largeFileThreshold = largeFileThreshold; this.consumer = consumer; } @@ -466,7 +465,7 @@ private void startComponent(long start, long end, long fileSize) { normalEnd = end; normalFileCount = 1L; normalWeight = Math.max(fileSize, openFileCost); - largeFile = splitLargeFiles && isLargeFile(fileSize, targetFileSize); + largeFile = fileSize > largeFileThreshold; vectorFileCount = 0L; componentFileCount = 1; blobFields.clear(); @@ -482,7 +481,7 @@ private void addNormalFile(long start, long end, long fileSize) { checkState( normalEnd == end, "Normal files in one overlapping row-id group must have the same row-id range."); - largeFile |= splitLargeFiles && isLargeFile(fileSize, targetFileSize); + largeFile |= fileSize > largeFileThreshold; normalFileCount = Math.addExact(normalFileCount, 1L); normalWeight = Math.addExact(normalWeight, Math.max(fileSize, openFileCost)); componentFileCount = Math.addExact(componentFileCount, 1); diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java index 1bb11c5be697..6726c56be832 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java @@ -41,6 +41,7 @@ import javax.annotation.Nullable; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -99,6 +100,11 @@ public DataEvolutionCompactCoordinator( validateOptions(options); long targetFileSize = options.targetFileSize(false); + long largeFileThreshold = + options.dataEvolutionCompactionSplitLargeFiles() + ? largeFileThreshold( + targetFileSize, options.dataEvolutionCompactionLargeFileRatio()) + : Long.MAX_VALUE; long openFileCost = options.splitOpenFileCost(); long compactMinFileNum = options.compactionMinFileNum(); Set blobInlineFields = options.blobInlineField(); @@ -118,7 +124,7 @@ public DataEvolutionCompactCoordinator( new DataEvolutionCompactRangePlanner.CandidateOptions( compactBlob, compactVector, - options.dataEvolutionCompactionSplitLargeFiles(), + largeFileThreshold, targetFileSize, options.blobTargetFileSize(), openFileCost, @@ -138,7 +144,7 @@ public DataEvolutionCompactCoordinator( new CompactPlanner( compactBlob, compactVector, - options.dataEvolutionCompactionSplitLargeFiles(), + largeFileThreshold, targetFileSize, options.blobTargetFileSize(), openFileCost, @@ -147,9 +153,12 @@ public DataEvolutionCompactCoordinator( currentBlobFieldIds); } - static boolean isLargeFile(long fileSize, long targetFileSize) { - // Subtraction avoids overflowing twice the target size. - return fileSize > targetFileSize && fileSize - targetFileSize > targetFileSize; + static long largeFileThreshold(long targetFileSize, double ratio) { + // Preserve decimal boundaries and saturate thresholds beyond the largest possible file. + return BigDecimal.valueOf(targetFileSize) + .multiply(BigDecimal.valueOf(ratio)) + .min(BigDecimal.valueOf(Long.MAX_VALUE)) + .longValue(); } public static void validateOptions(CoreOptions options) { @@ -242,7 +251,7 @@ static class CompactPlanner { private final boolean compactBlob; private final boolean compactVector; - private final boolean splitLargeFiles; + private final long largeFileThreshold; private final long targetFileSize; private final long blobTargetFileSize; private final long openFileCost; @@ -260,7 +269,7 @@ static class CompactPlanner { this( compactBlob, compactVector, - false, + Long.MAX_VALUE, targetFileSize, targetFileSize, openFileCost, @@ -275,7 +284,7 @@ static class CompactPlanner { CompactPlanner( boolean compactBlob, boolean compactVector, - boolean splitLargeFiles, + long largeFileThreshold, long targetFileSize, long blobTargetFileSize, long openFileCost, @@ -284,7 +293,7 @@ static class CompactPlanner { @Nullable Set currentBlobFieldIds) { this.compactBlob = compactBlob; this.compactVector = compactVector; - this.splitLargeFiles = splitLargeFiles; + this.largeFileThreshold = largeFileThreshold; this.targetFileSize = targetFileSize; this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; @@ -422,12 +431,7 @@ private List triggerTask( List tasks = new ArrayList<>(); boolean triggerNormalFile = dataFiles.size() >= compactMinFileNum - || (splitLargeFiles - && dataFiles.stream() - .anyMatch( - f -> - isLargeFile( - f.fileSize(), targetFileSize))); + || dataFiles.stream().anyMatch(f -> f.fileSize() > largeFileThreshold); if (triggerNormalFile) { tasks.add(new DataEvolutionNormalCompactTask(partition, dataFiles)); } diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java index 9c68c1cecb40..236f4fdf5367 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlanner.java @@ -158,7 +158,7 @@ private Queue planManifestGroup(List manifestGroup candidateOptions.blobTargetFileSize, candidateOptions.openFileCost, candidateOptions.compactMinFileNum, - candidateOptions.splitLargeFiles); + candidateOptions.largeFileThreshold); try { collectDeletedIdentifiers(manifestGroup, deletedIdentifiers, identifier); collectCandidateRanges(manifestGroup, deletedIdentifiers, identifier, candidateRanges); @@ -490,7 +490,7 @@ static final class CandidateOptions { private final boolean compactBlob; private final boolean compactVector; - private final boolean splitLargeFiles; + private final long largeFileThreshold; private final long targetFileSize; private final long blobTargetFileSize; private final long openFileCost; @@ -501,7 +501,7 @@ static final class CandidateOptions { CandidateOptions( boolean compactBlob, boolean compactVector, - boolean splitLargeFiles, + long largeFileThreshold, long targetFileSize, long blobTargetFileSize, long openFileCost, @@ -510,7 +510,7 @@ static final class CandidateOptions { @Nullable Set currentBlobFieldIds) { this.compactBlob = compactBlob; this.compactVector = compactVector; - this.splitLargeFiles = splitLargeFiles; + this.largeFileThreshold = largeFileThreshold; this.targetFileSize = targetFileSize; this.blobTargetFileSize = blobTargetFileSize; this.openFileCost = openFileCost; diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java index ca6a640ff2c9..a6a3db391157 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/CompactCandidateRangeCollectorTest.java @@ -19,6 +19,8 @@ package org.apache.paimon.append.dataevolution; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.util.ArrayList; import java.util.List; @@ -26,6 +28,7 @@ import static org.apache.paimon.append.dataevolution.CompactCandidateRangeCollector.IGNORED_DEDICATED_FILE; import static org.apache.paimon.append.dataevolution.CompactCandidateRangeCollector.NORMAL_FILE; import static org.apache.paimon.append.dataevolution.CompactCandidateRangeCollector.VECTOR_FILE; +import static org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator.largeFileThreshold; import static org.assertj.core.api.Assertions.assertThat; /** Tests for {@link CompactCandidateRangeCollector}. */ @@ -49,7 +52,8 @@ void testSelectsOnlyNormalFileBinsWhichCanCompact() { void testSplitLargeFilesUsesPhysicalSizeAndIncludesColumnUpdates() { for (boolean enabled : new boolean[] {false, true}) { CompactCandidateRangeCollector collector = - new CompactCandidateRangeCollector(16, 100L, 100L, 1000L, 10L, enabled); + new CompactCandidateRangeCollector( + 16, 100L, 100L, 1000L, 10L, enabled ? 200L : Long.MAX_VALUE); collector.add(0, NORMAL_FILE, 0L, 10L, 199L); collector.add(0, NORMAL_FILE, 10L, 10L, 200L); collector.add(0, NORMAL_FILE, 20L, 10L, 201L); @@ -64,10 +68,29 @@ void testSplitLargeFilesUsesPhysicalSizeAndIncludesColumnUpdates() { } } + @ParameterizedTest + @CsvSource({"1.0,100", "1.15,115", "1.5,150", "3.0,300"}) + void testCustomLargeFileRatioUsesIndividualPhysicalFileSize(double ratio, long threshold) { + CompactCandidateRangeCollector collector = + new CompactCandidateRangeCollector( + 16, 100L, 100L, 1000L, 10L, largeFileThreshold(100L, ratio)); + collector.add(0, NORMAL_FILE, 0L, 10L, threshold - 1); + collector.add(0, NORMAL_FILE, 10L, 10L, threshold); + collector.add(0, NORMAL_FILE, 20L, 10L, 10L); + collector.add(0, NORMAL_FILE, 20L, 10L, threshold + 1); + // Neither the sum of versions nor dedicated-file sizes bypass the minimum file count. + collector.add(0, NORMAL_FILE, 30L, 10L, threshold * 3 / 4); + collector.add(0, NORMAL_FILE, 30L, 10L, threshold * 3 / 4); + collector.add(0, IGNORED_DEDICATED_FILE, 0L, 10L, 1000L); + + assertThat(finish(collector)).containsExactly("20-29:2"); + } + @Test void testSplitThresholdDoesNotOverflow() { CompactCandidateRangeCollector collector = - new CompactCandidateRangeCollector(16, Long.MAX_VALUE, 100L, 1L, 2L, true); + new CompactCandidateRangeCollector( + 16, Long.MAX_VALUE, 100L, 1L, 2L, largeFileThreshold(Long.MAX_VALUE, 2.0d)); collector.add(0, NORMAL_FILE, 0L, 10L, Long.MAX_VALUE); assertThat(finish(collector)).isEmpty(); } @@ -183,7 +206,12 @@ private CompactCandidateRangeCollector collector( long openFileCost, long compactMinFileNum) { return new CompactCandidateRangeCollector( - 16, targetFileSize, blobTargetFileSize, openFileCost, compactMinFileNum, false); + 16, + targetFileSize, + blobTargetFileSize, + openFileCost, + compactMinFileNum, + Long.MAX_VALUE); } private List finish(CompactCandidateRangeCollector collector) { diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java index 18d8fd385e12..5d553f1a2507 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java @@ -48,6 +48,9 @@ import org.apache.paimon.utils.SnapshotManager; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; import java.util.ArrayList; @@ -62,6 +65,7 @@ import java.util.function.LongFunction; import java.util.stream.Collectors; +import static org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator.largeFileThreshold; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; @@ -89,6 +93,82 @@ table, false, false, mock(Snapshot.class))) .hasMessageContaining("materialize_deletion_vectors"); } + @Test + public void testLargeFileRatioOptions() { + assertThat(new CoreOptions(new Options()).dataEvolutionCompactionLargeFileRatio()) + .isEqualTo(2.0d); + for (double ratio : new double[] {1.0d, 1.15d, Double.MAX_VALUE}) { + Options options = new Options(); + options.set(CoreOptions.DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO, ratio); + assertThat(new CoreOptions(options).dataEvolutionCompactionLargeFileRatio()) + .isEqualTo(ratio); + } + } + + @ParameterizedTest + @ValueSource( + doubles = { + 0.0d, + 0.99d, + -1.0d, + Double.NaN, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY + }) + public void testRejectsInvalidLargeFileRatio(double ratio) { + Options options = new Options(); + options.set(CoreOptions.DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO, ratio); + assertThatThrownBy(() -> new CoreOptions(options).dataEvolutionCompactionLargeFileRatio()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(CoreOptions.DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO.key()); + } + + @Test + public void testLargeFileThresholdPreservesByteBoundaries() { + // Direct double multiplication rounds 100 * 1.15 below 115. + assertThat(largeFileThreshold(100L, 1.15d)).isEqualTo(115L); + assertThat(largeFileThreshold(3L, 1.5d)).isEqualTo(4L); + assertThat(largeFileThreshold((1L << 53) + 1, 1.0d)).isEqualTo((1L << 53) + 1); + assertThat(largeFileThreshold(Long.MAX_VALUE / 2, 2.0d)).isEqualTo(Long.MAX_VALUE - 1); + assertThat(largeFileThreshold(Long.MAX_VALUE / 2 + 1, 2.0d)).isEqualTo(Long.MAX_VALUE); + assertThat(largeFileThreshold(Long.MAX_VALUE, 1.0d)).isEqualTo(Long.MAX_VALUE); + assertThat(largeFileThreshold(100L, Double.MAX_VALUE)).isEqualTo(Long.MAX_VALUE); + } + + @ParameterizedTest + @CsvSource({"1.0,100", "1.15,115", "1.5,150", "3.0,300"}) + public void testCustomLargeFileRatioUsesIndividualPhysicalFileSize( + double ratio, long threshold) { + long versionSize = threshold * 3 / 4; + List entries = + Arrays.asList( + makeEntryWithSize("below.parquet", 0L, 10L, 0, threshold - 1), + makeEntryWithSize("boundary.parquet", 10L, 10L, 0, threshold), + makeEntryWithSize("base.parquet", 20L, 10L, 0, 10L), + makeEntryWithSize("large-update.parquet", 20L, 10L, 1, threshold + 1), + makeEntryWithSize("version1.parquet", 30L, 10L, 0, versionSize), + makeEntryWithSize("version2.parquet", 30L, 10L, 1, versionSize), + makeBlobEntry("large.blob", 0L, 10L, 1000L), + makeVectorStoreEntry("large.vector.lance", 10L, 10L, 1000L)); + DataEvolutionCompactCoordinator.CompactPlanner planner = + new DataEvolutionCompactCoordinator.CompactPlanner( + false, + false, + largeFileThreshold(100L, ratio), + 100L, + 100L, + 1000L, + 10L, + schemaId -> null, + null); + + List tasks = planner.compactPlan(entries); + + assertThat(tasks).hasSize(1); + assertThat(tasks.get(0).compactBefore()) + .containsExactly(entries.get(2).file(), entries.get(3).file()); + } + @Test public void testCompactPlannerSingleFile() { // Single file should not produce compaction tasks @@ -115,7 +195,15 @@ public void testSplitLargeFilesUsesPhysicalSizeAndIncludesColumnUpdates() { for (boolean enabled : new boolean[] {false, true}) { DataEvolutionCompactCoordinator.CompactPlanner planner = new DataEvolutionCompactCoordinator.CompactPlanner( - false, false, enabled, 100L, 100L, 1000L, 10L, schemaId -> null, null); + false, + false, + enabled ? 200L : Long.MAX_VALUE, + 100L, + 100L, + 1000L, + 10L, + schemaId -> null, + null); List tasks = planner.compactPlan(entries); if (enabled) { assertThat(tasks).hasSize(1); @@ -138,7 +226,7 @@ public void testSplitLargeFilesAndMergeSmallFilesKeepDedicatedFiles() { makeVectorStoreEntry("original.vector.lance", 0L, 30L, 1000L)); DataEvolutionCompactCoordinator.CompactPlanner planner = new DataEvolutionCompactCoordinator.CompactPlanner( - false, false, true, 100L, 100L, 1L, 2L, schemaId -> null, null); + false, false, 200L, 100L, 100L, 1L, 2L, schemaId -> null, null); List tasks = planner.compactPlan(entries); @@ -920,7 +1008,7 @@ private DataEvolutionCompactCoordinator.CompactPlanner blobPlanner( return new DataEvolutionCompactCoordinator.CompactPlanner( true, false, - false, + Long.MAX_VALUE, targetFileSize, targetFileSize, openFileCost, diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java index 28c3afe29e75..93059aeea1bb 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactRangePlannerTest.java @@ -400,7 +400,7 @@ private DataEvolutionCompactRangePlanner.CandidateOptions candidateOptions( return new DataEvolutionCompactRangePlanner.CandidateOptions( compactBlob, compactVector, - false, + Long.MAX_VALUE, 100L, 100L, 1L, diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java index d6231f79da4e..10560e0e6297 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java @@ -187,12 +187,17 @@ public void testSplitHistoricalLargeFile(boolean updateColumn) throws Exception options.put(CoreOptions.GLOBAL_INDEX_COLUMN_UPDATE_ACTION.key(), "ignore"); table = table.copy(options); long targetSize = table.coreOptions().targetFileSize(false); - assertThat(original.fileSize()).isGreaterThan(2 * targetSize); + assertThat(original.fileSize()).isGreaterThan(3 * targetSize); Snapshot snapshot = table.snapshotManager().latestSnapshot(); assertThat(new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan()) .isEmpty(); options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO.key(), "1000.0"); + table = table.copy(options); + assertThat(new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan()) + .isEmpty(); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO.key(), "3.0"); table = table.copy(options); List tasks = new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan(); From 4d34bc435be4d77bba4b8edca4878a47ae10354c Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 10 Sep 2026 09:44:20 +0800 Subject: [PATCH 4/5] [core] Respect dedicated file ranges when splitting normal files --- docs/docs/multimodal-table/data-evolution.mdx | 12 +- docs/generated/core_configuration.html | 2 +- .../java/org/apache/paimon/CoreOptions.java | 9 +- .../paimon/append/AppendOnlyWriter.java | 58 ++- .../DataEvolutionCompactCoordinator.java | 72 ++- .../DataEvolutionCompactTaskSerializer.java | 19 +- .../DataEvolutionNormalCompactTask.java | 54 +- .../paimon/io/RowDataRollingFileWriter.java | 49 ++ .../operation/DataEvolutionFileStoreScan.java | 30 +- .../operation/DataEvolutionSplitRead.java | 368 +++++++------ .../DataEvolutionConflictDetection.java | 48 +- .../apache/paimon/table/source/DataSplit.java | 21 +- .../paimon/append/AppendOnlyWriterTest.java | 21 + .../DataEvolutionCompactCoordinatorTest.java | 27 +- .../DataEvolutionNormalCompactTaskTest.java | 490 ++++++++++++++---- .../paimon/io/RollingFileWriterTest.java | 98 ++++ .../DataEvolutionFileStoreScanTest.java | 71 --- .../operation/DataEvolutionReadTest.java | 154 +++--- .../operation/DataEvolutionSplitReadTest.java | 279 ---------- .../commit/ConflictDetectionTest.java | 129 +---- .../DataEvolutionDeletionVectorTest.java | 109 ++-- .../table/source/DataSplitCompatibleTest.java | 53 -- .../pypaimon/read/reader/field_bunch.py | 32 +- .../scanner/chunk_shuffle_split_generator.py | 21 +- paimon-python/pypaimon/read/split_read.py | 134 ++--- .../data_evolution_spanning_dedicated_test.py | 237 --------- .../tests/write/conflict_detection_test.py | 78 +-- .../pypaimon/utils/data_evolution_utils.py | 42 +- .../write/commit/conflict_detection.py | 28 +- paimon-python/pypaimon/write/table_delete.py | 7 +- .../spark/procedure/CompactProcedure.java | 27 +- .../DataEvolutionRewriteExecutor.java | 45 +- .../procedure/CompactProcedureTestBase.scala | 71 ++- 33 files changed, 1321 insertions(+), 1574 deletions(-) delete mode 100644 paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py diff --git a/docs/docs/multimodal-table/data-evolution.mdx b/docs/docs/multimodal-table/data-evolution.mdx index 310627a05033..cf53a31de27b 100644 --- a/docs/docs/multimodal-table/data-evolution.mdx +++ b/docs/docs/multimodal-table/data-evolution.mdx @@ -626,7 +626,8 @@ The example uses Spark SQL and selects normal files strictly larger than 384 MB. `data-evolution.compaction.large-file-ratio` controls the multiplier relative to `target-file-size`; it defaults to `2.0` and accepts finite values of at least `1.0`, including fractional values such as `1.5`. Files strictly exceeding the threshold -qualify for compaction even if the file count is below `compaction.min.file-num`. +qualify for compaction below `compaction.min.file-num` when their dedicated-file +ranges allow splitting. Changing the ratio does not change the output target size. Compaction includes all column updates for the same row-ID range and rolls normal output files near `target-file-size`. Actual sizes depend on compression and the writer's size-check @@ -639,9 +640,12 @@ and their sizes do not trigger splitting. Separate dedicated-file compaction opt their existing behavior. Files referenced by older snapshots or tags remain until those references expire and snapshot expiration removes them. -After splitting, one dedicated file can cover multiple normal-file row-ID ranges. -All Java and PyPaimon readers must support this layout before enabling the option. -Disabling the option later does not undo this layout change. +Every BLOB or VECTOR file must remain fully contained in a single normal file's +row-ID range. After reaching the target size, compaction waits for a boundary that +does not cut through any dedicated file, including overlapping ranges from different +columns or versions. Output files may therefore exceed `target-file-size`. If these +ranges prevent any split, file size alone does not trigger a compaction task. Normal +merging based on `compaction.min.file-num` remains available. ### Materialize Deletion Vectors diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index fb0d358870b9..4cc3744b86e1 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -522,7 +522,7 @@
data-evolution.compaction.split-large-files
false Boolean - Whether data-evolution compaction selects normal data files larger than data-evolution.compaction.large-file-ratio times target-file-size, even below compaction.min.file-num. When enabled, normal compaction output rolls at target-file-size while preserving row IDs and logical deletions. This option does not rewrite associated BLOB or VECTOR files. + Whether data-evolution compaction selects normal data files larger than data-evolution.compaction.large-file-ratio times target-file-size, even below compaction.min.file-num when dedicated-file ranges allow splitting. Normal output rolls toward target-file-size without cutting through any BLOB or VECTOR file range, so output may exceed the target. Row IDs and logical deletions are preserved, and associated BLOB and VECTOR files are not rewritten by this option.
data-evolution.enabled
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index f33e450a11f8..68ca4c86fb23 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2641,10 +2641,11 @@ public String toString() { .withDescription( "Whether data-evolution compaction selects normal data files larger than " + "data-evolution.compaction.large-file-ratio times target-file-size, " - + "even below compaction.min.file-num. " - + "When enabled, normal compaction output rolls at target-file-size " - + "while preserving row IDs and logical deletions. This option does not " - + "rewrite associated BLOB or VECTOR files."); + + "even below compaction.min.file-num when dedicated-file ranges allow splitting. " + + "Normal output rolls toward target-file-size without cutting through " + + "any BLOB or VECTOR file range, so output may exceed the target. " + + "Row IDs and logical deletions are preserved, and associated " + + "BLOB and VECTOR files are not rewritten by this option."); public static final ConfigOption DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO = key("data-evolution.compaction.large-file-ratio") diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java index 906573ad73ca..f70e440fa673 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java @@ -60,6 +60,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.function.LongPredicate; import java.util.function.Supplier; import static org.apache.paimon.types.VectorType.fieldsInVectorFile; @@ -103,6 +104,8 @@ public class AppendOnlyWriter implements BatchRecordWriter, MemoryOwner { private final MemorySize maxDiskSize; @Nullable private CompactDeletionFile compactDeletionFile; + @Nullable private LongPredicate fileRollingPredicate; + private boolean writeStarted; private SinkWriter sinkWriter; private MemorySegmentPool memorySegmentPool; @@ -186,6 +189,20 @@ public AppendOnlyWriter( } } + /** + * Restricts automatic rolling of normal files to accepted boundaries, expressed as the number + * of records written since the last flush. Requires direct writes and must be configured before + * writing. Explicit flushes still close the current file. + */ + public AppendOnlyWriter withFileRollingPredicate(LongPredicate predicate) { + Preconditions.checkState(!writeStarted, "Must configure rolling before writing."); + Preconditions.checkState( + sinkWriter instanceof DirectSinkWriter, + "File rolling predicate requires direct writes."); + this.fileRollingPredicate = Preconditions.checkNotNull(predicate); + return this; + } + private BufferedSinkWriter createBufferedSinkWriter(boolean spillable) { return new BufferedSinkWriter<>( this::createRollingRowWriter, @@ -205,6 +222,7 @@ public void write(InternalRow rowData) throws Exception { "Append-only writer can only accept insert or update_after row kind, but current row kind is: %s. " + "You can configure 'ignore-delete' to ignore retract records.", rowData.getRowKind()); + writeStarted = true; boolean success = sinkWriter.write(rowData); if (!success) { flush(false, false); @@ -220,6 +238,7 @@ public void write(InternalRow rowData) throws Exception { @Override public void writeBundle(BundleRecords bundle) throws Exception { + writeStarted = true; if (sinkWriter instanceof BufferedSinkWriter) { for (InternalRow row : bundle) { write(row); @@ -346,23 +365,28 @@ private RollingFileWriter createRollingRowWriter() { blobContext, omitAllNonDedicatedWriteCols); } - return new RowDataRollingFileWriter( - fileIO, - schemaId, - fileFormat, - targetFileSize, - writeSchema, - pathFactory, - seqNumCounterProvider, - fileCompression, - statsCollectorFactories.statsCollectors(writeSchema.getFieldNames()), - fileIndexOptions, - fileSource, - asyncFileWrite, - statsDenseStore, - writeCols, - rowSidecarFileFormat, - targetFileRowNum); + RowDataRollingFileWriter writer = + new RowDataRollingFileWriter( + fileIO, + schemaId, + fileFormat, + targetFileSize, + writeSchema, + pathFactory, + seqNumCounterProvider, + fileCompression, + statsCollectorFactories.statsCollectors(writeSchema.getFieldNames()), + fileIndexOptions, + fileSource, + asyncFileWrite, + statsDenseStore, + writeCols, + rowSidecarFileFormat, + targetFileRowNum); + if (fileRollingPredicate != null) { + writer.withFileRollingPredicate(fileRollingPredicate); + } + return writer; } private void trySyncLatestCompaction(boolean blocking) diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java index 6726c56be832..5d80b8d97190 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinator.java @@ -58,7 +58,9 @@ import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; import static org.apache.paimon.types.BlobType.isBlobFileField; import static org.apache.paimon.types.VectorType.isVectorStoreFile; +import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange; import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; /** Compact coordinator to compact data evolution table. */ public class DataEvolutionCompactCoordinator { @@ -171,6 +173,12 @@ public static void validateOptions(CoreOptions options) { CoreOptions.DATA_EVOLUTION_COMPACTION_REWRITE_ROW_IDS.key()); } + /** Prevents this run from repeatedly compacting bins containing only its own normal outputs. */ + public DataEvolutionCompactCoordinator withCompletedNormalFiles(Set fileNames) { + planner.completedNormalFiles = checkNotNull(fileNames); + return this; + } + public List plan() { // scan files in snapshot List entries = scanner.scan(); @@ -258,6 +266,7 @@ static class CompactPlanner { private final long compactMinFileNum; private final LongFunction schemaFetcher; @Nullable private final Set currentBlobFieldIds; + private Set completedNormalFiles = Collections.emptySet(); @VisibleForTesting CompactPlanner( @@ -334,12 +343,10 @@ List compactPlan(List input) { } } - if (compactBlob) { - associateDedicatedFiles(blobFiles, treeMap, dataFileToBlobFiles); - } - if (compactVector) { - associateDedicatedFiles(vectorStoreFiles, treeMap, dataFileToVectorStoreFiles); - } + // Retained dedicated files constrain normal output boundaries even when their + // own compaction is disabled. + associateDedicatedFiles(blobFiles, treeMap, dataFileToBlobFiles); + associateDedicatedFiles(vectorStoreFiles, treeMap, dataFileToVectorStoreFiles); RangeHelper continuousDataRangeHelper = new RangeHelper<>( @@ -429,12 +436,32 @@ private List triggerTask( List dataFiles = compactBin.files(); List tasks = new ArrayList<>(); - boolean triggerNormalFile = - dataFiles.size() >= compactMinFileNum - || dataFiles.stream().anyMatch(f -> f.fileSize() > largeFileThreshold); - if (triggerNormalFile) { - tasks.add(new DataEvolutionNormalCompactTask(partition, dataFiles)); + List protectedRanges = new ArrayList<>(); + for (DataFileMeta dataFile : dataFiles) { + dataFileToBlobFiles + .getOrDefault(dataFile, Collections.emptyList()) + .forEach(file -> protectedRanges.add(file.nonNullRowIdRange())); + dataFileToVectorStoreFiles + .getOrDefault(dataFile, Collections.emptyList()) + .forEach(file -> protectedRanges.add(file.nonNullRowIdRange())); } + Range normalRange = checkContiguousRowRange(dataFiles); + boolean hasUncompactedFiles = + dataFiles.stream() + .anyMatch(file -> !completedNormalFiles.contains(file.fileName())); + boolean triggerNormalFile = + hasUncompactedFiles + && (dataFiles.size() >= compactMinFileNum + || (canSplit(normalRange, protectedRanges) + && dataFiles.stream() + .anyMatch( + f -> + f.fileSize() + > largeFileThreshold + && !completedNormalFiles + .contains( + f + .fileName())))); if (compactBlob) { if (triggerNormalFile) { @@ -486,9 +513,32 @@ private List triggerTask( } } } + if (triggerNormalFile) { + // Dedicated compaction in the same batch may combine existing file ranges. + // Its outputs must also fit inside a single normal output file. + for (DataEvolutionCompactTask task : tasks) { + protectedRanges.add(checkContiguousRowRange(task.compactBefore())); + } + if (dataFiles.size() >= compactMinFileNum + || canSplit(normalRange, protectedRanges)) { + tasks.add( + 0, + new DataEvolutionNormalCompactTask( + partition, dataFiles, protectedRanges)); + } + } return tasks; } + private boolean canSplit(Range normalRange, List protectedRanges) { + return normalRange.from < normalRange.to + && Range.sortAndMergeOverlap(protectedRanges).stream() + .noneMatch( + range -> + range.from <= normalRange.from + && range.to >= normalRange.to); + } + private CompactBin compactBin(List files, long groupWeight) { CompactBin bin = new CompactBin(targetFileSize); bin.add(files, groupWeight); diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java index 40ae2073dd17..e9cc9ed19106 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java @@ -27,6 +27,7 @@ import org.apache.paimon.io.DataOutputView; import org.apache.paimon.io.DataOutputViewStreamWrapper; import org.apache.paimon.table.source.DeletionFile; +import org.apache.paimon.utils.Range; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -40,7 +41,7 @@ public class DataEvolutionCompactTaskSerializer implements VersionedSerializer { - private static final int CURRENT_VERSION = 3; + private static final int CURRENT_VERSION = 4; private final DataFileMetaSerializer dataFileSerializer; @@ -73,7 +74,14 @@ private void serialize(DataEvolutionCompactTask task, DataOutputView view) throw serializeBinaryRow(task.partition(), view); dataFileSerializer.serializeList(task.compactBefore(), view); view.writeInt(task.type().code()); - if (task.type() == DataEvolutionCompactTask.TaskType.MATERIALIZE_DELETION) { + if (task.type() == DataEvolutionCompactTask.TaskType.NORMAL) { + List ranges = ((DataEvolutionNormalCompactTask) task).protectedRanges(); + view.writeInt(ranges.size()); + for (Range range : ranges) { + view.writeLong(range.from); + view.writeLong(range.to); + } + } else if (task.type() == DataEvolutionCompactTask.TaskType.MATERIALIZE_DELETION) { DeletionFile.serializeList( view, ((DataEvolutionMaterializeDeletionCompactTask) task).deletionFiles()); } @@ -117,7 +125,12 @@ private DataEvolutionCompactTask deserialize(int version, DataInputView view) DataEvolutionCompactTask.TaskType.fromCode(view.readInt()); switch (type) { case NORMAL: - return new DataEvolutionNormalCompactTask(partition, files); + int rangeCount = view.readInt(); + List ranges = new ArrayList<>(rangeCount); + for (int i = 0; i < rangeCount; i++) { + ranges.add(new Range(view.readLong(), view.readLong())); + } + return new DataEvolutionNormalCompactTask(partition, files, ranges); case BLOB: return new DataEvolutionBlobCompactTask(partition, files); case MATERIALIZE_DELETION: diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java index 62f455fb7c0e..8ad408a814b5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java @@ -20,6 +20,7 @@ import org.apache.paimon.AppendOnlyFileStore; import org.apache.paimon.CoreOptions; +import org.apache.paimon.append.AppendOnlyWriter; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.io.DataFileMeta; @@ -35,7 +36,7 @@ import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.Pair; -import org.apache.paimon.utils.RecordWriter; +import org.apache.paimon.utils.Range; import org.apache.paimon.utils.SetUtils; import org.slf4j.Logger; @@ -43,11 +44,15 @@ import javax.annotation.Nullable; +import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Function; +import java.util.function.LongPredicate; import java.util.stream.Collectors; import static org.apache.paimon.types.BlobType.fieldNamesInBlobFile; @@ -62,9 +67,33 @@ public class DataEvolutionNormalCompactTask extends DataEvolutionCompactTask { private static final Logger LOG = LoggerFactory.getLogger(DataEvolutionNormalCompactTask.class); + private final List protectedRanges; + public DataEvolutionNormalCompactTask(BinaryRow partition, List files) { + this(partition, files, Collections.emptyList()); + } + + public DataEvolutionNormalCompactTask( + BinaryRow partition, List files, List protectedRanges) { super(partition, files); checkContiguousRowRange(files); + this.protectedRanges = + Collections.unmodifiableList(Range.sortAndMergeOverlap(protectedRanges)); + } + + public List protectedRanges() { + return protectedRanges; + } + + @Override + public boolean equals(Object other) { + return super.equals(other) + && protectedRanges.equals(((DataEvolutionNormalCompactTask) other).protectedRanges); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), protectedRanges); } @Override @@ -88,6 +117,8 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E Map writeOptions = new HashMap<>(DYNAMIC_WRITE_OPTIONS); if (options.dataEvolutionCompactionSplitLargeFiles()) { + // Buffer flushes may close files before reaching a safe dedicated-file boundary. + writeOptions.put(CoreOptions.WRITE_BUFFER_FOR_APPEND.key(), "false"); writeOptions.put( CoreOptions.TARGET_FILE_SIZE.key(), options.targetFileSize(false) + " b"); } @@ -115,7 +146,10 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E AppendFileStoreWrite storeWrite = (AppendFileStoreWrite) store.newWrite(commitUser); storeWrite.withWriteType(readWriteType); storeWrite.withFileSource(FileSource.COMPACT); - RecordWriter writer = storeWrite.createWriter(partition, 0); + AppendOnlyWriter writer = (AppendOnlyWriter) storeWrite.createWriter(partition, 0); + if (options.dataEvolutionCompactionSplitLargeFiles() && !protectedRanges.isEmpty()) { + writer.withFileRollingPredicate(fileRollingPredicate(firstRowId)); + } reader.forEachRemaining( row -> { @@ -164,6 +198,22 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E return commitMessage(compactBefore, compactAfter); } + private LongPredicate fileRollingPredicate(long firstRowId) { + Iterator ranges = protectedRanges.iterator(); + return new LongPredicate() { + private Range current = ranges.hasNext() ? ranges.next() : null; + + @Override + public boolean test(long writtenRows) { + long lastRowId = firstRowId + (writtenRows - 1); + while (current != null && current.to <= lastRowId) { + current = ranges.hasNext() ? ranges.next() : null; + } + return current == null || lastRowId < current.from; + } + }; + } + @Nullable private long[] compactedColumnMaxSequenceNumbers( FileStoreTable table, DataFileMeta outputFile) { diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java index 9b35c002aaf9..ec2aa8b68352 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java @@ -28,15 +28,21 @@ import org.apache.paimon.statistics.SimpleColStatsCollector; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.LongCounter; +import org.apache.paimon.utils.Preconditions; import javax.annotation.Nullable; +import java.io.IOException; import java.util.List; +import java.util.function.LongPredicate; import java.util.function.Supplier; /** {@link RollingFileWriterImpl} for data files containing {@link InternalRow}. */ public class RowDataRollingFileWriter extends RollingFileWriterImpl { + @Nullable private LongPredicate fileRollingPredicate; + private boolean pendingRoll; + public RowDataRollingFileWriter( FileIO fileIO, long schemaId, @@ -94,4 +100,47 @@ public RowDataFileWriter get() { targetFileSize, targetFileRowNum); } + + /** + * Restricts automatic rolling to accepted boundaries, expressed as the cumulative number of + * records written by this writer. Must be configured before writing; closing the writer still + * closes the final file regardless of the predicate. + */ + public RowDataRollingFileWriter withFileRollingPredicate(LongPredicate predicate) { + Preconditions.checkState(recordCount() == 0, "Must configure rolling before writing."); + this.fileRollingPredicate = Preconditions.checkNotNull(predicate); + return this; + } + + @Override + protected void beforeWrite(InternalRow row) throws IOException { + if (pendingRoll && fileRollingPredicate.test(recordCount())) { + closeCurrentWriter(); + } + } + + @Override + protected void onRollingCondition(InternalRow row) throws IOException { + if (fileRollingPredicate == null || fileRollingPredicate.test(recordCount())) { + closeCurrentWriter(); + } else { + pendingRoll = true; + } + } + + @Override + protected void onCurrentWriterClosed() { + pendingRoll = false; + } + + @Override + public void writeBundle(BundleRecords bundle) throws IOException { + if (fileRollingPredicate == null) { + super.writeBundle(bundle); + } else { + for (InternalRow row : bundle) { + write(row); + } + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java index 214aeb147160..ee054d445c2f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionFileStoreScan.java @@ -223,14 +223,15 @@ private boolean filterByStats(List entries) { * a row-disjoint pre-ALTER group), one file is kept as a row-count representative so the reader * can emit the right number of NULL-filled rows. * - *

If Deletion-Vector is enabled, we always keep the oldest normal file for each normal - * row-id range as the anchor file to lookup corresponding Deletion Files. A retained dedicated - * file can span several normal ranges after normal-file compaction. + *

If Deletion-Vector is enabled, we always keep the oldest normal file for each group as the + * anchor file to lookup corresponding Deletion Files. */ private List pruneByReadType(List group) { if (readType == null || group.size() <= 1) { return group; } + ManifestEntry anchor = + deletionVectorsEnabled ? retrieveAnchorFile(group, ManifestEntry::file) : null; Set readFieldIds = new HashSet<>(); for (DataField f : readType.getFields()) { readFieldIds.add(f.id()); @@ -245,27 +246,8 @@ private List pruneByReadType(List group) { } } } - List normalFiles = - group.stream() - .filter(entry -> !isBlobFile(entry.file().fileName())) - .filter(entry -> !isVectorStoreFile(entry.file().fileName())) - .collect(Collectors.toList()); - RangeHelper rangeHelper = - new RangeHelper<>(entry -> entry.file().nonNullRowIdRange()); - List> normalGroups = rangeHelper.mergeOverlappingRanges(normalFiles); - Set keptFiles = new HashSet<>(kept); - for (List normalGroup : normalGroups) { - if (deletionVectorsEnabled) { - ManifestEntry anchor = retrieveAnchorFile(normalGroup, ManifestEntry::file); - if (anchor != null && keptFiles.add(anchor)) { - kept.add(anchor); - } - } else if (normalGroups.size() > 1 - && normalGroup.stream().noneMatch(keptFiles::contains)) { - // Preserve each normal range even when only a spanning dedicated column or a - // newly added column is projected. A single representative would lose rows. - kept.add(normalGroup.get(0)); - } + if (anchor != null && !kept.contains(anchor)) { + kept.add(anchor); } // Group must contribute at least one file so the reader sees rowCount and can NULL-fill // missing columns for the projection's rows. diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java index e8ab18f26954..919100ec1f8f 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/DataEvolutionSplitRead.java @@ -94,6 +94,7 @@ import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile; import static org.apache.paimon.utils.ListUtils.isNullOrEmpty; import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; /** * A union {@link SplitRead} to read multiple inner files to merge columns. @@ -230,10 +231,6 @@ private RecordReader createReader( List> splitByRowId = mergeRangesAndSort(files); for (List needMergeFiles : splitByRowId) { - List groupRowRanges = groupRowRanges(needMergeFiles, rowRanges); - if (groupRowRanges != null && groupRowRanges.isEmpty()) { - continue; - } if (needMergeFiles.size() == 1 || readRowType.getFields().isEmpty()) { // No need to merge fields, just create a single file reader suppliers.add( @@ -245,7 +242,7 @@ private RecordReader createReader( dataFilePathFactory, needMergeFiles.get(0), filters, - groupRowRanges, + rowRanges, readRowType, deletionVector); }); @@ -262,7 +259,7 @@ private RecordReader createReader( needMergeFiles, partition, dataFilePathFactory, - groupRowRanges, + rowRanges, readRowType, deletionVector); }); @@ -272,37 +269,6 @@ private RecordReader createReader( return ConcatRecordReader.create(suppliers); } - private static List groupRowRanges( - List files, @Nullable List rowRanges) { - DataFileMeta first = files.get(0); - if (isBlobFile(first.fileName()) || isVectorStoreFile(first.fileName())) { - return rowRanges; - } - Range normalRange = first.nonNullRowIdRange(); - boolean needsClipping = - files.stream() - .anyMatch( - file -> { - Range range = file.nonNullRowIdRange(); - return range.from < normalRange.from - || range.to > normalRange.to; - }); - if (!needsClipping) { - return rowRanges; - } - if (rowRanges == null) { - return Collections.singletonList(normalRange); - } - List intersections = new ArrayList<>(); - for (Range range : rowRanges) { - Range intersection = Range.intersection(normalRange, range); - if (intersection != null) { - intersections.add(intersection); - } - } - return intersections; - } - private RecordReader createReader(IndexedSplit indexedSplit) throws IOException { DataSplit dataSplit = indexedSplit.dataSplit(); List rowRanges = indexedSplit.rowRanges(); @@ -360,16 +326,7 @@ private DataEvolutionFileReader createUnionReader( DataFileMeta first = fieldsFiles.get(i).files().get(0); bunchDataSchemas[i] = schemaFetcher.apply(first.schemaId()).dataFileSchema(first.writeCols()); - RowType availableType = bunchDataSchemas[i].logicalRowType(); - if (fieldsFiles.get(i) instanceof VectorFileBunch) { - int fieldId = ((VectorFileBunch) fieldsFiles.get(i)).fieldId; - availableType = - new RowType( - availableType.getFields().stream() - .filter(field -> field.id() == fieldId) - .collect(Collectors.toList())); - } - bunchAvailTypes.add(rowTypeWithRowTracking(availableType)); + bunchAvailTypes.add(rowTypeWithRowTracking(bunchDataSchemas[i].logicalRowType())); } DataEvolutionReadPlanner.DataEvolutionReadPlan plan = new DataEvolutionReadPlanner(readRowType, bunchAvailTypes, nestedFieldEnabled) @@ -499,10 +456,10 @@ private RecordReader createFieldBunchReader( } else if (bunch instanceof VectorFileBunch) { // for vector bunch, sequential read all data files and concat them return sequentialReadFiles( - (VectorFileBunch) bunch, + bunch.files(), partition, dataFilePathFactory, - readRowType, + formatReaderMapping, rowRanges, deletionVector); } else if (bunch instanceof BlobFileBunch) { @@ -533,37 +490,22 @@ private RecordReader createFieldBunchReader( } private RecordReader sequentialReadFiles( - VectorFileBunch bunch, + List files, BinaryRow partition, DataFilePathFactory dataFilePathFactory, - RowType readRowType, + FormatReaderMapping formatReaderMapping, List rowRanges, @Nullable DeletionVectorWithRange deletionVector) throws IOException { List> readerSuppliers = new ArrayList<>(); - for (VectorFileRange selected : bunch.selectedFiles()) { - DataFileMeta file = selected.file; - List selectedRanges = new ArrayList<>(); - if (rowRanges == null) { - selectedRanges.add(selected.range); - } else { - for (Range range : rowRanges) { - Range intersection = Range.intersection(range, selected.range); - if (intersection != null) { - selectedRanges.add(intersection); - } - } - } - if (selectedRanges.isEmpty()) { - continue; - } + for (DataFileMeta file : files) { readerSuppliers.add( () -> createFileReader( partition, file, - vectorReaderMapping(file, readRowType), - selectedRanges, + formatReaderMapping, + rowRanges, readRowType, new FileReadTarget( DataFilePathFactory.formatIdentifier(file.fileName()), @@ -575,20 +517,6 @@ private RecordReader sequentialReadFiles( return ConcatRecordReader.create(readerSuppliers); } - private FormatReaderMapping vectorReaderMapping(DataFileMeta file, RowType readRowType) { - String formatIdentifier = DataFilePathFactory.formatIdentifier(file.fileName()); - TableSchema dataSchema = - schemaFetcher.apply(file.schemaId()).dataFileSchema(file.writeCols()); - List readFields = readRowType.getFields(); - boolean nestedFieldEnabled = nestedFieldEnabledFor(Collections.singletonList(file)); - List cacheKey = readerCacheKey(readFields, dataSchema.fields(), nestedFieldEnabled); - return formatReaderMappings.computeIfAbsent( - new FormatKey(file.schemaId(), formatIdentifier, cacheKey), - key -> - formatBuilder(readRowType, null, nestedFieldEnabled) - .build(formatIdentifier, schema, dataSchema, readFields, false)); - } - private static int findBlobFieldIndex(RowType rowType) { for (int i = 0; i < rowType.getFieldCount(); i++) { if (isBlobFileField(rowType.getTypeAt(i))) { @@ -1112,7 +1040,8 @@ public static List splitFieldBunches( boolean rowIdPushDown) { List fieldsFiles = new ArrayList<>(); Map blobBunchMap = new HashMap<>(); - Map vectorStoreBunchMap = new TreeMap<>(); + Map vectorStoreBunchMap = new TreeMap<>(); + long rowCount = -1; Range rowRange = null; for (DataFileMeta file : needMergeFiles) { if (isBlobFile(file.fileName())) { @@ -1125,17 +1054,20 @@ public static List splitFieldBunches( .add(file); } else if (isVectorStoreFile(file.fileName())) { RowType rowType = fileToRowType.apply(file); - final Range expectedRowRange = rowRange; - for (String column : file.writeCols()) { - int fieldId = rowType.getField(column).id(); - vectorStoreBunchMap - .computeIfAbsent( - fieldId, key -> new VectorFileBunch(fieldId, expectedRowRange)) - .add(file); - } + String fileFormat = DataFilePathFactory.formatIdentifier(file.fileName()); + VectorStoreBunchKey vectorStoreKey = + new VectorStoreBunchKey( + file.schemaId(), fileFormat, file.writeCols(), rowType); + final long expectedRowCount = rowCount; + vectorStoreBunchMap + .computeIfAbsent( + vectorStoreKey, + key -> new VectorFileBunch(expectedRowCount, rowIdPushDown)) + .add(file); } else { // Normal file, just add it to the current merge split fieldsFiles.add(new DataBunch(file)); + rowCount = file.rowCount(); rowRange = file.nonNullRowIdRange(); } } @@ -1156,9 +1088,6 @@ private static long bunchFirstRowId(FieldBunch bunch) { if (bunch instanceof BlobFileBunch) { return ((BlobFileBunch) bunch).logicalRange().from; } - if (bunch instanceof VectorFileBunch) { - return ((VectorFileBunch) bunch).selectedFiles().get(0).range.from; - } return bunch.files().get(0).nonNullFirstRowId(); } @@ -1222,10 +1151,7 @@ public long rowCount() { if (expectedRowRange != null) { for (Range range : merged) { Preconditions.checkState( - rowIdPushdown - ? range.hasIntersection(expectedRowRange) - : range.from >= expectedRowRange.from - && range.to <= expectedRowRange.to, + range.from >= expectedRowRange.from && range.to <= expectedRowRange.to, "Blob file range %s should be within normal file range %s.", range, expectedRowRange); @@ -1262,49 +1188,80 @@ public List files() { static class VectorFileBunch implements FieldBunch { final List files; - @Nullable final Range expectedRowRange; - final int fieldId; + final long expectedRowCount; + final boolean rowIdPushDown; - VectorFileBunch(int fieldId, @Nullable Range expectedRowRange) { + long latestFistRowId = -1; + long expectedNextFirstRowId = -1; + long latestMaxSequenceNumber = -1; + long rowCount; + + VectorFileBunch(long expectedRowCount, boolean rowIdPushDown) { this.files = new ArrayList<>(); - this.expectedRowRange = expectedRowRange; - this.fieldId = fieldId; + this.expectedRowCount = expectedRowCount; + this.rowIdPushDown = rowIdPushDown; } void add(DataFileMeta file) { - checkArgument( - isVectorStoreFile(file.fileName()), - "Only vector-store file can be added to this bunch."); - files.add(file); - } + if (!isVectorStoreFile(file.fileName())) { + throw new IllegalArgumentException( + "Only vector-store file can be added to this bunch."); + } + if (file.nonNullFirstRowId() == latestFistRowId) { + if (file.maxSequenceNumber() >= latestMaxSequenceNumber) { + throw new IllegalArgumentException( + "Vector file with same first row id should have decreasing sequence number."); + } + return; + } - List selectedFiles() { - List selected = new ArrayList<>(); - // A retained large vector file may be partly overwritten after its normal file was - // split. Preserve older prefixes and suffixes when these normal ranges merge again. - List newestFirst = new ArrayList<>(files); - newestFirst.sort(comparingLong(DataFileMeta::maxSequenceNumber).reversed()); - List covered = new ArrayList<>(); - for (DataFileMeta file : newestFirst) { - Range range = - expectedRowRange == null - ? file.nonNullRowIdRange() - : Range.intersection(file.nonNullRowIdRange(), expectedRowRange); - if (range == null) { - continue; + if (!files.isEmpty()) { + long firstRowId = file.nonNullFirstRowId(); + if (rowIdPushDown && firstRowId < expectedNextFirstRowId) { + if (file.maxSequenceNumber() > latestMaxSequenceNumber) { + DataFileMeta lastFile = files.remove(files.size() - 1); + rowCount -= lastFile.rowCount(); + } else { + return; + } + } else if (firstRowId < expectedNextFirstRowId) { + checkArgument( + file.maxSequenceNumber() < latestMaxSequenceNumber, + "Vector file with overlapping row id should have decreasing sequence number."); + return; + } else if (!rowIdPushDown && firstRowId > expectedNextFirstRowId) { + throw new IllegalArgumentException( + "Vector file first row id should be continuous, expect " + + expectedNextFirstRowId + + " but got " + + firstRowId); } - for (Range remaining : range.exclude(covered)) { - selected.add(new VectorFileRange(file, remaining)); + + if (!files.isEmpty()) { + checkArgument( + file.schemaId() == files.get(0).schemaId(), + "All files in this bunch should have the same schema id."); + checkArgument( + file.writeCols().equals(files.get(0).writeCols()), + "All files in this bunch should have the same write columns."); } - covered.add(range); } - selected.sort(comparingLong(file -> file.range.from)); - return selected; + + files.add(file); + rowCount += file.rowCount(); + if (expectedRowCount > 0) { + checkArgument( + rowCount <= expectedRowCount, + "Vector files row count exceed the expect " + expectedRowCount); + } + latestMaxSequenceNumber = file.maxSequenceNumber(); + latestFistRowId = file.nonNullFirstRowId(); + expectedNextFirstRowId = latestFistRowId + file.rowCount(); } @Override public long rowCount() { - return selectedFiles().stream().mapToLong(file -> file.range.count()).sum(); + return rowCount; } @Override @@ -1313,63 +1270,11 @@ public List files() { } } - @VisibleForTesting - static class VectorFileRange { - final DataFileMeta file; - final Range range; - - private VectorFileRange(DataFileMeta file, Range range) { - this.file = file; - this.range = range; - } - } - public static List> mergeRangesAndSort(List files) { // group by row id range ToLongFunction maxSeqF = DataFileMeta::maxSequenceNumber; RangeHelper rangeHelper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange); - List normalFiles = new ArrayList<>(); - List dedicatedFiles = new ArrayList<>(); - for (DataFileMeta file : files) { - if (isBlobFile(file.fileName()) || isVectorStoreFile(file.fileName())) { - dedicatedFiles.add(file); - } else { - normalFiles.add(file); - } - } - List> result = rangeHelper.mergeOverlappingRanges(normalFiles); - TreeMap> normalGroups = new TreeMap<>(); - for (List group : result) { - checkArgument( - rangeHelper.areAllRangesSame(group), - "Data files %s should be all row id ranges same.", - group); - normalGroups.put(group.get(0).nonNullFirstRowId(), group); - } - List unanchoredFiles = new ArrayList<>(); - for (DataFileMeta file : dedicatedFiles) { - Range range = file.nonNullRowIdRange(); - Map.Entry> entry = normalGroups.floorEntry(range.from); - if (entry == null) { - entry = normalGroups.ceilingEntry(range.from); - } - boolean attached = false; - while (entry != null && entry.getKey() <= range.to) { - List group = entry.getValue(); - if (group.get(0).nonNullRowIdRange().hasIntersection(range)) { - // Retain the physical range: readers need its original first row id to locate - // the selected rows in dedicated files shared by multiple normal groups. - group.add(file); - attached = true; - } - entry = normalGroups.higherEntry(entry.getKey()); - } - if (!attached) { - unanchoredFiles.add(file); - } - } - result.addAll(rangeHelper.mergeOverlappingRanges(unanchoredFiles)); - result.sort(comparingLong(group -> group.get(0).nonNullFirstRowId())); + List> result = rangeHelper.mergeOverlappingRanges(files); // in group, sort by blob/vector-store file and max_seq for (List group : result) { @@ -1400,12 +1305,8 @@ public static List> mergeRangesAndSort(List fil .thenComparing(reverseOrder(comparingLong(maxSeqF)))); // vector-store files sort by first row id then by reversed max sequence number - long normalFirstRowId = - dataFiles.isEmpty() ? Long.MIN_VALUE : dataFiles.get(0).nonNullFirstRowId(); vectorStoreFiles.sort( - comparingLong( - (DataFileMeta file) -> - Math.max(normalFirstRowId, file.nonNullFirstRowId())) + comparingLong(DataFileMeta::nonNullFirstRowId) .thenComparing(reverseOrder(comparingLong(maxSeqF)))); // concat data files, blob files, vector-store files @@ -1417,4 +1318,97 @@ public static List> mergeRangesAndSort(List fil return result; } + + static final class VectorStoreBunchKey implements Comparable { + public final long schemaId; + public final String formatIdentifier; + public final List writeCols; + + public VectorStoreBunchKey( + long schemaId, + String formatIdentifier, + List writeCols, + RowType preferredColOrder) { + this.schemaId = schemaId; + this.formatIdentifier = checkNotNull(formatIdentifier, "formatIdentifier"); + this.writeCols = normalizeWriteCols(writeCols, preferredColOrder); + } + + @Override + public int compareTo(VectorStoreBunchKey o) { + int c = Long.compare(this.schemaId, o.schemaId); + if (c != 0) { + return c; + } + + c = this.formatIdentifier.compareTo(o.formatIdentifier); + if (c != 0) { + return c; + } + + int n = Math.min(this.writeCols.size(), o.writeCols.size()); + for (int i = 0; i < n; i++) { + c = this.writeCols.get(i).compareTo(o.writeCols.get(i)); + if (c != 0) { + return c; + } + } + return Integer.compare(this.writeCols.size(), o.writeCols.size()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof VectorStoreBunchKey)) { + return false; + } + VectorStoreBunchKey that = (VectorStoreBunchKey) o; + return schemaId == that.schemaId + && formatIdentifier.equals(that.formatIdentifier) + && writeCols.equals(that.writeCols); + } + + @Override + public int hashCode() { + return Objects.hash(schemaId, formatIdentifier, writeCols); + } + + @Override + public String toString() { + return "VectorStoreBunchKey{schemaId=" + + schemaId + + ", format=" + + formatIdentifier + + ", writeCols=" + + writeCols + + "}"; + } + + private static List normalizeWriteCols(List writeCols, RowType rowType) { + if (writeCols == null || writeCols.isEmpty()) { + return Collections.emptyList(); + } + + Map colPosMap = new HashMap<>(); + List namesInRowType = rowType.getFieldNames(); + for (int i = 0; i < namesInRowType.size(); i++) { + colPosMap.putIfAbsent(namesInRowType.get(i), i); + } + + ArrayList sorted = new ArrayList<>(writeCols); + sorted.sort( + (a, b) -> { + int ia = colPosMap.getOrDefault(a, Integer.MAX_VALUE); + int ib = colPosMap.getOrDefault(b, Integer.MAX_VALUE); + if (ia != ib) { + return Integer.compare(ia, ib); + } + return a.compareTo(b); + }); + + return sorted; + } + } } diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java index 6434b78cbb11..056c6b8dca86 100644 --- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java +++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java @@ -249,7 +249,7 @@ protected Optional checkTableSpecificConflicts( return exception; } - exception = checkRowIdRangeConflicts(commitKind, baseEntries, deltaEntries, mergedEntries); + exception = checkRowIdRangeConflicts(commitKind, mergedEntries); if (exception.isPresent()) { return exception; } @@ -264,10 +264,7 @@ protected Optional checkTableSpecificConflicts( } private Optional checkRowIdRangeConflicts( - CommitKind commitKind, - List baseEntries, - List deltaEntries, - Collection mergedEntries) { + CommitKind commitKind, Collection mergedEntries) { if (rowIdCheckFromSnapshot == null && commitKind != CommitKind.COMPACT) { return Optional.empty(); } @@ -294,8 +291,7 @@ private Optional checkRowIdRangeConflicts( entries.stream() .filter(file -> dedicatedStorageFile(file.fileName())) .collect(Collectors.toList()); - return checkDedicatedFileRowIdRangeConflicts( - commitKind, baseEntries, deltaEntries, dataFiles, dedicatedFiles); + return checkDedicatedFileRowIdRangeConflicts(dataFiles, dedicatedFiles); } private Optional checkDataFileRowIdRangeConflicts( @@ -314,54 +310,18 @@ private Optional checkDataFileRowIdRangeConflicts( } private Optional checkDedicatedFileRowIdRangeConflicts( - CommitKind commitKind, - List baseEntries, - List deltaEntries, - List dataFiles, - List dedicatedFiles) { + List dataFiles, List dedicatedFiles) { if (dedicatedFiles.isEmpty()) { return Optional.empty(); } RowRangeIndex dataFileRowRangeIndex = rowRangeIndex(dataFiles, false); - RowRangeIndex contiguousDataFileRowRangeIndex = rowRangeIndex(dataFiles, true); - RowRangeIndex baseDataFileRowRangeIndex = - rowRangeIndex( - baseEntries.stream() - .filter(file -> file.firstRowId() != null) - .filter(file -> !dedicatedStorageFile(file.fileName())) - .collect(Collectors.toList()), - false); - Set addedFiles = - deltaEntries.stream() - .filter(file -> file.kind() == FileKind.ADD) - .map(FileEntry::identifier) - .collect(Collectors.toSet()); for (SimpleFileEntry dedicatedFile : dedicatedFiles) { Range dedicatedRange = dedicatedFile.nonNullRowIdRange(); if (dataFileRowRangeIndex.contains(dedicatedRange)) { continue; } - if (!addedFiles.contains(dedicatedFile.identifier())) { - // Normal-file compaction can change boundaries without rewriting dedicated files. - // A row-range scan may contain only part of an existing dedicated file, so check - // that all of its previously visible normal-file coverage remains present. - List previouslyCoveredRanges = - baseDataFileRowRangeIndex.intersectedRanges( - dedicatedRange.from, dedicatedRange.to); - if (!previouslyCoveredRanges.isEmpty() - && previouslyCoveredRanges.stream() - .allMatch(contiguousDataFileRowRangeIndex::contains)) { - continue; - } - } else if (commitKind == CommitKind.COMPACT - && contiguousDataFileRowRangeIndex.contains(dedicatedRange)) { - // Dedicated compaction may merge files across the new normal-file boundaries. - // New DML files must still fit one range to reject stale MERGE INTO writers. - continue; - } - List intersectingRanges = dataFileRowRangeIndex.intersectedRanges(dedicatedRange.from, dedicatedRange.to); List intersectingDataFiles = diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java index 3adc53ffc1fa..88bf60f019c8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/DataSplit.java @@ -40,7 +40,6 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.utils.FunctionWithIOException; import org.apache.paimon.utils.InternalRowUtils; -import org.apache.paimon.utils.Range; import org.apache.paimon.utils.RangeHelper; import org.apache.paimon.utils.SerializationUtils; @@ -57,9 +56,7 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; import static org.apache.paimon.io.DataFilePathFactory.INDEX_PATH_SUFFIX; -import static org.apache.paimon.types.VectorType.isVectorStoreFile; import static org.apache.paimon.utils.Preconditions.checkArgument; /** Input splits. Needed by most batch computation engines. */ @@ -191,19 +188,13 @@ private boolean dataEvolutionRowCountAvailable() { private long dataEvolutionMergedRowCount() { long sum = 0L; RangeHelper rangeHelper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange); - for (List group : rangeHelper.mergeOverlappingRanges(dataFiles)) { - List ranges = - group.stream() - .filter( - f -> - !isBlobFile(f.fileName()) - && !isVectorStoreFile(f.fileName())) - .map(DataFileMeta::nonNullRowIdRange) - .collect(Collectors.toList()); - if (ranges.isEmpty()) { - group.stream().map(DataFileMeta::nonNullRowIdRange).forEach(ranges::add); + List> ranges = rangeHelper.mergeOverlappingRanges(dataFiles); + for (List group : ranges) { + long maxCount = 0; + for (DataFileMeta file : group) { + maxCount = Math.max(maxCount, file.rowCount()); } - sum += Range.sortAndMergeOverlap(ranges, false).stream().mapToLong(Range::count).sum(); + sum += maxCount; } if (dataDeletionFiles != null) { for (DeletionFile deletionFile : dataDeletionFiles) { diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java index 3b2464d311d3..4cb411f7a870 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java @@ -171,6 +171,27 @@ public void testSingleWrite() throws Exception { .isEqualTo(CoreOptions.FILE_FORMAT_AVRO); } + @Test + public void testFileRollingPredicate() throws Exception { + AppendOnlyWriter writer = + createEmptyWriter(64) + .withFileRollingPredicate(count -> count == 1250 || count == 2300); + for (int i = 0; i < 2500; i++) { + writer.write(row(i, "value", PART)); + } + CommitIncrement increment = writer.prepareCommit(true); + writer.close(); + + List files = increment.newFilesIncrement().newFiles(); + assertThat(files).extracting(DataFileMeta::rowCount).containsExactly(1250L, 1050L, 200L); + assertThat(files) + .extracting(DataFileMeta::minSequenceNumber) + .containsExactly(0L, 1250L, 2300L); + assertThat(files) + .extracting(DataFileMeta::maxSequenceNumber) + .containsExactly(1249L, 2299L, 2499L); + } + @Test public void testBinaryColumnStatsRoundTrip() throws Exception { RowType binarySchema = diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java index 5d553f1a2507..b9475b207673 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java @@ -45,6 +45,7 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.CloseableIterator; +import org.apache.paimon.utils.Range; import org.apache.paimon.utils.SnapshotManager; import org.junit.jupiter.api.Test; @@ -215,6 +216,22 @@ public void testSplitLargeFilesUsesPhysicalSizeAndIncludesColumnUpdates() { } } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testDoNotSplitInsideDedicatedFile(boolean vector) { + List entries = + Arrays.asList( + makeEntryWithSize("large.parquet", 0L, 10L, 0, 1000L), + vector + ? makeVectorStoreEntry("whole.vector.lance", 0L, 10L, 10L) + : makeBlobEntry("whole.blob", 0L, 10L, 10L)); + DataEvolutionCompactCoordinator.CompactPlanner planner = + new DataEvolutionCompactCoordinator.CompactPlanner( + false, false, 200L, 100L, 100L, 1L, 2L, schemaId -> null, null); + + assertThat(planner.compactPlan(entries)).isEmpty(); + } + @Test public void testSplitLargeFilesAndMergeSmallFilesKeepDedicatedFiles() { List entries = @@ -222,8 +239,8 @@ public void testSplitLargeFilesAndMergeSmallFilesKeepDedicatedFiles() { makeEntryWithSize("large.parquet", 0L, 10L, 0, 201L), makeEntryWithSize("small1.parquet", 10L, 10L, 0, 20L), makeEntryWithSize("small2.parquet", 20L, 10L, 0, 20L), - makeBlobEntry("original.blob", 0L, 30L, 1000L), - makeVectorStoreEntry("original.vector.lance", 0L, 30L, 1000L)); + makeBlobEntry("original.blob", 0L, 5L, 1000L), + makeVectorStoreEntry("original.vector.lance", 5L, 5L, 1000L)); DataEvolutionCompactCoordinator.CompactPlanner planner = new DataEvolutionCompactCoordinator.CompactPlanner( false, false, 200L, 100L, 100L, 1L, 2L, schemaId -> null, null); @@ -1045,7 +1062,11 @@ public void testSerializerBasic() throws IOException { createDataFileMeta("file2.parquet", 100L, 100L, 0, 1024)); DataEvolutionCompactTask task = - new DataEvolutionNormalCompactTask(BinaryRow.EMPTY_ROW, files); + new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + files, + Arrays.asList( + new Range(0L, 49L), new Range(50L, 149L), new Range(150L, 199L))); byte[] bytes = serializer.serialize(task); DataEvolutionCompactTask deserialized = diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java index 10560e0e6297..38518d3a8572 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java @@ -246,124 +246,365 @@ public void testSplitHistoricalLargeFile(boolean updateColumn) throws Exception } @Test - public void testSplitRetainsMultipleBlobColumnsAndVectorFiles() throws Exception { - Schema schema = + public void testSplitAtBlobBoundariesRetainsDedicatedFiles() throws Exception { + FileStoreTable table = createBlobSegmentsTable(); + List dedicated = dedicatedFiles(table); + assertThat(dedicated).hasSize(3); + Map options = new HashMap<>(); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + DataEvolutionCompactTask merged = compactSingleTask(table.copy(options)); + assertThat(merged.compactAfter()).hasSize(1); + assertThat(merged.compactAfter().get(0).nonNullRowIdRange()).isEqualTo(new Range(0, 3749)); + + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "10"); + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); + options.put(CoreOptions.WRITE_BUFFER_FOR_APPEND.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + table = table.copy(options); + DataEvolutionCompactTask split = compactSingleTask(table); + // Size rolling is checked after 1000 rows. It must wait another 250 rows for the BLOB end. + assertThat(split.compactAfter().stream().map(DataFileMeta::nonNullRowIdRange)) + .containsExactly(new Range(0, 1249), new Range(1250, 2499), new Range(2500, 3749)); + assertDedicatedFilesContained(table, dedicated); + assertBlobValues(table); + // Each resulting normal range is entirely protected, so another size-only pass is useless. + assertThat( + new DataEvolutionCompactCoordinator( + table, + false, + false, + table.snapshotManager().latestSnapshot()) + .plan()) + .isEmpty(); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testNormalSplitRespectsBlobCompactionOutputRange(boolean mergeNormalVersions) + throws Exception { + FileStoreTable table = createBlobSegmentsTable(); + List originalBlobs = dedicatedFiles(table); + assertThat(originalBlobs).hasSize(3); + Map options = new HashMap<>(); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + DataEvolutionCompactTask initialMerge = compactSingleTask(table.copy(options)); + assertThat(initialMerge.compactAfter()).hasSize(1); + DataFileMeta originalNormal = initialMerge.compactAfter().get(0); + if (mergeNormalVersions) { + writeProjectedRange(table, "id", 0, 3750, 0, true); + } + + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + table = table.copy(options); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + List tasks = + new DataEvolutionCompactCoordinator(table, true, false, snapshot).plan(); + if (mergeNormalVersions) { + assertThat(tasks) + .extracting(DataEvolutionCompactTask::type) + .containsExactly( + DataEvolutionCompactTask.TaskType.NORMAL, + DataEvolutionCompactTask.TaskType.BLOB); + } else { + assertThat(tasks) + .extracting(DataEvolutionCompactTask::type) + .containsExactly(DataEvolutionCompactTask.TaskType.BLOB); + } + DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer(); + List messages = new ArrayList<>(); + List restored = new ArrayList<>(); + for (DataEvolutionCompactTask planned : tasks) { + DataEvolutionCompactTask task = + serializer.deserialize(serializer.getVersion(), serializer.serialize(planned)); + restored.add(task); + messages.add(task.doCompact(table, "compact-normal-and-blob")); + } + if (mergeNormalVersions) { + assertThat(restored.get(0).compactBefore()).hasSize(2); + } + assertThat(restored.get(restored.size() - 1).compactBefore()) + .containsExactlyInAnyOrderElementsOf(originalBlobs); + // The old BLOB boundaries are safe individually, but the planned BLOB merge removes them. + // Any normal rewrite must therefore retain the planned BLOB output's complete range. + for (DataEvolutionCompactTask task : restored) { + assertThat(task.compactAfter()).hasSize(1); + assertThat(task.compactAfter().get(0).nonNullRowIdRange()) + .isEqualTo(new Range(0, 3749)); + } + messages.addAll( + new DataEvolutionCompactionCommitPreparation(table, snapshot).prepare(messages)); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(messages); + } + if (!mergeNormalVersions) { + assertThat( + table.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .filter( + file -> + !isBlobFile(file.fileName()) + && !isVectorStoreFile(file.fileName()))) + .containsExactly(originalNormal); + } + List compactedBlobs = dedicatedFiles(table); + assertThat(compactedBlobs).hasSize(1).doesNotContainAnyElementsOf(originalBlobs); + assertDedicatedFilesContained(table, compactedBlobs); + assertBlobValues(table); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testOverlappingDedicatedRangesPreventSplit(boolean sameColumn) throws Exception { + catalog.createTable( + identifier(), Schema.newBuilder() .column("id", DataTypes.INT()) .column("b1", DataTypes.BLOB()) .column("b2", DataTypes.BLOB()) - .column("v", DataTypes.VECTOR(2, DataTypes.FLOAT())) .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") - .option(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "128 kb") - .option(CoreOptions.VECTOR_TARGET_FILE_SIZE.key(), "128 kb") + .option(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "128 mb") + .build(), + false); + FileStoreTable table = getTableDefault(); + writeProjectedRange(table, "id", 0, 3750, 0, false); + writeProjectedRange(table, "b1", 0, 1250, 0, true); + writeProjectedRange(table, "b1", 1250, 2500, 0, true); + String overlappingColumn = sameColumn ? "b1" : "b2"; + writeProjectedRange(table, overlappingColumn, 0, 2500, 1, true); + writeProjectedRange(table, overlappingColumn, 2500, 1250, 1, true); + List dedicated = dedicatedFiles(table); + assertThat(dedicated).hasSize(4); + assertThat(dedicated).allSatisfy(file -> assertThat(file.rowCount()).isLessThan(3750)); + Map options = new HashMap<>(); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "10"); + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + FileStoreTable splitTable = table.copy(options); + assertThat( + new DataEvolutionCompactCoordinator( + splitTable, + false, + false, + table.snapshotManager().latestSnapshot()) + .plan()) + .isEmpty(); + + // Ordinary version merging remains useful even though no dedicated-safe split is possible. + writeProjectedRange(table, "id", 0, 3750, 0, true); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + DataEvolutionCompactTask merged = compactSingleTask(table.copy(options)); + assertThat(merged.compactBefore()).hasSize(2); + assertThat(merged.compactAfter()).hasSize(1); + assertThat(merged.compactAfter().get(0).nonNullRowIdRange()).isEqualTo(new Range(0, 3749)); + assertDedicatedFilesContained(table, dedicated); + ReadBuilder readBuilder = table.newReadBuilder(); + List ids = new ArrayList<>(); + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan())) { + reader.forEachRemaining( + row -> { + int id = row.getInt(0); + ids.add(id); + assertThat(row.getBlob(1).toData()) + .containsExactly((byte) (id + (sameColumn ? 1 : 0))); + if (sameColumn) { + assertThat(row.isNullAt(2)).isTrue(); + } else { + assertThat(row.getBlob(2).toData()).containsExactly((byte) (id + 1)); + } + }); + } + assertThat(ids) + .containsExactlyElementsOf( + java.util.stream.IntStream.range(0, 3750) + .boxed() + .collect(Collectors.toList())); + } + + @Test + public void testFullRangeVectorPreventsSplit() throws Exception { + catalog.createTable( + identifier(), + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("vector", DataTypes.VECTOR(2, DataTypes.FLOAT())) + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.VECTOR_TARGET_FILE_SIZE.key(), "128 mb") .option(CoreOptions.VECTOR_FILE_FORMAT.key(), "json") .option(CoreOptions.FILE_COMPRESSION.key(), "none") - .build(); - catalog.createTable(identifier(), schema, false); + .build(), + false); FileStoreTable table = getTableDefault(); BatchWriteBuilder builder = table.newBatchWriteBuilder(); try (BatchTableWrite write = builder.newWrite(); BatchTableCommit commit = builder.newCommit()) { for (int i = 0; i < 2500; i++) { write.write( - GenericRow.of( - i, - new BlobData(new byte[] {(byte) i}), - new BlobData(new byte[] {(byte) (i + 1)}), - BinaryVector.fromPrimitiveArray(new float[] {i, i + 1}))); + GenericRow.of(i, BinaryVector.fromPrimitiveArray(new float[] {i, i + 1}))); } commit.commit(write.prepareCommit()); } catalog.alterTable( identifier(), - Collections.singletonList(SchemaChange.renameColumn("v", "renamed_v")), + Collections.singletonList(SchemaChange.renameColumn("vector", "renamed_vector")), false); table = getTableDefault(); - List dedicatedFiles = - table.store().newScan().plan().files().stream() - .map(ManifestEntry::file) - .filter( - file -> - isBlobFile(file.fileName()) - || isVectorStoreFile(file.fileName())) - .collect(Collectors.toList()); - assertThat(dedicatedFiles).hasSize(3); + List dedicated = dedicatedFiles(table); + assertThat(dedicated).hasSize(1); + assertThat(dedicated.get(0).nonNullRowIdRange()).isEqualTo(new Range(0, 2499)); Map options = new HashMap<>(); options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); - table = table.copy(options); - List tasks = - new DataEvolutionCompactCoordinator( - table, false, false, table.snapshotManager().latestSnapshot()) - .plan(); - assertThat(tasks).hasSize(1); - DataEvolutionCompactTask task = tasks.get(0); - assertThat(task.compactBefore()).hasSize(1); - try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { - commit.commit(Collections.singletonList(task.doCompact(table, "split-dedicated"))); + assertThat( + new DataEvolutionCompactCoordinator( + table.copy(options), + false, + false, + table.snapshotManager().latestSnapshot()) + .plan()) + .isEmpty(); + + writeProjectedRange(table, "id", 0, 2500, 0, true); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + DataEvolutionCompactTask merged = compactSingleTask(table.copy(options)); + assertThat(merged.compactAfter()).hasSize(1); + assertDedicatedFilesContained(table, dedicated); + ReadBuilder readBuilder = table.newReadBuilder(); + List ids = new ArrayList<>(); + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan())) { + reader.forEachRemaining( + row -> { + int id = row.getInt(0); + ids.add(id); + assertThat(row.getVector(1).toFloatArray()).containsExactly(id, id + 1); + }); } - List normalRanges = - task.compactAfter().stream() - .filter( - file -> - !isBlobFile(file.fileName()) - && !isVectorStoreFile(file.fileName())) - .map(DataFileMeta::nonNullRowIdRange) - .collect(Collectors.toList()); - assertThat(normalRanges.size()).isGreaterThan(1); - assertThat(task.compactAfter()) + assertThat(ids) + .containsExactlyElementsOf( + java.util.stream.IntStream.range(0, 2500) + .boxed() + .collect(Collectors.toList())); + } + + @Test + public void testCompletedParquetOutputsDoNotRepeatSmallFileCompaction() throws Exception { + catalog.createTable( + identifier(), + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("payload", DataTypes.STRING()) + .column("blob", DataTypes.BLOB()) + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.FILE_FORMAT.key(), "parquet") + .option(CoreOptions.FILE_COMPRESSION.key(), "snappy") + .option(CoreOptions.TARGET_FILE_SIZE.key(), "8 mb") + .option(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "128 mb") + .option(CoreOptions.SOURCE_SPLIT_OPEN_FILE_COST.key(), "4 mb") + .option(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2") + .option( + CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), + "true") + .option("parquet.enable.dictionary", "false") + .option("parquet.page.size", String.valueOf(16 * 1024 * 1024)) + .build(), + false); + FileStoreTable table = getTableDefault(); + char[] payloadChars = new char[10 * 1024]; + Arrays.fill(payloadChars, 'a'); + BinaryString payload = BinaryString.fromString(new String(payloadChars)); + for (int batch = 0; batch < 3; batch++) { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + for (int i = batch * 1000; i < (batch + 1) * 1000; i++) { + write.write(GenericRow.of(i, payload, new BlobData(new byte[] {(byte) i}))); + } + commit.commit(write.prepareCommit()); + } + } + List dedicated = dedicatedFiles(table); + assertThat(dedicated).hasSize(3); + + DataEvolutionCompactTask compacted = compactSingleTask(table); + assertThat(compacted.compactBefore()).hasSize(3); + List outputs = compacted.compactAfter(); + // Parquet reaches the byte target using an uncompressed page buffer at each BLOB end. + // Closing compresses the page, so all three outputs remain small enough to merge again. + assertThat(outputs).extracting(DataFileMeta::rowCount).containsExactly(1000L, 1000L, 1000L); + assertThat(outputs) .allSatisfy( file -> - assertThat( - isBlobFile(file.fileName()) - || isVectorStoreFile(file.fileName())) - .isFalse()); + assertThat(file.fileSize()) + .isLessThan(table.coreOptions().splitOpenFileCost())); + assertDedicatedFilesContained(table, dedicated); + + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + List repeated = + new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan(); + assertThat(repeated).hasSize(1); + assertThat(repeated.get(0).compactBefore()).containsExactlyInAnyOrderElementsOf(outputs); assertThat( - table.store().newScan().plan().files().stream() - .map(ManifestEntry::file) - .filter( - file -> - isBlobFile(file.fileName()) - || isVectorStoreFile(file.fileName())) - .collect(Collectors.toList())) - .containsExactlyInAnyOrderElementsOf(dedicatedFiles); - - // Reading the new layout does not depend on the compaction option remaining enabled. - options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "false"); - table = table.copy(options); - assertDedicatedValues(table); + new DataEvolutionCompactCoordinator(table, false, false, snapshot) + .withCompletedNormalFiles( + outputs.stream() + .map(DataFileMeta::fileName) + .collect(Collectors.toSet())) + .plan()) + .isEmpty(); - // Merging split normal files must also leave spanning dedicated files untouched. - options.put(CoreOptions.TARGET_FILE_SIZE.key(), "128 mb"); - options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); - table = table.copy(options); - tasks = - new DataEvolutionCompactCoordinator( - table, false, false, table.snapshotManager().latestSnapshot()) - .plan(); - assertThat(tasks).hasSize(1); - task = tasks.get(0); - assertThat(task.compactBefore()).hasSize(normalRanges.size()); - try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { - commit.commit(Collections.singletonList(task.doCompact(table, "merge-split-normal"))); + List ids = new ArrayList<>(); + ReadBuilder readBuilder = table.newReadBuilder(); + try (RecordReader reader = + readBuilder.newRead().createReader(readBuilder.newScan().plan())) { + reader.forEachRemaining( + row -> { + int id = row.getInt(0); + ids.add(id); + assertThat(row.getString(1)).isEqualTo(payload); + assertThat(row.getBlob(2).toData()).containsExactly((byte) id); + }); } - assertThat(task.compactAfter()).hasSize(1); - assertThat( - table.store().newScan().plan().files().stream() - .map(ManifestEntry::file) - .filter( - file -> - isBlobFile(file.fileName()) - || isVectorStoreFile(file.fileName())) - .collect(Collectors.toList())) - .containsExactlyInAnyOrderElementsOf(dedicatedFiles); - assertDedicatedValues(table); + assertThat(ids) + .containsExactlyElementsOf( + java.util.stream.IntStream.range(0, 3000) + .boxed() + .collect(Collectors.toList())); } - private void assertDedicatedValues(FileStoreTable table) throws Exception { - ReadBuilder readBuilder = table.newReadBuilder(); + private FileStoreTable createBlobSegmentsTable() throws Exception { + catalog.createTable( + identifier(), + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("blob", DataTypes.BLOB()) + .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") + .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") + .option(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "128 mb") + .build(), + false); + FileStoreTable table = getTableDefault(); + for (int batch = 0; batch < 3; batch++) { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + for (int i = batch * 1250; i < (batch + 1) * 1250; i++) { + write.write(GenericRow.of(i, new BlobData(new byte[] {(byte) i}))); + } + commit.commit(write.prepareCommit()); + } + } + return table; + } + + private void assertBlobValues(FileStoreTable table) throws Exception { List ids = new ArrayList<>(); + ReadBuilder readBuilder = table.newReadBuilder(); try (RecordReader reader = readBuilder.newRead().createReader(readBuilder.newScan().plan())) { reader.forEachRemaining( @@ -371,17 +612,90 @@ private void assertDedicatedValues(FileStoreTable table) throws Exception { int id = row.getInt(0); ids.add(id); assertThat(row.getBlob(1).toData()).containsExactly((byte) id); - assertThat(row.getBlob(2).toData()).containsExactly((byte) (id + 1)); - assertThat(row.getVector(3).toFloatArray()).containsExactly(id, id + 1); }); } assertThat(ids) .containsExactlyElementsOf( - java.util.stream.IntStream.range(0, 2500) + java.util.stream.IntStream.range(0, 3750) .boxed() .collect(Collectors.toList())); } + private void writeProjectedRange( + FileStoreTable table, + String column, + int from, + int count, + int valueOffset, + boolean existingRows) + throws Exception { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = + builder.newWrite().withWriteType(table.rowType().project(column)); + BatchTableCommit commit = builder.newCommit()) { + for (int i = from; i < from + count; i++) { + write.write( + GenericRow.of( + "id".equals(column) + ? i + : new BlobData(new byte[] {(byte) (i + valueOffset)}))); + } + List messages = write.prepareCommit(); + if (existingRows) { + assignFirstRowId(messages, from); + } + commit.commit(messages); + } + } + + private DataEvolutionCompactTask compactSingleTask(FileStoreTable table) throws Exception { + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + List tasks = + new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan(); + assertThat(tasks).hasSize(1); + DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer(); + DataEvolutionCompactTask task = + serializer.deserialize(serializer.getVersion(), serializer.serialize(tasks.get(0))); + List messages = new ArrayList<>(); + messages.add(task.doCompact(table, "compact-dedicated-boundaries")); + messages.addAll( + new DataEvolutionCompactionCommitPreparation(table, snapshot).prepare(messages)); + try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) { + commit.commit(messages); + } + return task; + } + + private List dedicatedFiles(FileStoreTable table) { + return table.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .filter(file -> isBlobFile(file.fileName()) || isVectorStoreFile(file.fileName())) + .collect(Collectors.toList()); + } + + private void assertDedicatedFilesContained(FileStoreTable table, List expected) { + assertThat(dedicatedFiles(table)).containsExactlyInAnyOrderElementsOf(expected); + List normalRanges = + table.store().newScan().plan().files().stream() + .map(ManifestEntry::file) + .filter( + file -> + !isBlobFile(file.fileName()) + && !isVectorStoreFile(file.fileName())) + .map(DataFileMeta::nonNullRowIdRange) + .collect(Collectors.toList()); + for (DataFileMeta file : expected) { + Range dedicated = file.nonNullRowIdRange(); + assertThat( + normalRanges.stream() + .anyMatch( + normal -> + normal.from <= dedicated.from + && normal.to >= dedicated.to)) + .isTrue(); + } + } + private void write() throws Exception { createTableDefault(); diff --git a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java index 1676d7ebd42b..5d5bb6e5da2e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java @@ -171,6 +171,104 @@ public void testRollingByRowsWithBundle() throws IOException { assertThat(files.get(2).rowCount()).isEqualTo(30); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testRollingAtSafeBoundaries(boolean bundled) throws IOException { + RowDataRollingFileWriter writer = + createRowDataWriter(Long.MAX_VALUE, 3) + .withFileRollingPredicate(count -> count == 5 || count == 6 || count == 9); + writeRows(writer, 12, bundled); + writer.close(); + + // Boundary 6 must not roll a fresh file below target; boundary 9 uses the cumulative + // count, and close retains the final short file. + assertThat(writer.result()).extracting(DataFileMeta::rowCount).containsExactly(5L, 4L, 3L); + assertWrittenRows(writer.result(), 12); + } + + @Test + public void testSizeRollingAtSafeBoundariesBetweenChecks() throws IOException { + RowDataRollingFileWriter writer = + createRowDataWriter(TARGET_FILE_SIZE, Long.MAX_VALUE) + .withFileRollingPredicate(count -> count == 1250 || count == 2300); + writeRows(writer, 2500, false); + writer.close(); + + // Avro checks size at rows 1000 and 2000. A pending roll must honor the next safe + // boundary without waiting for another size check. + assertThat(writer.result()) + .extracting(DataFileMeta::rowCount) + .containsExactly(1250L, 1050L, 200L); + assertWrittenRows(writer.result(), 2500); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testRowDataRollingWithoutPredicate(boolean bundled) throws IOException { + RowDataRollingFileWriter writer = createRowDataWriter(Long.MAX_VALUE, 3); + writeRows(writer, 12, bundled); + writer.close(); + + assertThat(writer.result()) + .extracting(DataFileMeta::rowCount) + .containsExactly(bundled ? new Long[] {12L} : new Long[] {3L, 3L, 3L, 3L}); + assertWrittenRows(writer.result(), 12); + } + + private RowDataRollingFileWriter createRowDataWriter( + long targetFileSize, long targetFileRowNum) { + return new RowDataRollingFileWriter( + LocalFileIO.create(), + 0L, + FileFormat.fromIdentifier("avro", new Options()), + targetFileSize, + SCHEMA, + new DataFilePathFactory( + new Path(tempDir + "/bucket-0"), + "avro", + CoreOptions.DATA_FILE_PREFIX.defaultValue(), + CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), + CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), + CoreOptions.FILE_COMPRESSION.defaultValue(), + null), + () -> new LongCounter(0), + CoreOptions.FILE_COMPRESSION.defaultValue(), + SimpleColStatsCollector.createFullStatsFactories(SCHEMA.getFieldCount()), + new FileIndexOptions(), + FileSource.APPEND, + true, + false, + null, + null, + targetFileRowNum); + } + + private static void writeRows(RowDataRollingFileWriter writer, int count, boolean bundled) + throws IOException { + if (bundled) { + writer.writeBundle(bundle(count)); + } else { + for (int i = 0; i < count; i++) { + writer.write(GenericRow.of(i)); + } + } + } + + private void assertWrittenRows(List files, int count) throws IOException { + List actual = new ArrayList<>(); + for (DataFileMeta file : files) { + actual.addAll( + readIntsFromRowFile( + FileFormat.fromIdentifier("avro", new Options()), + new Path(tempDir + "/bucket-0/" + file.fileName()))); + } + List expected = new ArrayList<>(); + for (int i = 0; i < count; i++) { + expected.add(i); + } + assertThat(actual).containsExactlyElementsOf(expected); + } + private static SingleUseBundleRecords bundle(int rowCount) { List rows = new ArrayList<>(); for (int i = 0; i < rowCount; i++) { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java index f7ca2e207940..1d2c9b654c48 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionFileStoreScanTest.java @@ -663,77 +663,6 @@ public void testIntersectsRowRanges() { assertThat(index.intersects(100, 200)).isFalse(); } - @Test - public void testDedicatedProjectionKeepsEveryNormalDeletionVectorAnchor() { - DataEvolutionFileStoreScan scan = pruningScan(true, "f1"); - ManifestEntry first = pruningEntry("first.parquet", "f0", 0, 5, 0); - ManifestEntry second = pruningEntry("second.parquet", "f0", 5, 5, 0); - ManifestEntry update = pruningEntry("update.parquet", "f0", 0, 5, 1); - ManifestEntry blob = pruningEntry("spanning.blob", "f1", 0, 10, 0); - - assertThat(scan.postFilterManifestEntries(Arrays.asList(first, second, update, blob))) - .containsExactlyInAnyOrder(first, second, blob); - } - - @Test - public void testMissingColumnProjectionKeepsEverySplitNormalRange() { - DataEvolutionFileStoreScan scan = pruningScan(false, "f2"); - ManifestEntry first = pruningEntry("first.parquet", "f0", 0, 5, 0); - ManifestEntry second = pruningEntry("second.parquet", "f0", 5, 5, 0); - ManifestEntry blob = pruningEntry("spanning.blob", "f1", 0, 10, 0); - - assertThat(scan.postFilterManifestEntries(Arrays.asList(first, second, blob))) - .containsExactlyInAnyOrder(first, second); - } - - @Test - public void testPartialColumnProjectionKeepsMissingSplitNormalRange() { - DataEvolutionFileStoreScan scan = pruningScan(false, "f2"); - ManifestEntry first = pruningEntry("first.parquet", "f0", 0, 5, 0); - ManifestEntry second = pruningEntry("second.parquet", "f0", 5, 5, 0); - ManifestEntry update = pruningEntry("update.parquet", "f2", 0, 5, 1); - ManifestEntry vector = pruningEntry("spanning.vector.json", "f1", 0, 10, 0); - - assertThat(scan.postFilterManifestEntries(Arrays.asList(first, second, update, vector))) - .containsExactlyInAnyOrder(update, second); - } - - private DataEvolutionFileStoreScan pruningScan(boolean deletionVectorsEnabled, String field) { - TableSchema schema = TableSchema.create(0L, createSchema("f0", "f1", "f2")); - DataEvolutionFileStoreScan scan = - new DataEvolutionFileStoreScan( - null, null, null, null, schema, null, null, deletionVectorsEnabled); - scan.withReadType(schema.logicalRowType().project(Collections.singletonList(field))); - return scan; - } - - private ManifestEntry pruningEntry( - String fileName, String column, long firstRowId, long rowCount, long sequence) { - return ManifestEntry.create( - FileKind.ADD, - createBinaryRow(0), - 0, - 0, - DataFileMeta.create( - fileName, - 100L, - rowCount, - BinaryRow.EMPTY_ROW, - BinaryRow.EMPTY_ROW, - SimpleStats.EMPTY_STATS, - SimpleStats.EMPTY_STATS, - sequence, - sequence, - 0L, - 0, - null, - null, - FileSource.APPEND, - null, - firstRowId, - Collections.singletonList(column))); - } - private Schema createSchema(String... fieldNames) { Schema.Builder builder = Schema.newBuilder(); for (int i = 0; i < fieldNames.length; i++) { diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java index ce8d7aa6bcc6..748dbe7bdfd6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionReadTest.java @@ -51,7 +51,7 @@ public class DataEvolutionReadTest { @BeforeEach public void setUp() { - vectorBunch = new VectorFileBunch(0, null); + vectorBunch = new VectorFileBunch(Long.MAX_VALUE, false); } @Test @@ -94,69 +94,81 @@ public void testAddNonVectorFileThrowsException() { } @Test - public void testNewVectorVersionPreservesOldTail() { - DataFileMeta oldFile = createVectorFile("old", 0, 100, 1); - DataFileMeta newFile = createVectorFile("new", 0, 50, 2); - vectorBunch.add(oldFile); - vectorBunch.add(newFile); + public void testAddVectorFileWithSameFirstRowId() { + DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); + DataFileMeta vectorEntry2 = createVectorFile("vector2", 0, 50, 2); - assertVectorSelection(vectorBunch, newFile, new Range(0, 49), oldFile, new Range(50, 99)); + vectorBunch.add(vectorEntry1); + // Adding file with same firstRowId but higher sequence number should throw exception + assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Vector file with same first row id should have decreasing sequence number."); } @Test - public void testNewVectorVersionReplacesCoveredOldRows() { - DataFileMeta newFile = createVectorFile("new", 0, 100, 2); - DataFileMeta oldFile = createVectorFile("old", 0, 50, 1); - vectorBunch.add(newFile); - vectorBunch.add(oldFile); + public void testAddVectorFileWithSameFirstRowIdAndLowerSequenceNumber() { + DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 2); + DataFileMeta vectorEntry2 = createVectorFile("vector2", 0, 50, 1); - assertThat(vectorBunch.rowCount()).isEqualTo(100); - assertThat(vectorBunch.selectedFiles()).hasSize(1); - assertThat(vectorBunch.selectedFiles().get(0).file).isEqualTo(newFile); - assertThat(vectorBunch.selectedFiles().get(0).range).isEqualTo(new Range(0, 99)); + vectorBunch.add(vectorEntry1); + // Adding file with same firstRowId and lower sequence number should be ignored + vectorBunch.add(vectorEntry2); + + assertThat(vectorBunch.files).hasSize(1); + assertThat(vectorBunch.files.get(0)).isEqualTo(vectorEntry1); } @Test - public void testOlderOverlappingVectorFileSuppliesUncoveredTail() { - DataFileMeta newFile = createVectorFile("new", 0, 100, 2); - DataFileMeta oldFile = createVectorFile("old", 50, 150, 1); - vectorBunch.add(newFile); - vectorBunch.add(oldFile); + public void testAddVectorFileWithOverlappingRowId() { + DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 2); + DataFileMeta vectorEntry2 = createVectorFile("vector2", 50, 150, 1); - assertVectorSelection(vectorBunch, newFile, new Range(0, 99), oldFile, new Range(100, 199)); + vectorBunch.add(vectorEntry1); + // Adding file with overlapping row id and lower sequence number should be ignored + vectorBunch.add(vectorEntry2); + + assertThat(vectorBunch.files).hasSize(1); + assertThat(vectorBunch.files.get(0)).isEqualTo(vectorEntry1); } @Test - public void testNewOverlappingVectorFilePreservesOldPrefix() { - DataFileMeta oldFile = createVectorFile("old", 0, 100, 1); - DataFileMeta newFile = createVectorFile("new", 50, 150, 2); - vectorBunch.add(oldFile); - vectorBunch.add(newFile); + public void testAddVectorFileWithOverlappingRowIdAndHigherSequenceNumber() { + DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); + DataFileMeta vectorEntry2 = createVectorFile("vector2", 50, 150, 2); - assertVectorSelection(vectorBunch, oldFile, new Range(0, 49), newFile, new Range(50, 199)); + vectorBunch.add(vectorEntry1); + // Adding file with overlapping row id and higher sequence number should throw exception + assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Vector file with overlapping row id should have decreasing sequence number."); } @Test - public void testVectorSelectionRetainsGaps() { - DataFileMeta first = createVectorFile("first", 0, 100, 1); - DataFileMeta second = createVectorFile("second", 200, 300, 1); - vectorBunch.add(first); - vectorBunch.add(second); + public void testAddVectorFileWithNonContinuousRowId() { + DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); + DataFileMeta vectorEntry2 = createVectorFile("vector2", 200, 300, 1); - assertVectorSelection(vectorBunch, first, new Range(0, 99), second, new Range(200, 499)); + vectorBunch.add(vectorEntry1); + // Adding file with non-continuous row id should throw exception + assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "Vector file first row id should be continuous, expect 100 but got 200"); } @Test - public void testVectorSelectionRetainsPhysicalColumnNames() { - DataFileMeta oldFile = createVectorFile("old", 0, 100, 1); - DataFileMeta renamedFile = - createVectorFileWithCols( - "renamed", 100, 200, 2, Collections.singletonList("renamed_vector")); - vectorBunch.add(oldFile); - vectorBunch.add(renamedFile); - - assertVectorSelection( - vectorBunch, oldFile, new Range(0, 99), renamedFile, new Range(100, 299)); + public void testAddVectorFileWithDifferentWriteCols() { + DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); + DataFileMeta vectorEntry2 = + createVectorFileWithCols("vector2", 100, 200, 1, Arrays.asList("different_col")); + + vectorBunch.add(vectorEntry1); + // Adding file with different write columns should throw exception + assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("All files in this bunch should have the same write columns."); } @Test @@ -441,13 +453,14 @@ private DataFileMeta createFile( } @Test - void testVectorSelectionAcrossSchemas() { - DataFileMeta oldFile = createVectorFileWithSchema("old", 0, 100, 1, 0L); - DataFileMeta newFile = createVectorFileWithSchema("new", 100, 200, 2, 1L); - vectorBunch.add(oldFile); - vectorBunch.add(newFile); + void testAddVectorFilesWithDifferentSchemaId() { + DataFileMeta vectorEntry1 = createVectorFileWithSchema("vector1", 0, 100, 1, 0L); + DataFileMeta vectorEntry2 = createVectorFileWithSchema("vector2", 100, 200, 1, 1L); - assertVectorSelection(vectorBunch, oldFile, new Range(0, 99), newFile, new Range(100, 299)); + vectorBunch.add(vectorEntry1); + assertThatThrownBy(() -> vectorBunch.add(vectorEntry2)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("All files in this bunch should have the same schema id."); } @Test @@ -485,30 +498,25 @@ void testBlobBunchRejectsRangeOutsideNormalFile() { } @Test - public void testVectorSelectionClipsToNormalRange() { - VectorFileBunch clipped = new VectorFileBunch(0, new Range(50, 149)); - DataFileMeta oldFile = createVectorFile("old", 0, 200, 1); - DataFileMeta newFile = createVectorFile("new", 100, 100, 2); - clipped.add(oldFile); - clipped.add(newFile); - - assertVectorSelection(clipped, oldFile, new Range(50, 99), newFile, new Range(100, 149)); - assertThat(oldFile.nonNullRowIdRange()).isEqualTo(new Range(0, 199)); - assertThat(newFile.nonNullRowIdRange()).isEqualTo(new Range(100, 199)); - } + public void testRowIdPushDown() { + VectorFileBunch vectorBunch = new VectorFileBunch(Long.MAX_VALUE, true); + DataFileMeta vectorEntry1 = createVectorFile("vector1", 0, 100, 1); + DataFileMeta vectorEntry2 = createVectorFile("vector2", 200, 300, 1); + vectorBunch.add(vectorEntry1); + VectorFileBunch finalVectorBunch = vectorBunch; + DataFileMeta finalVectorEntry = vectorEntry2; + assertThatCode(() -> finalVectorBunch.add(finalVectorEntry)).doesNotThrowAnyException(); + + vectorBunch = new VectorFileBunch(Long.MAX_VALUE, true); + vectorEntry1 = createVectorFile("vector1", 0, 100, 1); + vectorEntry2 = createVectorFile("vector2", 50, 200, 2); + vectorBunch.add(vectorEntry1); + vectorBunch.add(vectorEntry2); + assertThat(vectorBunch.files).containsExactlyInAnyOrder(vectorEntry2); - private static void assertVectorSelection( - VectorFileBunch bunch, - DataFileMeta first, - Range firstRange, - DataFileMeta second, - Range secondRange) { - assertThat(bunch.selectedFiles()).hasSize(2); - assertThat(bunch.selectedFiles().get(0).file).isEqualTo(first); - assertThat(bunch.selectedFiles().get(0).range).isEqualTo(firstRange); - assertThat(bunch.selectedFiles().get(1).file).isEqualTo(second); - assertThat(bunch.selectedFiles().get(1).range).isEqualTo(secondRange); - assertThat(bunch.rowCount()).isEqualTo(firstRange.count() + secondRange.count()); + VectorFileBunch finalVectorBunch2 = vectorBunch; + DataFileMeta vectorEntry3 = createVectorFile("vector2", 250, 100, 2); + assertThatCode(() -> finalVectorBunch2.add(vectorEntry3)).doesNotThrowAnyException(); } /** Creates a normal (non-blob) file for testing. */ diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java index b12b6058146f..cd60d981643a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java @@ -20,9 +20,6 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryString; -import org.apache.paimon.data.BinaryVector; -import org.apache.paimon.data.BlobData; -import org.apache.paimon.data.BlobPlaceholder; import org.apache.paimon.data.GenericArray; import org.apache.paimon.data.GenericRow; import org.apache.paimon.data.InternalRow; @@ -38,15 +35,9 @@ import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; -import org.apache.paimon.stats.SimpleStats; -import org.apache.paimon.table.FileStoreTable; -import org.apache.paimon.table.FileStoreTableFactory; -import org.apache.paimon.table.SpecialFields; import org.apache.paimon.table.source.DataSplit; -import org.apache.paimon.table.source.Split; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; @@ -54,17 +45,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.IntFunction; -import java.util.stream.Collectors; -import java.util.stream.IntStream; import static org.apache.paimon.data.BinaryRow.EMPTY_ROW; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -177,271 +163,6 @@ public void testSplitWithMultipleVectorStoreFilesPerGroup() { assertEquals(Arrays.asList(file4, file5, file6), result.get(1)); } - @Test - public void testSplitWithDedicatedFilesSpanningNormalGroups() { - DataFileMeta first = createFile("first.parquet", 0, 4, 3); - DataFileMeta middle = createFile("middle.parquet", 4, 4, 3); - DataFileMeta last = createFile("last.parquet", 8, 4, 3); - DataFileMeta blob = createFile("blob.blob", 0, 12, 1); - DataFileMeta vector = createFile("vector.vector.json", 0, 12, 1); - DataFileMeta vectorUpdate = createFile("update.vector.json", 4, 4, 2); - - List> groups = - DataEvolutionSplitRead.mergeRangesAndSort( - Arrays.asList(blob, vector, middle, last, first, vectorUpdate)); - - assertEquals( - Arrays.asList( - Arrays.asList(first, blob, vector), - Arrays.asList(middle, blob, vectorUpdate, vector), - Arrays.asList(last, blob, vector)), - groups); - // Associations must retain physical file offsets for readers of the second and third group. - assertEquals(new Range(0, 11), groups.get(2).get(1).nonNullRowIdRange()); - } - - @ParameterizedTest - @CsvSource({ - "false,4,false", - "true,4,false", - "false,12,false", - "true,12,false", - "false,4,true", - "true,4,true", - "false,12,true", - "true,12,true" - }) - public void testReadSpanningDedicatedFiles( - boolean indexed, int normalRowCount, boolean renameBeforeUpdate) throws Exception { - LocalFileIO fileIO = new LocalFileIO(); - Path tablePath = new Path(tempDir.resolve("spanning").toUri()); - SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); - TableSchema schema = - schemaManager.createTable( - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("blob", DataTypes.BLOB()) - .column("vector", DataTypes.VECTOR(2, DataTypes.FLOAT())) - .column("vector2", DataTypes.VECTOR(2, DataTypes.FLOAT())) - .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") - .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") - .build()); - FileStoreTable table = FileStoreTableFactory.create(fileIO, tablePath, schema); - FileStorePathFactory pathFactory = table.store().pathFactory(); - Path bucketPath = pathFactory.bucketPath(EMPTY_ROW, 0); - fileIO.mkdirs(bucketPath); - RowType rowType = schema.logicalRowType(); - List files = new ArrayList<>(); - for (int from = 0; from < 12; from += normalRowCount) { - files.add( - writeProjectedFile( - fileIO, - bucketPath, - "normal-" + from + ".parquet", - "parquet", - rowType.project("id"), - from, - normalRowCount, - 3, - GenericRow::of)); - } - files.add( - writeProjectedFile( - fileIO, - bucketPath, - "base.blob", - "blob", - rowType.project("blob"), - 0, - 12, - 1, - i -> GenericRow.of(new BlobData(new byte[] {(byte) i})))); - files.add( - writeProjectedFile( - fileIO, - bucketPath, - "update.blob", - "blob", - rowType.project("blob"), - 2, - 8, - 2, - i -> - GenericRow.of( - i % 2 == 0 - ? BlobPlaceholder.INSTANCE - : new BlobData(new byte[] {(byte) (i + 20)})))); - files.add( - writeProjectedFile( - fileIO, - bucketPath, - "base.vector.json", - "json", - rowType.project("vector", "vector2"), - 0, - 12, - 1, - i -> - GenericRow.of( - BinaryVector.fromPrimitiveArray(new float[] {i, i + 1}), - BinaryVector.fromPrimitiveArray( - new float[] {i + 100, i + 101})))); - if (renameBeforeUpdate) { - schema = - schemaManager.commitChanges( - SchemaChange.renameColumn("vector", "renamed_vector")); - } - files.add( - writeProjectedFile( - fileIO, - bucketPath, - "update.vector.json", - "json", - schema.logicalRowType() - .project(renameBeforeUpdate ? "renamed_vector" : "vector"), - 4, - 4, - 2, - i -> - GenericRow.of( - BinaryVector.fromPrimitiveArray( - new float[] {i + 20, i + 21})), - schema.id())); - if (!renameBeforeUpdate) { - schema = - schemaManager.commitChanges( - SchemaChange.renameColumn("vector", "renamed_vector")); - } - RowType allReadType = SpecialFields.rowTypeWithRowId(schema.logicalRowType()); - DataSplit dataSplit = - DataSplit.builder() - .withPartition(EMPTY_ROW) - .withBucket(0) - .withBucketPath(bucketPath.toString()) - .withDataFiles(files) - .rawConvertible(false) - .build(); - for (boolean vectorOnly : new boolean[] {false, true}) { - RowType readType = - vectorOnly - ? allReadType.project( - "renamed_vector", "vector2", SpecialFields.ROW_ID.name()) - : allReadType; - DataSplit projectedSplit = - vectorOnly - ? dataSplit - .filterDataFile( - file -> - org.apache.paimon.types.VectorType - .isVectorStoreFile(file.fileName())) - .get() - : dataSplit; - Split split = - indexed - ? new IndexedSplit( - projectedSplit, - Arrays.asList( - new Range(1, 2), new Range(5, 5), new Range(8, 9)), - null) - : projectedSplit; - DataEvolutionSplitRead splitRead = - new DataEvolutionSplitRead( - fileIO, - schemaManager, - schema, - readType, - table.coreOptions(), - pathFactory); - List ids = new ArrayList<>(); - try (RecordReader reader = splitRead.createReader(split)) { - reader.forEachRemaining( - row -> { - int id = (int) row.getLong(vectorOnly ? 2 : 4); - ids.add(id); - if (!vectorOnly) { - assertEquals(id, row.getInt(0)); - int expectedBlob = id >= 2 && id < 10 && id % 2 != 0 ? id + 20 : id; - assertEquals((byte) expectedBlob, row.getBlob(1).toData()[0]); - } - int expectedVector = id >= 4 && id < 8 ? id + 20 : id; - org.assertj.core.api.Assertions.assertThat( - row.getVector(vectorOnly ? 0 : 2).toFloatArray()) - .containsExactly(expectedVector, expectedVector + 1); - org.assertj.core.api.Assertions.assertThat( - row.getVector(vectorOnly ? 1 : 3).toFloatArray()) - .containsExactly(id + 100, id + 101); - }); - } - assertEquals( - indexed - ? Arrays.asList(1, 2, 5, 8, 9) - : IntStream.range(0, 12).boxed().collect(Collectors.toList()), - ids); - } - } - - private static DataFileMeta writeProjectedFile( - LocalFileIO fileIO, - Path bucketPath, - String name, - String formatIdentifier, - RowType writeType, - int firstRowId, - int rowCount, - int sequence, - IntFunction rowFactory) - throws IOException { - return writeProjectedFile( - fileIO, - bucketPath, - name, - formatIdentifier, - writeType, - firstRowId, - rowCount, - sequence, - rowFactory, - 0); - } - - private static DataFileMeta writeProjectedFile( - LocalFileIO fileIO, - Path bucketPath, - String name, - String formatIdentifier, - RowType writeType, - int firstRowId, - int rowCount, - int sequence, - IntFunction rowFactory, - long schemaId) - throws IOException { - Path filePath = new Path(bucketPath, name); - FileFormat format = FileFormat.fromIdentifier(formatIdentifier, new Options()); - try (PositionOutputStream output = fileIO.newOutputStream(filePath, false)) { - FormatWriter writer = format.createWriterFactory(writeType).create(output, "none"); - for (int i = firstRowId; i < firstRowId + rowCount; i++) { - writer.addElement(rowFactory.apply(i)); - } - writer.close(); - } - return DataFileMeta.forAppend( - name, - fileIO.getFileStatus(filePath).getLen(), - rowCount, - SimpleStats.EMPTY_STATS, - sequence, - sequence, - schemaId, - Collections.emptyList(), - null, - FileSource.APPEND, - null, - null, - (long) firstRowId, - writeType.getFieldNames()); - } - @Test public void testRowSidecarFileName() { DataFileMeta file = diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java index 8cba65317de3..787d0761ea3e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java @@ -38,8 +38,6 @@ import org.apache.paimon.utils.SnapshotManager; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import javax.annotation.Nullable; @@ -1363,7 +1361,6 @@ void testCheckRowIdRangeConflictsUsesRetryableExceptionForDataFiles() { @Test void testCheckRowIdRangeConflictsReportsDedicatedFileSpanningDataFiles() { DataEvolutionConflictDetection detection = createConflictDetection(); - detection.setRowIdCheckFromSnapshot(1L); Optional exception = detection.checkConflicts( @@ -1374,7 +1371,7 @@ void testCheckRowIdRangeConflictsReportsDedicatedFileSpanningDataFiles() { Collections.singletonList(createFileEntryWithRowId("p1.blob", ADD, 0L, 4L)), Collections.emptyList(), null, - Snapshot.CommitKind.APPEND); + Snapshot.CommitKind.COMPACT); assertThat(exception).isPresent(); assertThat(exception.get()) @@ -1386,130 +1383,6 @@ void testCheckRowIdRangeConflictsReportsDedicatedFileSpanningDataFiles() { .hasMessageContaining("f2"); } - @ParameterizedTest - @ValueSource(strings = {"retained.blob", "retained.vector.json"}) - void testNormalSplitRetainsDedicatedFileAcrossAdjacentRanges(String dedicatedFile) { - DataEvolutionConflictDetection detection = createConflictDetection(); - - assertThat( - detection.checkConflicts( - snapshot(1), - Arrays.asList( - createFileEntryWithRowId("normal", ADD, 0L, 4L), - createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), - Arrays.asList( - createFileEntryWithRowId("normal", DELETE, 0L, 4L), - createFileEntryWithRowId("split-1", ADD, 0L, 2L), - createFileEntryWithRowId("split-2", ADD, 2L, 2L)), - Collections.emptyList(), - null, - Snapshot.CommitKind.COMPACT)) - .isEmpty(); - } - - @ParameterizedTest - @ValueSource(strings = {"retained.blob", "retained.vector.json"}) - void testNormalCompactionRetainsDedicatedFileOutsideScannedRange(String dedicatedFile) { - DataEvolutionConflictDetection detection = createConflictDetection(); - - // A later compaction scans only one of the normal ranges inside the retained file. - assertThat( - detection.checkConflicts( - snapshot(1), - Arrays.asList( - createFileEntryWithRowId("normal", ADD, 2L, 2L), - createFileEntryWithRowId(dedicatedFile, ADD, 0L, 6L)), - Arrays.asList( - createFileEntryWithRowId("normal", DELETE, 2L, 2L), - createFileEntryWithRowId("split-1", ADD, 2L, 1L), - createFileEntryWithRowId("split-2", ADD, 3L, 1L)), - Collections.emptyList(), - null, - Snapshot.CommitKind.COMPACT)) - .isEmpty(); - } - - @ParameterizedTest - @ValueSource(strings = {"retained.blob", "retained.vector.json"}) - void testNormalSplitRejectsGapUnderRetainedDedicatedFile(String dedicatedFile) { - DataEvolutionConflictDetection detection = createConflictDetection(); - - Optional exception = - detection.checkConflicts( - snapshot(1), - Arrays.asList( - createFileEntryWithRowId("normal", ADD, 0L, 4L), - createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), - Arrays.asList( - createFileEntryWithRowId("normal", DELETE, 0L, 4L), - createFileEntryWithRowId("split-1", ADD, 0L, 1L), - createFileEntryWithRowId("split-2", ADD, 2L, 2L)), - Collections.emptyList(), - null, - Snapshot.CommitKind.COMPACT); - - assertThat(exception).isPresent(); - assertThat(exception.get()).hasMessageContaining("dedicated file"); - } - - @ParameterizedTest - @ValueSource(strings = {"retained.blob", "retained.vector.json"}) - void testNormalCompactionRejectsOrphanedDedicatedFile(String dedicatedFile) { - DataEvolutionConflictDetection detection = createConflictDetection(); - - Optional exception = - detection.checkConflicts( - snapshot(1), - Arrays.asList( - createFileEntryWithRowId("normal", ADD, 0L, 4L), - createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), - Collections.singletonList( - createFileEntryWithRowId("normal", DELETE, 0L, 4L)), - Collections.emptyList(), - null, - Snapshot.CommitKind.COMPACT); - - assertThat(exception).isPresent(); - assertThat(exception.get()).hasMessageContaining("dedicated file"); - } - - @ParameterizedTest - @ValueSource(strings = {"compacted.blob", "compacted.vector.json"}) - void testDedicatedCompactionMaySpanSplitNormalFiles(String dedicatedFile) { - DataEvolutionConflictDetection detection = createConflictDetection(); - - assertThat( - detection.checkConflicts( - snapshot(1), - Arrays.asList( - createFileEntryWithRowId("split-1", ADD, 0L, 2L), - createFileEntryWithRowId("split-2", ADD, 2L, 2L)), - Collections.singletonList( - createFileEntryWithRowId(dedicatedFile, ADD, 0L, 4L)), - Collections.emptyList(), - null, - Snapshot.CommitKind.COMPACT)) - .isEmpty(); - } - - @ParameterizedTest - @ValueSource(strings = {"merge-normal", "merge.blob", "merge.vector.json"}) - void testMergeRejectsStaleFileAcrossSplitNormalRanges(String mergeFile) { - DataEvolutionConflictDetection detection = createConflictDetection(); - - Optional exception = - detection.checkRowIdExistence( - Arrays.asList( - createFileEntryWithRowId("split-1", ADD, 0L, 2L), - createFileEntryWithRowId("split-2", ADD, 2L, 2L)), - Collections.singletonList(createFileEntryWithRowId(mergeFile, ADD, 0L, 4L)), - 4L, - Snapshot.CommitKind.APPEND); - - assertThat(exception).isPresent(); - assertThat(exception.get()).isInstanceOf(RowIdExistenceConflictException.class); - } - @Test void testCheckRowIdRangeConflictsRejectsOverlappingNormalFiles() { DataEvolutionConflictDetection detection = createConflictDetection(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java index a5ad4f1a9ffc..8f189a791760 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator; import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask; +import org.apache.paimon.append.dataevolution.DataEvolutionCompactTaskSerializer; import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation; import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorMaterializeCoordinator; import org.apache.paimon.append.dataevolution.DataEvolutionRowIdReassigner; @@ -593,48 +594,76 @@ public void testSplitLargeFilePreservesDeletionVectorsAndBlob(boolean bitmap64) options.put(CoreOptions.DELETION_VECTOR_BITMAP64.key(), String.valueOf(bitmap64)); options.put(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "128 mb"); FileStoreTable table = getTableDefault().copy(options); - BatchWriteBuilder builder = table.newBatchWriteBuilder(); - try (BatchTableWrite write = builder.newWrite(); - BatchTableCommit commit = builder.newCommit()) { - for (int rowId = 0; rowId < 2500; rowId++) { - write.write( - GenericRow.of( - rowId, - BinaryString.fromString("name-" + rowId), - BinaryString.fromString("base-" + rowId), - new BlobData(new byte[] {(byte) rowId}))); + for (int batch = 0; batch < 3; batch++) { + BatchWriteBuilder builder = table.newBatchWriteBuilder(); + try (BatchTableWrite write = builder.newWrite(); + BatchTableCommit commit = builder.newCommit()) { + for (int rowId = batch * 1250; rowId < (batch + 1) * 1250; rowId++) { + write.write( + GenericRow.of( + rowId, + BinaryString.fromString("name-" + rowId), + BinaryString.fromString("base-" + rowId), + new BlobData(new byte[] {(byte) rowId}))); + } + commit.commit(write.prepareCommit()); } - commit.commit(write.prepareCommit()); } - Range range = new Range(0, 2499); - commitDeletionVectors( - table, Collections.singletonList(new DvSpec(range, 0, 999, 2000, 2499))); - List expected = readRows(table.newReadBuilder()); - String oldAnchor = anchorFilesByRange(table).get(range); List blobs = currentDataFiles(table, BinaryRow.EMPTY_ROW).stream() .filter(file -> isBlobFile(file.fileName())) .collect(Collectors.toList()); - assertThat(blobs).hasSize(1); + assertThat(blobs).hasSize(3); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + compactDataEvolutionTable(table.copy(options), false); + Range range = new Range(0, 3749); + assertRegularFileRowRanges( + currentDataFiles(table, BinaryRow.EMPTY_ROW), Collections.singletonList(range)); + long[] deletedRowIds = {0, 1249, 1250, 2499, 2500, 3749}; + commitDeletionVectors(table, Collections.singletonList(new DvSpec(range, deletedRowIds))); + List expected = readRows(table.newReadBuilder()); + String oldAnchor = anchorFilesByRange(table).get(range); + + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "10"); options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); table = table.copy(options); - compactDataEvolutionTable(table, false); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + List tasks = + new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan(); + assertThat(tasks).hasSize(1); + DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer(); + DataEvolutionCompactTask task = + serializer.deserialize(serializer.getVersion(), serializer.serialize(tasks.get(0))); + commit(table, prepareCompactionMessages(table, snapshot, Collections.singletonList(task))); List files = currentDataFiles(table, BinaryRow.EMPTY_ROW); List normalFiles = files.stream() .filter(file -> !isBlobFile(file.fileName())) .collect(Collectors.toList()); - assertThat(normalFiles.size()).isGreaterThan(1); - assertThat(normalFiles.stream().mapToLong(DataFileMeta::rowCount).sum()).isEqualTo(2500); + assertRegularFileRowRanges( + files, + Arrays.asList(new Range(0, 1249), new Range(1250, 2499), new Range(2500, 3749))); + assertThat(normalFiles.stream().mapToLong(DataFileMeta::rowCount).sum()).isEqualTo(3750); assertThat(files.stream().filter(file -> isBlobFile(file.fileName()))) .containsExactlyInAnyOrderElementsOf(blobs); + for (DataFileMeta blob : blobs) { + Range blobRange = blob.nonNullRowIdRange(); + assertThat( + normalFiles.stream() + .map(DataFileMeta::nonNullRowIdRange) + .anyMatch( + normal -> + normal.from <= blobRange.from + && normal.to >= blobRange.to)) + .isTrue(); + } assertThat(readRows(table.newReadBuilder())).containsExactlyElementsOf(expected); List expectedAnchors = new ArrayList<>(); for (DataFileMeta file : normalFiles) { Range fileRange = file.nonNullRowIdRange(); - if (Arrays.stream(new long[] {0, 999, 2000, 2499}) + if (Arrays.stream(deletedRowIds) .anyMatch(id -> id >= fileRange.from && id <= fileRange.to)) { expectedAnchors.add(file.fileName()); } @@ -656,47 +685,15 @@ public void testSplitLargeFilePreservesDeletionVectorsAndBlob(boolean bitmap64) reader.forEachRemaining(row -> actualBlobs.add(row.getBlob(0).toData()[0])); } List expectedBlobs = new ArrayList<>(); - for (int rowId = 0; rowId < 2500; rowId++) { - if (rowId != 0 && rowId != 999 && rowId != 2000 && rowId != 2499) { + for (int rowId = 0; rowId < 3750; rowId++) { + final int id = rowId; + if (Arrays.stream(deletedRowIds).noneMatch(deleted -> deleted == id)) { expectedBlobs.add((byte) rowId); } } assertThat(actualBlobs).containsExactlyElementsOf(expectedBlobs); } - @Test - public void testNormalCompactRetainsSpanningBlobFile() throws Exception { - createTableDefault(); - FileStoreTable table = getTableDefault(); - writeBaseRows(table); - writeBlobRange(table, 5L, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114); - List blobs = - currentDataFiles(table, BinaryRow.EMPTY_ROW).stream() - .filter(file -> isBlobFile(file.fileName())) - .collect(Collectors.toList()); - Map options = new HashMap<>(); - options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); - options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); - compactDataEvolutionTable(table.copy(options), false); - - assertThat( - currentDataFiles(table, BinaryRow.EMPTY_ROW).stream() - .filter(file -> isBlobFile(file.fileName()))) - .containsExactlyInAnyOrderElementsOf(blobs); - List expectedRows = new ArrayList<>(); - for (int rowId = 0; rowId < 15; rowId++) { - expectedRows.add( - rowId - + "|name-" - + rowId - + "|base-" - + rowId - + "|" - + (rowId < 5 ? rowId : rowId + 100)); - } - assertThat(readRows(table.newReadBuilder())).containsExactlyElementsOf(expectedRows); - } - @Test public void testCompactRenamesDeletionVectorForSameRowRange() throws Exception { createTableDefault(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java index 6519d1ed18d7..e3f5403c68be 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/DataSplitCompatibleTest.java @@ -93,59 +93,6 @@ public void testSplitMergedRowCount() { assertThat(split.mergedRowCount()).hasValue(5700L); } - @Test - public void testMergedRowCountWithSpanningDedicatedFiles() { - List files = - Arrays.asList( - newRowTrackedDataFile("first.parquet", 0, 5), - newRowTrackedDataFile("update.parquet", 0, 5), - newRowTrackedDataFile("second.parquet", 5, 5), - newRowTrackedDataFile("first.blob", 0, 7), - newRowTrackedDataFile("second.blob", 7, 3)); - DataSplit split = newDataSplit(false, files, null); - assertThat(split.mergedRowCount()).hasValue(10L); - - split = - newDataSplit( - false, - files, - Arrays.asList( - new DeletionFile("dv", 0, 1, 1L), - null, - new DeletionFile("dv", 1, 1, 2L), - null, - null)); - assertThat(split.mergedRowCount()).hasValue(7L); - - // A row-range scan may retain a dedicated file extending outside its normal file. - split = newDataSplit(false, Arrays.asList(files.get(2), files.get(3), files.get(4)), null); - assertThat(split.mergedRowCount()).hasValue(5L); - - // Projection can prune the normal file from another component packed in this split. - List projectedFiles = new ArrayList<>(files); - projectedFiles.add(newRowTrackedDataFile("projected.blob", 10, 10)); - assertThat(newDataSplit(false, projectedFiles, null).mergedRowCount()).hasValue(20L); - } - - private DataFileMeta newRowTrackedDataFile(String name, long firstRowId, long rowCount) { - return DataFileMeta.forAppend( - name, - 1024, - rowCount, - SimpleStats.EMPTY_STATS, - 0L, - rowCount - 1, - 1, - Collections.emptyList(), - null, - null, - null, - null, - null, - null) - .assignFirstRowId(firstRowId); - } - @Test public void testDeletionFilesSerialize() throws Exception { List dataFiles = diff --git a/paimon-python/pypaimon/read/reader/field_bunch.py b/paimon-python/pypaimon/read/reader/field_bunch.py index bbc60efe6e10..65ab1af046ed 100644 --- a/paimon-python/pypaimon/read/reader/field_bunch.py +++ b/paimon-python/pypaimon/read/reader/field_bunch.py @@ -171,9 +171,8 @@ def finish(self) -> None: physical_row_count = sum(row_range.count() for row_range in merged) if self.expected_row_range is not None: for row_range in merged: - if (not self.row_id_push_down - and (row_range.from_ < self.expected_row_range.from_ - or row_range.to > self.expected_row_range.to)): + if (row_range.from_ < self.expected_row_range.from_ + or row_range.to > self.expected_row_range.to): raise ValueError( f"Blob file range {row_range} should be within normal " f"file range {self.expected_row_range}." @@ -237,33 +236,6 @@ def _file_type_label(self) -> str: class VectorBunch(_SpecialFieldBunch): """Files for partial field (vector files).""" - def __init__(self, expected_row_count: int, row_id_push_down: bool = False, - expected_row_range: Optional[Range] = None, field_id: Optional[int] = None): - super().__init__(expected_row_count, row_id_push_down) - self.expected_row_range = expected_row_range - self.field_id = field_id - - def add(self, file: DataFileMeta) -> None: - if not self._is_special_file(file.file_name): - raise ValueError("Only vector file can be added to a vector bunch.") - self._files.append(file) - - def segments(self): - """Select the newest physical file for each part of the normal range.""" - covered = [] - segments = [] - for file in sorted(self._files, - key=lambda f: (-f.max_sequence_number, f.file_name)): - selected = ([file.row_id_range()] if self.expected_row_range is None - else Range.and_([file.row_id_range()], [self.expected_row_range])) - for row_range in selected: - segments.extend((visible, file) for visible in row_range.exclude(covered)) - covered.append(row_range) - return sorted(segments, key=lambda segment: segment[0].from_) - - def row_count(self) -> int: - return sum(row_range.count() for row_range, _ in self.segments()) - def _is_special_file(self, file_name: str) -> bool: return DataFileMeta.is_vector_file(file_name) diff --git a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py b/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py index 8da60a6bdc4e..89216322cfe1 100644 --- a/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py +++ b/paimon-python/pypaimon/read/scanner/chunk_shuffle_split_generator.py @@ -30,8 +30,9 @@ from pypaimon.read.split import DataSplit, Split from pypaimon.table.row.generic_row import GenericRow from pypaimon.table.source.deletion_file import DeletionFile -from pypaimon.utils.data_evolution_utils import retrieve_anchor_file, split_normal_file_groups +from pypaimon.utils.data_evolution_utils import retrieve_anchor_file from pypaimon.utils.range import Range +from pypaimon.utils.range_helper import RangeHelper def _null_safe_partition_key(partition_values) -> tuple: @@ -576,9 +577,6 @@ def _chunk_to_split(self, chunk: _Chunk) -> Split: row_ranges.append(seg.row_range) row_ranges.sort(key=lambda r: r.from_) - # A retained dedicated file can belong to several normal ranges in one chunk. - all_files = list({file.file_name: file for file in all_files}.values()) - data_deletion_files = self._get_deletion_files_for_split( all_files, chunk.partition, @@ -604,20 +602,23 @@ def _chunk_to_split(self, chunk: _Chunk) -> Split: def _split_by_row_id_with_range( files: List[DataFileMeta], ) -> List[Tuple[Range, List[DataFileMeta]]]: - """Group by normal ranges so each chunk uses the correct deletion-vector anchor.""" + """Group files by overlapping row_id range, returning (range, files) + pairs sorted by ``range.from_``. + + Mirrors :meth:`DataEvolutionSplitGenerator._split_by_row_id` but + also returns the merged row_id range per group, which the chunk + slicer needs to drive row-count accumulation. + """ for f in files: if f.row_id_range() is None: raise ValueError( "chunk_shuffle for data evolution tables requires row tracking; " f"file {f.file_name} is missing first_row_id" ) - groups = split_normal_file_groups(files) + groups = RangeHelper(lambda f: f.row_id_range()).merge_overlapping_ranges(files) result = [] for group in groups: - normal_files = [f for f in group - if not DataFileMeta.is_blob_file(f.file_name) - and not DataFileMeta.is_vector_file(f.file_name)] - ranges = [f.row_id_range() for f in normal_files or group] + ranges = [f.row_id_range() for f in group] merged = Range(min(r.from_ for r in ranges), max(r.to for r in ranges)) result.append((merged, group)) return sorted(result, key=lambda kv: kv[0].from_) diff --git a/paimon-python/pypaimon/read/split_read.py b/paimon-python/pypaimon/read/split_read.py index feca4f649c50..f00976f1f0fc 100644 --- a/paimon-python/pypaimon/read/split_read.py +++ b/paimon-python/pypaimon/read/split_read.py @@ -17,7 +17,6 @@ import os from abc import ABC, abstractmethod -from copy import copy from functools import partial from typing import Callable, Dict, List, Optional, Tuple @@ -81,7 +80,7 @@ from pypaimon.schema.data_types import DataField, PyarrowFieldParser from pypaimon.table.special_fields import SpecialFields from pypaimon.globalindex.indexed_split import IndexedSplit -from pypaimon.utils.data_evolution_utils import retrieve_anchor_file, split_normal_file_groups +from pypaimon.utils.data_evolution_utils import retrieve_anchor_file KEY_PREFIX = "_KEY_" KEY_FIELD_ID_START = 1000000 @@ -1158,20 +1157,16 @@ def _create_raw_reader(self) -> RecordReader: split_by_row_id = self._split_by_row_id(files) for need_merge_files in split_by_row_id: - group_read = self._read_for_normal_range(need_merge_files) - if group_read.row_ranges == []: - continue deletion_vector = self._read_deletion_vector(need_merge_files) if len(need_merge_files) == 1 or not self.read_fields: # No need to merge fields, just create a single file reader suppliers.append( - lambda f=need_merge_files[0], dv=deletion_vector, read=group_read: read._create_file_reader( - f, read._get_final_read_data_fields(), dv) + lambda f=need_merge_files[0], dv=deletion_vector: self._create_file_reader( + f, self._get_final_read_data_fields(), dv) ) else: suppliers.append( - lambda files=need_merge_files, dv=deletion_vector, read=group_read: - read._create_union_reader(files, dv) + lambda files=need_merge_files, dv=deletion_vector: self._create_union_reader(files, dv) ) merge_reader = ConcatBatchReader( @@ -1229,10 +1224,7 @@ def _apply_deletion_vector(self, reader, reader_range: Range, deletion_vector): if dv.is_empty(): return reader - selected_ranges = ([reader_range] if self.row_ranges is None - else Range.and_([reader_range], self.row_ranges)) - if any(dv_range.from_ > selected.from_ or dv_range.to < selected.to - for selected in selected_ranges): + if dv_range.from_ > reader_range.from_ or dv_range.to < reader_range.to: raise ValueError( f"Deletion vector range {dv_range} should contain reader range {reader_range}." ) @@ -1282,7 +1274,7 @@ def _create_prescan_reader(self, field_names): return prescan_read._create_raw_reader() def _split_by_row_id(self, files: List[DataFileMeta]) -> List[List[DataFileMeta]]: - """Split by normal row ranges, retaining spanning dedicated files in each group.""" + """Split files by firstRowId for data evolution.""" # Sort files by firstRowId and then by maxSequenceNumber def sort_key(file: DataFileMeta) -> tuple: @@ -1290,39 +1282,41 @@ def sort_key(file: DataFileMeta) -> tuple: is_special = 1 if (DataFileMeta.is_blob_file(file.file_name) or DataFileMeta.is_vector_file(file.file_name)) else 0 max_seq = file.max_sequence_number - return (is_special, first_row_id, -max_seq) - - tracked_files = [file for file in files if file.first_row_id is not None] - groups = split_normal_file_groups(tracked_files) - for group in groups: - normal_ranges = { - (file.first_row_id, file.row_count) for file in group - if not DataFileMeta.is_blob_file(file.file_name) - and not DataFileMeta.is_vector_file(file.file_name) - } - if len(normal_ranges) > 1: - raise ValueError(f"There are overlapping files in the split: {group}") - group.sort(key=sort_key) - return [[file] for file in files if file.first_row_id is None] + groups - - def _read_for_normal_range(self, files: List[DataFileMeta]): - normal_files = [file for file in files - if not DataFileMeta.is_blob_file(file.file_name) - and not DataFileMeta.is_vector_file(file.file_name)] - if not normal_files or normal_files[0].first_row_id is None: - return self - normal_range = normal_files[0].row_id_range() - if all(file.row_id_range().from_ >= normal_range.from_ - and file.row_id_range().to <= normal_range.to for file in files): - return self - - # Suppliers are lazy: keep the clipping range on a separate read instance. - group_read = copy(self) - group_read.row_ranges = ( - [normal_range] if self.row_ranges is None - else Range.and_([normal_range], self.row_ranges) - ) - return group_read + return (first_row_id, is_special, -max_seq) + + sorted_files = sorted(files, key=sort_key) + + # Split files by firstRowId + split_by_row_id = [] + last_row_id = -1 + check_row_id_start = 0 + current_split = [] + + for file in sorted_files: + first_row_id = file.first_row_id + if first_row_id is None: + split_by_row_id.append([file]) + continue + + if (not DataFileMeta.is_blob_file(file.file_name) + and not DataFileMeta.is_vector_file(file.file_name) + and first_row_id != last_row_id): + if current_split: + split_by_row_id.append(current_split) + if first_row_id < check_row_id_start: + raise ValueError( + f"There are overlapping files in the split: {files}, " + f"the wrong file is: {file}" + ) + current_split = [] + last_row_id = first_row_id + check_row_id_start = first_row_id + file.row_count + current_split.append(file) + + if current_split: + split_by_row_id.append(current_split) + + return split_by_row_id def _create_union_reader(self, need_merge_files: List[DataFileMeta], deletion_vector=None) -> RecordReader: """Create a DataEvolutionFileReader for merging multiple files.""" @@ -1361,10 +1355,9 @@ def _create_union_reader(self, need_merge_files: List[DataFileMeta], deletion_ve if DataFileMeta.is_blob_file(first_file.file_name): field_ids = [self._get_field_id_from_write_cols(first_file)] elif DataFileMeta.is_vector_file(first_file.file_name): - field_ids = [bunch.field_id, SpecialFields.ROW_ID.id, - SpecialFields.SEQUENCE_NUMBER.id] + field_ids = self._get_field_ids_from_write_cols(first_file.write_cols) elif first_file.write_cols: - field_ids = self._get_field_ids_from_write_cols(first_file) + field_ids = self._get_field_ids_from_write_cols(first_file.write_cols) else: # For regular files without write_cols, derive field IDs from # the file's schema version, not the current table schema. @@ -1431,14 +1424,6 @@ def _create_union_reader(self, need_merge_files: List[DataFileMeta], deletion_ve blob_parallelism=self._blob_parallelism, logical_ranges=[bunch.logical_range()], ) - elif isinstance(bunch, VectorBunch): - vector_read = copy(self) - suppliers = [ - partial(vector_read._create_vector_segment_reader, - file, read_field_names, row_range, deletion_vector) - for row_range, file in bunch.segments() - ] - file_record_readers[i] = MergeAllBatchReader(suppliers, batch_size=batch_size) elif len(bunch.files()) == 1: suppliers = [lambda r=self._create_file_reader( bunch.files()[0], read_field_names, deletion_vector @@ -1501,16 +1486,6 @@ def _create_raw_blob_file_reader( file_size=file.file_size, ) - def _create_vector_segment_reader(self, file, read_fields, row_range, deletion_vector): - segment_read = copy(self) - segment_read.row_ranges = ( - [row_range] if self.row_ranges is None - else Range.and_([row_range], self.row_ranges) - ) - if not segment_read.row_ranges: - return EmptyRecordBatchReader() - return segment_read._create_file_reader(file, read_fields, deletion_vector) - def _split_field_bunches(self, need_merge_files: List[DataFileMeta]) -> List[FieldBunch]: """Split files into field bunches.""" @@ -1529,13 +1504,10 @@ def _split_field_bunches(self, need_merge_files: List[DataFileMeta]) -> List[Fie row_count, row_id_push_down, row_range) blob_bunch_map[field_id].add(file) elif DataFileMeta.is_vector_file(file.file_name): - for field_id in self._get_field_ids_from_write_cols(file): - if field_id in (SpecialFields.ROW_ID.id, SpecialFields.SEQUENCE_NUMBER.id): - continue - if field_id not in vector_bunch_map: - vector_bunch_map[field_id] = VectorBunch( - row_count, row_id_push_down, row_range, field_id) - vector_bunch_map[field_id].add(file) + field_id = self._get_field_id_from_write_cols(file) + if field_id not in vector_bunch_map: + vector_bunch_map[field_id] = VectorBunch(row_count, row_id_push_down) + vector_bunch_map[field_id].add(file) else: fields_files.append(DataBunch(file)) row_count = file.row_count @@ -1551,8 +1523,6 @@ def _split_field_bunches(self, need_merge_files: List[DataFileMeta]) -> List[Fie def _bunch_first_row_id(bunch: FieldBunch) -> int: if isinstance(bunch, BlobBunch): return bunch.logical_range().from_ - if isinstance(bunch, VectorBunch): - return bunch.segments()[0][0].from_ return bunch.files()[0].first_row_id def _get_field_id_from_write_cols(self, file: DataFileMeta) -> int: @@ -1560,17 +1530,17 @@ def _get_field_id_from_write_cols(self, file: DataFileMeta) -> int: if not file.write_cols or len(file.write_cols) == 0: raise ValueError("Blob/vector file must have write columns") - # write_cols names belong to the file's historical schema. + # Find the field by name in the table schema field_name = file.write_cols[0] - for field in self._resolve_schema(file.schema_id).fields: + for field in self.table.fields: if field.name == field_name: return field.id raise ValueError(f"Field {field_name} not found in table schema") - def _get_field_ids_from_write_cols(self, file: DataFileMeta) -> List[int]: + def _get_field_ids_from_write_cols(self, write_cols: List[str]) -> List[int]: field_ids = [] - for field_name in file.write_cols: - for field in self._resolve_schema(file.schema_id).fields: + for field_name in write_cols: + for field in self.table.fields: if field.name == field_name: field_ids.append(field.id) field_ids.append(SpecialFields.ROW_ID.id) diff --git a/paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py b/paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py deleted file mode 100644 index da9d47094f99..000000000000 --- a/paimon-python/pypaimon/tests/data_evolution_spanning_dedicated_test.py +++ /dev/null @@ -1,237 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import os -import tempfile -import unittest - -import pyarrow as pa - -from pypaimon import CatalogFactory, Schema -from pypaimon.manifest.schema.data_file_meta import DataFileMeta -from pypaimon.read.split import DataSplit -from pypaimon.schema.schema_change import SchemaChange -from pypaimon.utils.range import Range - - -class DataEvolutionSpanningDedicatedTest(unittest.TestCase): - def setUp(self): - self.tempdir = tempfile.TemporaryDirectory() - self.catalog = CatalogFactory.create({ - 'warehouse': os.path.join(self.tempdir.name, 'warehouse')}) - catalog = self.catalog - catalog.create_database('default', True) - schema = pa.schema([ - ('id', pa.int32()), - ('payload', pa.large_binary()), - ('embedding', pa.list_(pa.float32(), 2)), - ]) - catalog.create_table('default.spanning', Schema.from_pyarrow_schema( - schema, options={ - 'row-tracking.enabled': 'true', - 'data-evolution.enabled': 'true', - 'deletion-vectors.enabled': 'true', - 'vector.file.format': 'parquet', - 'read.batch-size': '2', - }), False) - self.table = catalog.get_table('default.spanning') - self.data = pa.Table.from_pydict({ - 'id': list(range(10)), - 'payload': [f'blob-{i}'.encode() for i in range(10)], - 'embedding': [[float(i), float(i + 1)] for i in range(10)], - }, schema=schema) - - # Reproduce the persistent layout after splitting only normal files. - # Readers must support it even with split-large-files absent or disabled. - self._write(self.table.copy({'target-file-row-num': '3'}), ['id'], self.data) - self._write(self.table, ['payload'], self.data.slice(0, 7), 0) - self._write(self.table, ['payload'], self.data.slice(7, 3), 7) - self._write(self.table, ['embedding'], self.data, 0) - - def tearDown(self): - self.tempdir.cleanup() - - @staticmethod - def _write(table, columns, data, first_row_id=None): - builder = table.new_batch_write_builder() - writer = builder.new_write().with_write_type(columns) - commit = builder.new_commit() - try: - writer.write_arrow(data.select(columns)) - messages = writer.prepare_commit() - if first_row_id is not None: - for message in messages: - for file in message.new_files: - file.first_row_id = first_row_id - commit.commit(messages) - finally: - writer.close() - commit.close() - - def _assert_rows(self, actual, expected_ids): - actual = actual.sort_by([('_ROW_ID', 'ascending')]) - self.assertEqual(actual['_ROW_ID'].to_pylist(), expected_ids) - self.assertEqual(actual['payload'].to_pylist(), - [f'blob-{i}'.encode() for i in expected_ids]) - self.assertEqual(actual['embedding'].to_pylist(), - [[float(i), float(i + 1)] for i in expected_ids]) - - def test_full_scan_projection_and_row_ranges(self): - builder = self.table.new_read_builder().with_projection( - ['payload', '_ROW_ID', 'embedding']) - splits = builder.new_scan().plan().splits() - self.assertEqual(sum(split.merged_row_count() for split in splits), 10) - self._assert_rows(builder.new_read().to_arrow(splits), list(range(10))) - - splits = builder.new_scan().with_row_ranges([Range(2, 7)]).plan().splits() - self._assert_rows(builder.new_read().to_arrow(splits), list(range(2, 8))) - - def test_deletions_and_chunk_shuffle_use_each_normal_anchor(self): - builder = self.table.new_batch_write_builder() - messages = builder.new_update().delete_by_row_id([2, 4, 8]) - commit = builder.new_commit() - try: - commit.commit(messages) - finally: - commit.close() - - expected = [0, 1, 3, 5, 6, 7, 9] - read_builder = self.table.new_read_builder().with_projection( - ['payload', '_ROW_ID', 'embedding']) - splits = read_builder.new_scan().plan().splits() - self.assertEqual(sum(split.merged_row_count() for split in splits), len(expected)) - self._assert_rows(read_builder.new_read().to_arrow(splits), expected) - - splits = read_builder.new_scan().with_row_ranges([Range(2, 7)]).plan().splits() - self._assert_rows(read_builder.new_read().to_arrow(splits), [3, 5, 6, 7]) - - chunks = read_builder.new_scan().with_chunk_shuffle( - seed=17, chunk_size=4).plan().splits() - self.assertEqual(sum(split.merged_row_count() for split in chunks), len(expected)) - for split in chunks: - names = [file.file_name for file in split.files] - self.assertEqual(len(names), len(set(names))) - self._assert_rows(read_builder.new_read().to_arrow(chunks), expected) - - def test_vector_middle_update_survives_normal_merge_and_rename(self): - self.catalog.alter_table('default.spanning', [ - SchemaChange.rename_column('embedding', 'renamed')], False) - self.table = self.catalog.get_table('default.spanning') - updated = self.data.slice(3, 3).set_column( - 2, self.data.schema.field('embedding'), - pa.array([[103.0, 104.0], [104.0, 105.0], [105.0, 106.0]], - type=self.data.schema.field('embedding').type)) - updated = updated.rename_columns(['id', 'payload', 'renamed']) - self._write(self.table, ['renamed'], updated, 3) - expected = [[float(i), float(i + 1)] for i in range(10)] - expected[3:6] = [[103.0, 104.0], [104.0, 105.0], [105.0, 106.0]] - read_builder = self.table.new_read_builder() - actual = read_builder.new_read().to_arrow( - read_builder.new_scan().plan().splits()).sort_by([('id', 'ascending')]) - self.assertEqual(actual['renamed'].to_pylist(), expected) - - self._merge_normal_files(self.table, self.data) - builder = self.table.new_read_builder().with_projection(['_ROW_ID', 'renamed']) - for selected in [None, [Range(1, 7)]]: - scan = builder.new_scan() - if selected is not None: - scan.with_row_ranges(selected) - actual = builder.new_read().to_arrow(scan.plan().splits()).sort_by( - [('_ROW_ID', 'ascending')]) - ids = list(range(10)) if selected is None else list(range(1, 8)) - self.assertEqual(actual['_ROW_ID'].to_pylist(), ids) - self.assertEqual(actual['renamed'].to_pylist(), [expected[i] for i in ids]) - - @staticmethod - def _merge_normal_files(table, data): - # Re-merge normal ranges while keeping every dedicated version unchanged. - old_files = [file for split in table.new_read_builder().new_scan().plan().splits() - for file in split.files - if not DataFileMeta.is_blob_file(file.file_name) - and not DataFileMeta.is_vector_file(file.file_name)] - builder = table.new_batch_write_builder() - writer = builder.new_write().with_write_type(['id']) - commit = builder.new_commit() - try: - writer.write_arrow(data.select(['id'])) - messages = writer.prepare_commit() - messages[0].deleted_files = old_files - for file in messages[0].new_files: - file.first_row_id = 0 - commit.commit(messages) - finally: - writer.close() - commit.close() - - def test_multi_vector_file_keeps_untouched_column_during_partial_update(self): - vector_type = pa.list_(pa.float32(), 2) - schema = pa.schema([('id', pa.int32()), ('left', vector_type), ('right', vector_type)]) - data = pa.Table.from_pydict({ - 'id': list(range(10)), - 'left': [[float(i), float(i + 1)] for i in range(10)], - 'right': [[float(i + 20), float(i + 21)] for i in range(10)], - }, schema=schema) - self.catalog.create_table('default.multi_vector', Schema.from_pyarrow_schema( - schema, options={ - 'row-tracking.enabled': 'true', - 'data-evolution.enabled': 'true', - 'vector.file.format': 'parquet', - }), False) - table = self.catalog.get_table('default.multi_vector') - self._write(table.copy({'target-file-row-num': '3'}), ['id'], data) - self._write(table, ['left', 'right'], data, 0) - vector_files = [file for split in table.new_read_builder().new_scan().plan().splits() - for file in split.files if DataFileMeta.is_vector_file(file.file_name)] - self.assertEqual([file.write_cols for file in vector_files], [['left', 'right']]) - - self.catalog.alter_table('default.multi_vector', [ - SchemaChange.rename_column('left', 'renamed')], False) - table = self.catalog.get_table('default.multi_vector') - updates = [[103.0, 104.0], [104.0, 105.0], [105.0, 106.0]] - self._write(table, ['renamed'], pa.Table.from_pydict( - {'renamed': updates}, schema=pa.schema([('renamed', vector_type)])), 3) - expected = data['left'].to_pylist() - expected[3:6] = updates - - for merge_normal_files in [False, True]: - if merge_normal_files: - self._merge_normal_files(table, data) - builder = table.new_read_builder().with_projection(['_ROW_ID', 'renamed', 'right']) - for selected in [None, [Range(1, 7)]]: - scan = builder.new_scan() - if selected is not None: - scan.with_row_ranges(selected) - actual = builder.new_read().to_arrow(scan.plan().splits()).sort_by( - [('_ROW_ID', 'ascending')]) - ids = list(range(10)) if selected is None else list(range(1, 8)) - self.assertEqual(actual['_ROW_ID'].to_pylist(), ids) - self.assertEqual(actual['renamed'].to_pylist(), [expected[i] for i in ids]) - self.assertEqual(actual['right'].to_pylist(), - [[float(i + 20), float(i + 21)] for i in ids]) - - # With DVs disabled, a vector-only projection may prune every normal file. - vector_builder = table.new_read_builder().with_projection(['renamed', 'right']) - splits = vector_builder.new_scan().plan().splits() - projected_splits = [DataSplit( - files=[file for file in split.files if DataFileMeta.is_vector_file(file.file_name)], - partition=split.partition, bucket=split.bucket, raw_convertible=False) - for split in splits] - for input_splits in [splits, projected_splits]: - actual = vector_builder.new_read().to_arrow(input_splits) - rows = sorted(actual.to_pylist(), key=lambda row: row['right'][0]) - self.assertEqual([row['renamed'] for row in rows], expected) - self.assertEqual([row['right'] for row in rows], data['right'].to_pylist()) diff --git a/paimon-python/pypaimon/tests/write/conflict_detection_test.py b/paimon-python/pypaimon/tests/write/conflict_detection_test.py index b6ec24f8572a..a82d9b0ee3a7 100644 --- a/paimon-python/pypaimon/tests/write/conflict_detection_test.py +++ b/paimon-python/pypaimon/tests/write/conflict_detection_test.py @@ -229,15 +229,13 @@ def _make_detection(self): def test_reports_dedicated_file_spanning_data_files(self): detection = self._make_detection() - detection._row_id_check_from_snapshot = 1 entries = [ _make_entry("f1", kind=0, first_row_id=0, row_count=2), _make_entry("f2", kind=0, first_row_id=2, row_count=2), _make_entry("p1.blob", kind=0, first_row_id=0, row_count=4), ] - result = detection.check_row_id_range_conflicts( - "APPEND", entries, entries[:2], entries[2:]) + result = detection.check_row_id_range_conflicts("COMPACT", entries) self.assertIsNotNone(result) self.assertIn("dedicated file", str(result)) @@ -253,7 +251,7 @@ def test_allows_adjacent_data_files(self): _make_entry("f2", kind=0, first_row_id=2, row_count=2), ] - result = detection.check_row_id_range_conflicts("COMPACT", entries, entries, []) + result = detection.check_row_id_range_conflicts("COMPACT", entries) self.assertIsNone(result) @@ -264,80 +262,10 @@ def test_allows_dedicated_file_covered_by_one_data_file(self): _make_entry("p1.blob", kind=0, first_row_id=1, row_count=2), ] - result = detection.check_row_id_range_conflicts( - "COMPACT", entries, entries[:1], entries[1:]) + result = detection.check_row_id_range_conflicts("COMPACT", entries) self.assertIsNone(result) - def test_normal_split_retains_dedicated_file(self): - for dedicated_file in ("retained.blob", "retained.vector.json"): - with self.subTest(dedicated_file=dedicated_file): - base = [ - _make_entry("normal", first_row_id=0, row_count=4), - _make_entry(dedicated_file, first_row_id=0, row_count=4), - ] - delta = [ - _make_entry("normal", kind=1, first_row_id=0, row_count=4), - _make_entry("split-1", first_row_id=0, row_count=2), - _make_entry("split-2", first_row_id=2, row_count=2), - ] - self.assertIsNone(self._make_detection().check_conflicts( - None, base, delta, "COMPACT")) - - def test_normal_compaction_retains_dedicated_file_outside_scanned_range(self): - for dedicated_file in ("retained.blob", "retained.vector.json"): - with self.subTest(dedicated_file=dedicated_file): - base = [ - _make_entry("normal", first_row_id=2, row_count=2), - _make_entry(dedicated_file, first_row_id=0, row_count=6), - ] - delta = [ - _make_entry("normal", kind=1, first_row_id=2, row_count=2), - _make_entry("split-1", first_row_id=2, row_count=1), - _make_entry("split-2", first_row_id=3, row_count=1), - ] - self.assertIsNone(self._make_detection().check_conflicts( - None, base, delta, "COMPACT")) - - def test_normal_compaction_rejects_gap_or_orphan_under_retained_dedicated_file(self): - for dedicated_file in ("retained.blob", "retained.vector.json"): - for remaining in ( - [], - [_make_entry("split-1", first_row_id=0, row_count=1), - _make_entry("split-2", first_row_id=2, row_count=2)]): - with self.subTest(dedicated_file=dedicated_file, remaining=remaining): - base = [ - _make_entry("normal", first_row_id=0, row_count=4), - _make_entry(dedicated_file, first_row_id=0, row_count=4), - ] - delta = [_make_entry("normal", kind=1, first_row_id=0, row_count=4)] + remaining - result = self._make_detection().check_conflicts(None, base, delta, "COMPACT") - self.assertIsNotNone(result) - self.assertIn("dedicated file", str(result)) - - def test_dedicated_compaction_may_span_split_normal_files(self): - for dedicated_file in ("compacted.blob", "compacted.vector.json"): - with self.subTest(dedicated_file=dedicated_file): - base = [ - _make_entry("split-1", first_row_id=0, row_count=2), - _make_entry("split-2", first_row_id=2, row_count=2), - ] - delta = [_make_entry(dedicated_file, first_row_id=0, row_count=4)] - self.assertIsNone(self._make_detection().check_conflicts( - None, base, delta, "COMPACT")) - - def test_merge_rejects_stale_file_across_split_normal_ranges(self): - for merge_file in ("merge-normal", "merge.blob", "merge.vector.json"): - with self.subTest(merge_file=merge_file): - detection = self._make_detection() - detection._row_id_check_from_snapshot = 1 - base = [ - _make_entry("split-1", first_row_id=0, row_count=2), - _make_entry("split-2", first_row_id=2, row_count=2), - ] - delta = [_make_entry(merge_file, first_row_id=0, row_count=4)] - self.assertIsNotNone(detection.check_conflicts(None, base, delta, "APPEND")) - class TestOverwriteConflictDetection(unittest.TestCase): diff --git a/paimon-python/pypaimon/utils/data_evolution_utils.py b/paimon-python/pypaimon/utils/data_evolution_utils.py index 1dcecd4e8007..06c5d3626563 100644 --- a/paimon-python/pypaimon/utils/data_evolution_utils.py +++ b/paimon-python/pypaimon/utils/data_evolution_utils.py @@ -17,53 +17,13 @@ """Utilities for data-evolution tables.""" -from bisect import bisect_left -from typing import Callable, Iterable, List, TypeVar +from typing import Callable, Iterable, TypeVar from pypaimon.manifest.schema.data_file_meta import DataFileMeta -from pypaimon.utils.range_helper import RangeHelper T = TypeVar("T") -def split_normal_file_groups(files: List[DataFileMeta]) -> List[List[DataFileMeta]]: - """Keep normal ranges separate and attach every intersecting dedicated file.""" - normal_files = [] - dedicated_files = [] - for file in files: - if DataFileMeta.is_blob_file(file.file_name) or DataFileMeta.is_vector_file(file.file_name): - dedicated_files.append(file) - else: - normal_files.append(file) - - helper = RangeHelper(lambda file: file.row_id_range()) - groups = helper.merge_overlapping_ranges(normal_files) - starts = [min(file.row_id_range().from_ for file in group) for group in groups] - ends = [max(file.row_id_range().to for file in group) for group in groups] - unassociated = [] - for file in dedicated_files: - file_range = file.row_id_range() - index = bisect_left(ends, file_range.from_) - associated = False - while index < len(groups) and starts[index] <= file_range.to: - groups[index].append(file) - associated = True - index += 1 - if not associated: - unassociated.append(file) - - groups.extend(helper.merge_overlapping_ranges(unassociated)) - - def group_start(group): - normal_starts = [file.row_id_range().from_ for file in group - if not DataFileMeta.is_blob_file(file.file_name) - and not DataFileMeta.is_vector_file(file.file_name)] - return min(normal_starts or [file.row_id_range().from_ for file in group]) - - groups.sort(key=group_start) - return groups - - def retrieve_anchor_file( entries: Iterable[T], file_meta_func: Callable[[T], DataFileMeta] = lambda entry: entry, diff --git a/paimon-python/pypaimon/write/commit/conflict_detection.py b/paimon-python/pypaimon/write/commit/conflict_detection.py index ec839e3b2230..2792c84fca36 100644 --- a/paimon-python/pypaimon/write/commit/conflict_detection.py +++ b/paimon-python/pypaimon/write/commit/conflict_detection.py @@ -270,8 +270,7 @@ def check_conflicts( if conflict is not None: return conflict - conflict = self.check_row_id_range_conflicts( - commit_kind, merged_entries, base_entries, delta_entries) + conflict = self.check_row_id_range_conflicts(commit_kind, merged_entries) if conflict is not None: return conflict @@ -568,8 +567,7 @@ def check_row_id_existence(self, base_entries, delta_entries, next_row_id=None): return None - def check_row_id_range_conflicts(self, commit_kind, commit_entries, - base_entries, delta_entries): + def check_row_id_range_conflicts(self, commit_kind, commit_entries): if not self.data_evolution_enabled: return None if self._row_id_check_from_snapshot is None and commit_kind != "COMPACT": @@ -599,7 +597,7 @@ def check_row_id_range_conflicts(self, commit_kind, commit_entries, if self._is_dedicated_file(entry.file.file_name) ] conflict = self._check_dedicated_file_row_id_range_conflicts( - commit_kind, data_files, dedicated_files, base_entries, delta_entries) + data_files, dedicated_files) if conflict is not None: return conflict @@ -618,35 +616,17 @@ def _check_data_file_row_id_range_conflicts(self, range_helper, data_files): return None def _check_dedicated_file_row_id_range_conflicts( - self, commit_kind, data_files, dedicated_files, base_entries, delta_entries): + self, data_files, dedicated_files): if not dedicated_files: return None data_ranges = self._data_file_row_ranges(data_files) - base_ranges = self._data_file_row_ranges([ - entry for entry in base_entries - if entry.file.first_row_id is not None - and not self._is_dedicated_file(entry.file.file_name) - ]) - added_files = {entry.identifier() for entry in delta_entries if entry.kind == 0} for dedicated_file in dedicated_files: dedicated_range = dedicated_file.file.row_id_range() if any(self._contains(row_range, dedicated_range) for row_range in data_ranges): continue - if dedicated_file.identifier() not in added_files: - # A normal-only compaction retains dedicated files even when their ranges span - # multiple output files. Partial scans only expose part of the base coverage. - previously_covered = Range.and_([dedicated_range], base_ranges) - if previously_covered and all( - not row_range.exclude(data_ranges) for row_range in previously_covered): - continue - elif commit_kind == "COMPACT" and not dedicated_range.exclude(data_ranges): - # Dedicated compaction may merge across normal-file boundaries, but newly - # written DML files still need to fit one range to reject stale writers. - continue - intersecting_ranges = [ row_range for row_range in data_ranges if row_range.overlaps(dedicated_range) diff --git a/paimon-python/pypaimon/write/table_delete.py b/paimon-python/pypaimon/write/table_delete.py index 6e0f4a7617ee..5c475c31b1f6 100644 --- a/paimon-python/pypaimon/write/table_delete.py +++ b/paimon-python/pypaimon/write/table_delete.py @@ -27,9 +27,12 @@ from pypaimon.manifest.index_manifest_entry import IndexManifestEntry from pypaimon.manifest.index_manifest_file import IndexManifestFile from pypaimon.manifest.schema.data_file_meta import DataFileMeta +from pypaimon.read.scanner.data_evolution_split_generator import ( + DataEvolutionSplitGenerator, +) from pypaimon.table.row.generic_row import GenericRow from pypaimon.table.source.deletion_file import DeletionFile -from pypaimon.utils.data_evolution_utils import retrieve_anchor_file, split_normal_file_groups +from pypaimon.utils.data_evolution_utils import retrieve_anchor_file from pypaimon.utils.file_store_path_factory import FileStorePathFactory from pypaimon.utils.range import Range from pypaimon.write.commit_message import CommitMessage @@ -141,7 +144,7 @@ def _scan_anchor_ranges(self) -> Tuple[int, List[_AnchorRange]]: for file in split.files if file.row_id_range() is not None ] - for group in split_normal_file_groups(files): + for group in DataEvolutionSplitGenerator._split_by_row_id(files): anchor = retrieve_anchor_file(group) anchors.append( _AnchorRange( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java index 832bd5478083..218dc6d0a958 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java @@ -98,6 +98,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -106,7 +107,9 @@ import scala.collection.Seq; import static org.apache.paimon.CoreOptions.createCommitUser; +import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile; import static org.apache.paimon.spark.utils.SparkProcedureUtils.readParallelism; +import static org.apache.paimon.types.VectorType.isVectorStoreFile; import static org.apache.paimon.utils.Preconditions.checkArgument; import static org.apache.spark.sql.types.DataTypes.StringType; @@ -665,6 +668,7 @@ static void executeDataEvolutionCompaction( return; } AtomicReference coordinatorRef = new AtomicReference<>(); + Set completedNormalFiles = new HashSet<>(); Function> taskPlanner = planningSnapshot -> { DataEvolutionCompactCoordinator coordinator = coordinatorRef.get(); @@ -685,11 +689,31 @@ static void executeDataEvolutionCompaction( false, planningSnapshot, candidateFilesPerBatch); + coordinator.withCompletedNormalFiles(completedNormalFiles); coordinatorRef.set(coordinator); } return filterIdlePartitions( coordinator.plan(), table, partitionPredicate, partitionIdleTime); }; + Consumer> commitObserver = null; + if (table.coreOptions().dataEvolutionCompactionSplitLargeFiles()) { + commitObserver = + messages -> { + // Output sizes can still trigger splitting or small-file merging. + // Skip bins containing only this invocation's completed normal outputs. + for (CommitMessage message : messages) { + for (DataFileMeta file : + ((CommitMessageImpl) message) + .compactIncrement() + .compactAfter()) { + if (!isBlobFile(file.fileName()) + && !isVectorStoreFile(file.fileName())) { + completedNormalFiles.add(file.fileName()); + } + } + } + }; + } DataEvolutionRewriteExecutor.execute( table, snapshot, @@ -699,7 +723,8 @@ static void executeDataEvolutionCompaction( commitConfigurer, relation == null ? null - : new DataEvolutionCompactMergeConflictRewriter(table, relation)::rewrite); + : new DataEvolutionCompactMergeConflictRewriter(table, relation)::rewrite, + commitObserver); } private static List filterIdlePartitions( diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java index b664b82f7eaf..e964f7b61c31 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java @@ -46,6 +46,7 @@ import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Function; import static org.apache.paimon.CoreOptions.createCommitUser; @@ -84,6 +85,26 @@ static void execute( SparkSession sparkSession, CommitConfigurer commitConfigurer, @Nullable CommitMessageRewriter commitMessageRewriter) { + execute( + table, + initialSnapshot, + taskPlanner, + javaSparkContext, + sparkSession, + commitConfigurer, + commitMessageRewriter, + null); + } + + static void execute( + FileStoreTable table, + Snapshot initialSnapshot, + Function> taskPlanner, + JavaSparkContext javaSparkContext, + SparkSession sparkSession, + CommitConfigurer commitConfigurer, + @Nullable CommitMessageRewriter commitMessageRewriter, + @Nullable Consumer> commitObserver) { CommitMessageSerializer messageSerializer = new CommitMessageSerializer(); String commitUser = createCommitUser(table.coreOptions().toConfiguration()); Snapshot preparationSnapshot = initialSnapshot; @@ -167,7 +188,8 @@ static void execute( commitUser, sparkSession, commitConfigurer, - commitMessageRewriter); + commitMessageRewriter, + commitObserver); checkArgument( committedSnapshot.id() > preparationSnapshot.id(), "Committed data evolution rewrite snapshot %s must be newer than preparation snapshot %s.", @@ -194,7 +216,8 @@ private static Snapshot commitWithMergeConflictRetry( String commitUser, SparkSession sparkSession, CommitConfigurer commitConfigurer, - @Nullable CommitMessageRewriter commitMessageRewriter) { + @Nullable CommitMessageRewriter commitMessageRewriter, + @Nullable Consumer> commitObserver) { int retryCount = 0; long startMillis = System.currentTimeMillis(); RetryWaiter retryWaiter = @@ -271,12 +294,18 @@ private static Snapshot commitWithMergeConflictRetry( } throw conflict; } - return table.snapshotManager() - .latestSnapshotOfUser(commitUser) - .orElseThrow( - () -> - new IllegalStateException( - "Cannot find the committed data evolution rewrite snapshot.")); + Snapshot committedSnapshot = + table.snapshotManager() + .latestSnapshotOfUser(commitUser) + .orElseThrow( + () -> + new IllegalStateException( + "Cannot find the committed data evolution rewrite snapshot.")); + if (commitObserver != null) { + // Include successful retry artifacts, not just the original staged outputs. + commitObserver.accept(preparedMessages); + } + return committedSnapshot; } catch (RuntimeException conflict) { if (commitMessageRewriter == null || !isMergeConflict(conflict) diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala index d9f9c3f865fb..fac1bcfdadea 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala @@ -35,6 +35,7 @@ import org.apache.paimon.spark.commands.{DataEvolutionCompactMergeConflictRewrit import org.apache.paimon.spark.commands.CompactRowIdRangeIndex import org.apache.paimon.spark.utils.SparkProcedureUtils import org.apache.paimon.table.FileStoreTable +import org.apache.paimon.table.sink.CommitMessageImpl import org.apache.paimon.table.source.{DataSplit, EndOfScanException, IncrementalSplit} import org.apache.paimon.table.source.snapshot.SnapshotReader import org.apache.paimon.utils.Range @@ -1764,6 +1765,56 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT } } + test("Paimon Procedure: split oversized files once per compact invocation across batches") { + withTable("T") { + sql(""" + |CREATE TABLE T (id INT, value STRING, pt STRING) + |TBLPROPERTIES ( + | 'bucket' = '-1', + | 'file.format' = 'avro', + | 'row-tracking.enabled' = 'true', + | 'data-evolution.enabled' = 'true', + | 'compaction.min.file-num' = '10') + |PARTITIONED BY (pt) + |""".stripMargin) + sql( + "INSERT INTO T SELECT /*+ REPARTITION(1) */ * FROM VALUES (1, 'a', 'p0'), (2, 'b', 'p0') AS S(id, value, pt)") + sql( + "INSERT INTO T SELECT /*+ REPARTITION(1) */ * FROM VALUES (3, 'c', 'p1'), (4, 'd', 'p1') AS S(id, value, pt)") + + val table = loadTable("T").copy( + Map( + "target-file-size" -> "1 b", + "data-evolution.compaction.split-large-files" -> "true").asJava) + assert(normalDataFiles(table).size == 2) + for (_ <- 0 until 2) { + val beforeFiles = normalDataFiles(table).map(_.fileName()).toSet + val beforeSnapshot = lastSnapshotId(table) + val attempts = new AtomicInteger() + CompactProcedure.executeDataEvolutionCompaction( + table, + null, + null, + null, + new JavaSparkContext(spark.sparkContext), + spark, + Int.box(1), + _ => assert(attempts.incrementAndGet() <= 2, "Recompacted this invocation's output") + ) + + val afterFiles = normalDataFiles(table) + assert(attempts.get() == 2) + assert(lastSnapshotId(table) == beforeSnapshot + 2) + assert(afterFiles.size == 2) + assert(afterFiles.forall(file => file.rowCount() == 2 && file.fileSize() > 2)) + assert(afterFiles.forall(file => !beforeFiles.contains(file.fileName()))) + } + checkAnswer( + sql("SELECT id, value, pt FROM T ORDER BY id"), + Seq(Row(1, "a", "p0"), Row(2, "b", "p0"), Row(3, "c", "p1"), Row(4, "d", "p1"))) + } + } + test("Paimon Procedure: materialize deletion vectors across planner batches") { withTable("T") { sql(""" @@ -1969,6 +2020,8 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT PaimonRelation.getPaimonRelation(spark.table("T").queryExecution.analyzed) val javaSparkContext = new JavaSparkContext(spark.sparkContext) val attempts = new AtomicInteger() + val observedCommits = new AtomicInteger() + val observedFiles = new util.HashSet[String]() val rewriteSnapshotId = new AtomicLong(-1L) val mergeFileAfterRewrite = new AtomicReference[DataFileMeta]() partialUpdate(table, "SELECT * FROM VALUES (1, 10), (2, 20) AS S(id, value)") @@ -2023,7 +2076,19 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT javaSparkContext, spark, configurer, - messageRewriter + messageRewriter, + messages => { + observedCommits.incrementAndGet() + messages.asScala.foreach { + message => + message + .asInstanceOf[CommitMessageImpl] + .compactIncrement() + .compactAfter() + .asScala + .foreach(file => observedFiles.add(file.fileName())) + } + } ) Assertions.assertThat(attempts.get()).isEqualTo(2) @@ -2036,6 +2101,10 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT file.fileName() != mergeFile.fileName()) assert(bridgeFiles.size == 1, bridgeFiles) val bridgeFile = bridgeFiles.head + assert(observedCommits.get() == 1) + assert(observedFiles.size() == 2) + assert(observedFiles.contains(bridgeFile.fileName())) + assert(!observedFiles.contains(mergeFile.fileName())) assert(bridgeFile.maxSequenceNumber() == rewriteSnapshotId.get()) assert(bridgeFile.maxSequenceNumber() < mergeFile.maxSequenceNumber()) assert(mergeFile.maxSequenceNumber() < table.latestSnapshot().get().id()) From 4d4d2072f91b17cd6b1c6d33ef851f3a102046d7 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Thu, 10 Sep 2026 10:59:08 +0800 Subject: [PATCH 5/5] [core] Plan normal compaction output ranges before writing --- docs/docs/multimodal-table/data-evolution.mdx | 16 +-- docs/generated/core_configuration.html | 2 +- .../java/org/apache/paimon/CoreOptions.java | 5 +- .../paimon/append/AppendOnlyWriter.java | 58 ++++------- .../DataEvolutionNormalCompactTask.java | 87 +++++++++------- .../paimon/io/RowDataRollingFileWriter.java | 49 ---------- .../paimon/append/AppendOnlyWriterTest.java | 21 ---- .../DataEvolutionCompactCoordinatorTest.java | 70 +++++++++++++ .../DataEvolutionNormalCompactTaskTest.java | 62 ++++++++---- .../paimon/io/RollingFileWriterTest.java | 98 ------------------- .../procedure/CompactProcedureTestBase.scala | 43 +++++--- 11 files changed, 223 insertions(+), 288 deletions(-) diff --git a/docs/docs/multimodal-table/data-evolution.mdx b/docs/docs/multimodal-table/data-evolution.mdx index cf53a31de27b..c02437b42813 100644 --- a/docs/docs/multimodal-table/data-evolution.mdx +++ b/docs/docs/multimodal-table/data-evolution.mdx @@ -629,9 +629,11 @@ including fractional values such as `1.5`. Files strictly exceeding the threshol qualify for compaction below `compaction.min.file-num` when their dedicated-file ranges allow splitting. Changing the ratio does not change the output target size. Compaction includes all -column updates for the same row-ID range and rolls normal output files near -`target-file-size`. Actual sizes depend on compression and the writer's size-check -granularity; the last file may be smaller. +column updates for the same row-ID range. Before writing, it estimates rows per output +from the total normal input file size, the logical row count, and `target-file-size`. +It then adjusts the estimated cut points to safe dedicated-file boundaries. +Actual output sizes can differ from the target because of compression, data skew, +overwritten column versions, and dedicated-file ranges; the last file may be smaller. The write-time `target-file-row-num` limit does not apply to compaction. Row IDs, column updates, and logical deletions are preserved. This option only rewrites @@ -641,10 +643,10 @@ their existing behavior. Files referenced by older snapshots or tags remain unti references expire and snapshot expiration removes them. Every BLOB or VECTOR file must remain fully contained in a single normal file's -row-ID range. After reaching the target size, compaction waits for a boundary that -does not cut through any dedicated file, including overlapping ranges from different -columns or versions. Output files may therefore exceed `target-file-size`. If these -ranges prevent any split, file size alone does not trigger a compaction task. Normal +row-ID range. An estimated cut inside a dedicated file moves to the end of its range, +including any overlapping ranges from different columns or versions and ranges produced +by dedicated compaction in the same batch. Output files may therefore exceed +`target-file-size`. If these ranges prevent any split, file size alone does not trigger a compaction task. Normal merging based on `compaction.min.file-num` remains available. ### Materialize Deletion Vectors diff --git a/docs/generated/core_configuration.html b/docs/generated/core_configuration.html index 4cc3744b86e1..bd231c88c047 100644 --- a/docs/generated/core_configuration.html +++ b/docs/generated/core_configuration.html @@ -522,7 +522,7 @@

data-evolution.compaction.split-large-files
false Boolean - Whether data-evolution compaction selects normal data files larger than data-evolution.compaction.large-file-ratio times target-file-size, even below compaction.min.file-num when dedicated-file ranges allow splitting. Normal output rolls toward target-file-size without cutting through any BLOB or VECTOR file range, so output may exceed the target. Row IDs and logical deletions are preserved, and associated BLOB and VECTOR files are not rewritten by this option. + Whether data-evolution compaction selects normal data files larger than data-evolution.compaction.large-file-ratio times target-file-size, even below compaction.min.file-num when dedicated-file ranges allow splitting. Normal output ranges are estimated from input file sizes and row counts toward target-file-size, then adjusted to avoid cutting through any BLOB or VECTOR file range. Actual output sizes may differ from the target. Row IDs and logical deletions are preserved, and associated BLOB and VECTOR files are not rewritten by this option.
data-evolution.enabled
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java index 68ca4c86fb23..41b338cb9e41 100644 --- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java @@ -2642,8 +2642,9 @@ public String toString() { "Whether data-evolution compaction selects normal data files larger than " + "data-evolution.compaction.large-file-ratio times target-file-size, " + "even below compaction.min.file-num when dedicated-file ranges allow splitting. " - + "Normal output rolls toward target-file-size without cutting through " - + "any BLOB or VECTOR file range, so output may exceed the target. " + + "Normal output ranges are estimated from input file sizes and row counts " + + "toward target-file-size, then adjusted to avoid cutting through any " + + "BLOB or VECTOR file range. Actual output sizes may differ from the target. " + "Row IDs and logical deletions are preserved, and associated " + "BLOB and VECTOR files are not rewritten by this option."); diff --git a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java index f70e440fa673..906573ad73ca 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java @@ -60,7 +60,6 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; -import java.util.function.LongPredicate; import java.util.function.Supplier; import static org.apache.paimon.types.VectorType.fieldsInVectorFile; @@ -104,8 +103,6 @@ public class AppendOnlyWriter implements BatchRecordWriter, MemoryOwner { private final MemorySize maxDiskSize; @Nullable private CompactDeletionFile compactDeletionFile; - @Nullable private LongPredicate fileRollingPredicate; - private boolean writeStarted; private SinkWriter sinkWriter; private MemorySegmentPool memorySegmentPool; @@ -189,20 +186,6 @@ public AppendOnlyWriter( } } - /** - * Restricts automatic rolling of normal files to accepted boundaries, expressed as the number - * of records written since the last flush. Requires direct writes and must be configured before - * writing. Explicit flushes still close the current file. - */ - public AppendOnlyWriter withFileRollingPredicate(LongPredicate predicate) { - Preconditions.checkState(!writeStarted, "Must configure rolling before writing."); - Preconditions.checkState( - sinkWriter instanceof DirectSinkWriter, - "File rolling predicate requires direct writes."); - this.fileRollingPredicate = Preconditions.checkNotNull(predicate); - return this; - } - private BufferedSinkWriter createBufferedSinkWriter(boolean spillable) { return new BufferedSinkWriter<>( this::createRollingRowWriter, @@ -222,7 +205,6 @@ public void write(InternalRow rowData) throws Exception { "Append-only writer can only accept insert or update_after row kind, but current row kind is: %s. " + "You can configure 'ignore-delete' to ignore retract records.", rowData.getRowKind()); - writeStarted = true; boolean success = sinkWriter.write(rowData); if (!success) { flush(false, false); @@ -238,7 +220,6 @@ public void write(InternalRow rowData) throws Exception { @Override public void writeBundle(BundleRecords bundle) throws Exception { - writeStarted = true; if (sinkWriter instanceof BufferedSinkWriter) { for (InternalRow row : bundle) { write(row); @@ -365,28 +346,23 @@ private RollingFileWriter createRollingRowWriter() { blobContext, omitAllNonDedicatedWriteCols); } - RowDataRollingFileWriter writer = - new RowDataRollingFileWriter( - fileIO, - schemaId, - fileFormat, - targetFileSize, - writeSchema, - pathFactory, - seqNumCounterProvider, - fileCompression, - statsCollectorFactories.statsCollectors(writeSchema.getFieldNames()), - fileIndexOptions, - fileSource, - asyncFileWrite, - statsDenseStore, - writeCols, - rowSidecarFileFormat, - targetFileRowNum); - if (fileRollingPredicate != null) { - writer.withFileRollingPredicate(fileRollingPredicate); - } - return writer; + return new RowDataRollingFileWriter( + fileIO, + schemaId, + fileFormat, + targetFileSize, + writeSchema, + pathFactory, + seqNumCounterProvider, + fileCompression, + statsCollectorFactories.statsCollectors(writeSchema.getFieldNames()), + fileIndexOptions, + fileSource, + asyncFileWrite, + statsDenseStore, + writeCols, + rowSidecarFileFormat, + targetFileRowNum); } private void trySyncLatestCompaction(boolean blocking) diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java index 8ad408a814b5..6a46a4425fa7 100644 --- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java +++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java @@ -20,13 +20,14 @@ import org.apache.paimon.AppendOnlyFileStore; import org.apache.paimon.CoreOptions; -import org.apache.paimon.append.AppendOnlyWriter; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.data.InternalRow; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.operation.AppendFileStoreWrite; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.reader.RecordReaderIterator; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -37,6 +38,7 @@ import org.apache.paimon.utils.FileStorePathFactory; import org.apache.paimon.utils.Pair; import org.apache.paimon.utils.Range; +import org.apache.paimon.utils.RecordWriter; import org.apache.paimon.utils.SetUtils; import org.slf4j.Logger; @@ -44,15 +46,14 @@ import javax.annotation.Nullable; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Function; -import java.util.function.LongPredicate; import java.util.stream.Collectors; import static org.apache.paimon.types.BlobType.fieldNamesInBlobFile; @@ -119,8 +120,7 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E if (options.dataEvolutionCompactionSplitLargeFiles()) { // Buffer flushes may close files before reaching a safe dedicated-file boundary. writeOptions.put(CoreOptions.WRITE_BUFFER_FOR_APPEND.key(), "false"); - writeOptions.put( - CoreOptions.TARGET_FILE_SIZE.key(), options.targetFileSize(false) + " b"); + writeOptions.put(CoreOptions.TARGET_FILE_SIZE.key(), Long.MAX_VALUE + " b"); } table = table.copy(writeOptions); long firstRowId = compactBefore.get(0).nonNullFirstRowId(); @@ -146,25 +146,27 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E AppendFileStoreWrite storeWrite = (AppendFileStoreWrite) store.newWrite(commitUser); storeWrite.withWriteType(readWriteType); storeWrite.withFileSource(FileSource.COMPACT); - AppendOnlyWriter writer = (AppendOnlyWriter) storeWrite.createWriter(partition, 0); - if (options.dataEvolutionCompactionSplitLargeFiles() && !protectedRanges.isEmpty()) { - writer.withFileRollingPredicate(fileRollingPredicate(firstRowId)); + RecordWriter writer = storeWrite.createWriter(partition, 0); + List outputRanges = + options.dataEvolutionCompactionSplitLargeFiles() + ? planOutputRanges(options.targetFileSize(false)) + : Collections.singletonList(checkContiguousRowRange(compactBefore)); + List writeResult = new ArrayList<>(); + try (RecordReaderIterator iterator = new RecordReaderIterator<>(reader)) { + for (Range range : outputRanges) { + for (long remaining = range.count(); remaining > 0; remaining--) { + checkArgument(iterator.hasNext(), "Missing rows in normal compaction input."); + writer.write(iterator.next()); + } + List output = + writer.prepareCommit(false).newFilesIncrement().newFiles(); + checkArgument( + output.size() == 1, + "Each planned compaction range should produce one normal file."); + writeResult.add(output.get(0)); + } + checkArgument(!iterator.hasNext(), "Unexpected extra rows in normal compaction input."); } - - reader.forEachRemaining( - row -> { - try { - writer.write(row); - } catch (Exception e) { - throw new RuntimeException(e); - } - }); - - List writeResult = writer.prepareCommit(false).newFilesIncrement().newFiles(); - checkArgument( - options.dataEvolutionCompactionSplitLargeFiles() || writeResult.size() == 1, - "Data evolution compaction should produce one file unless splitting is enabled."); - try { writer.close(); storeWrite.close(); @@ -198,20 +200,33 @@ public CommitMessage doCompact(FileStoreTable table, String commitUser) throws E return commitMessage(compactBefore, compactAfter); } - private LongPredicate fileRollingPredicate(long firstRowId) { - Iterator ranges = protectedRanges.iterator(); - return new LongPredicate() { - private Range current = ranges.hasNext() ? ranges.next() : null; - - @Override - public boolean test(long writtenRows) { - long lastRowId = firstRowId + (writtenRows - 1); - while (current != null && current.to <= lastRowId) { - current = ranges.hasNext() ? ranges.next() : null; - } - return current == null || lastRowId < current.from; + @VisibleForTesting + List planOutputRanges(long targetFileSize) { + Range inputRange = checkContiguousRowRange(compactBefore); + double inputSize = compactBefore.stream().mapToDouble(DataFileMeta::fileSize).sum(); + long targetRows = Math.max(1L, (long) (inputRange.count() * (targetFileSize / inputSize))); + List result = new ArrayList<>(); + int protectedIndex = 0; + long start = inputRange.from; + while (true) { + long end = start + Math.min(targetRows - 1, inputRange.to - start); + while (protectedIndex < protectedRanges.size() + && protectedRanges.get(protectedIndex).to <= end) { + protectedIndex++; + } + if (protectedIndex < protectedRanges.size() + && protectedRanges.get(protectedIndex).from <= end) { + end = protectedRanges.get(protectedIndex).to; } - }; + checkArgument( + end <= inputRange.to, + "Dedicated range must be contained in the normal compaction range."); + result.add(new Range(start, end)); + if (end == inputRange.to) { + return result; + } + start = end + 1; + } } @Nullable diff --git a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java index ec2aa8b68352..9b35c002aaf9 100644 --- a/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java +++ b/paimon-core/src/main/java/org/apache/paimon/io/RowDataRollingFileWriter.java @@ -28,21 +28,15 @@ import org.apache.paimon.statistics.SimpleColStatsCollector; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.LongCounter; -import org.apache.paimon.utils.Preconditions; import javax.annotation.Nullable; -import java.io.IOException; import java.util.List; -import java.util.function.LongPredicate; import java.util.function.Supplier; /** {@link RollingFileWriterImpl} for data files containing {@link InternalRow}. */ public class RowDataRollingFileWriter extends RollingFileWriterImpl { - @Nullable private LongPredicate fileRollingPredicate; - private boolean pendingRoll; - public RowDataRollingFileWriter( FileIO fileIO, long schemaId, @@ -100,47 +94,4 @@ public RowDataFileWriter get() { targetFileSize, targetFileRowNum); } - - /** - * Restricts automatic rolling to accepted boundaries, expressed as the cumulative number of - * records written by this writer. Must be configured before writing; closing the writer still - * closes the final file regardless of the predicate. - */ - public RowDataRollingFileWriter withFileRollingPredicate(LongPredicate predicate) { - Preconditions.checkState(recordCount() == 0, "Must configure rolling before writing."); - this.fileRollingPredicate = Preconditions.checkNotNull(predicate); - return this; - } - - @Override - protected void beforeWrite(InternalRow row) throws IOException { - if (pendingRoll && fileRollingPredicate.test(recordCount())) { - closeCurrentWriter(); - } - } - - @Override - protected void onRollingCondition(InternalRow row) throws IOException { - if (fileRollingPredicate == null || fileRollingPredicate.test(recordCount())) { - closeCurrentWriter(); - } else { - pendingRoll = true; - } - } - - @Override - protected void onCurrentWriterClosed() { - pendingRoll = false; - } - - @Override - public void writeBundle(BundleRecords bundle) throws IOException { - if (fileRollingPredicate == null) { - super.writeBundle(bundle); - } else { - for (InternalRow row : bundle) { - write(row); - } - } - } } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java index 4cb411f7a870..3b2464d311d3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java @@ -171,27 +171,6 @@ public void testSingleWrite() throws Exception { .isEqualTo(CoreOptions.FILE_FORMAT_AVRO); } - @Test - public void testFileRollingPredicate() throws Exception { - AppendOnlyWriter writer = - createEmptyWriter(64) - .withFileRollingPredicate(count -> count == 1250 || count == 2300); - for (int i = 0; i < 2500; i++) { - writer.write(row(i, "value", PART)); - } - CommitIncrement increment = writer.prepareCommit(true); - writer.close(); - - List files = increment.newFilesIncrement().newFiles(); - assertThat(files).extracting(DataFileMeta::rowCount).containsExactly(1250L, 1050L, 200L); - assertThat(files) - .extracting(DataFileMeta::minSequenceNumber) - .containsExactly(0L, 1250L, 2300L); - assertThat(files) - .extracting(DataFileMeta::maxSequenceNumber) - .containsExactly(1249L, 2299L, 2499L); - } - @Test public void testBinaryColumnStatsRoundTrip() throws Exception { RowType binarySchema = diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java index b9475b207673..0ce6a252a523 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactCoordinatorTest.java @@ -1127,6 +1127,76 @@ public void testNormalCompactTaskRejectsDisjointRowRanges() { .hasMessageContaining("contiguous row range"); } + @Test + public void testPlanNormalOutputRangesAtDedicatedBoundaries() { + List files = + Collections.singletonList(createDataFileMeta("file.parquet", 100, 10, 0, 1000)); + DataEvolutionNormalCompactTask task = + new DataEvolutionNormalCompactTask(BinaryRow.EMPTY_ROW, files); + assertThat(task.planOutputRanges(400)) + .containsExactly(new Range(100, 103), new Range(104, 107), new Range(108, 109)); + + task = + new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + files, + Arrays.asList( + new Range(105, 107), new Range(103, 106), new Range(108, 109))); + // A cut after 103 lies inside overlapping dedicated files and moves to 107. + // The adjacent dedicated file starting at 108 must not prevent this boundary. + assertThat(task.planOutputRanges(400)) + .containsExactly(new Range(100, 107), new Range(108, 109)); + // A cut immediately before a dedicated file is also safe. + assertThat(task.planOutputRanges(300)) + .containsExactly(new Range(100, 102), new Range(103, 107), new Range(108, 109)); + } + + @Test + public void testPlanNormalOutputRangesUsesLogicalRowCountAcrossVersions() { + DataEvolutionNormalCompactTask task = + new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + Arrays.asList( + createDataFileMeta("base.parquet", 100, 10, 0, 400), + createDataFileMeta("update.parquet", 100, 10, 1, 600))); + assertThat(task.planOutputRanges(500)) + .containsExactly(new Range(100, 104), new Range(105, 109)); + } + + @Test + public void testPlanNormalOutputRangesAvoidsSizeAndRowIdOverflow() { + DataEvolutionNormalCompactTask task = + new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + Arrays.asList( + createDataFileMeta( + "base.parquet", Long.MAX_VALUE - 10, 10, 0, Long.MAX_VALUE), + createDataFileMeta( + "update.parquet", + Long.MAX_VALUE - 10, + 10, + 1, + Long.MAX_VALUE))); + assertThat(task.planOutputRanges(Long.MAX_VALUE)) + .containsExactly( + new Range(Long.MAX_VALUE - 10, Long.MAX_VALUE - 6), + new Range(Long.MAX_VALUE - 5, Long.MAX_VALUE - 1)); + task = + new DataEvolutionNormalCompactTask( + BinaryRow.EMPTY_ROW, + Collections.singletonList( + createDataFileMeta( + "last.parquet", + Long.MAX_VALUE - 9, + 10, + 0, + Long.MAX_VALUE))); + assertThat(task.planOutputRanges(Long.MAX_VALUE / 2)) + .containsExactly( + new Range(Long.MAX_VALUE - 9, Long.MAX_VALUE - 5), + new Range(Long.MAX_VALUE - 4, Long.MAX_VALUE)); + } + @Test public void testSerializerMaterializeDeletionTask() throws IOException { DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer(); diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java index 38518d3a8572..6d15a8c345fa 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java @@ -245,8 +245,9 @@ public void testSplitHistoricalLargeFile(boolean updateColumn) throws Exception .isEmpty(); } - @Test - public void testSplitAtBlobBoundariesRetainsDedicatedFiles() throws Exception { + @ParameterizedTest + @ValueSource(ints = {1, 2000}) + public void testSplitAtBlobBoundariesRetainsDedicatedFiles(int estimatedRows) throws Exception { FileStoreTable table = createBlobSegmentsTable(); List dedicated = dedicatedFiles(table); assertThat(dedicated).hasSize(3); @@ -257,23 +258,52 @@ public void testSplitAtBlobBoundariesRetainsDedicatedFiles() throws Exception { assertThat(merged.compactAfter().get(0).nonNullRowIdRange()).isEqualTo(new Range(0, 3749)); options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "10"); - options.put(CoreOptions.TARGET_FILE_SIZE.key(), "1 b"); + long targetSize = + Math.max(1L, merged.compactAfter().get(0).fileSize() * estimatedRows / 3750); + options.put(CoreOptions.TARGET_FILE_SIZE.key(), targetSize + " b"); options.put(CoreOptions.WRITE_BUFFER_FOR_APPEND.key(), "true"); + options.put(CoreOptions.TARGET_FILE_ROW_NUM.key(), "10"); options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_SPLIT_LARGE_FILES.key(), "true"); + options.put(CoreOptions.DATA_EVOLUTION_COMPACTION_LARGE_FILE_RATIO.key(), "1.0"); table = table.copy(options); DataEvolutionCompactTask split = compactSingleTask(table); - // Size rolling is checked after 1000 rows. It must wait another 250 rows for the BLOB end. + // Estimated cuts move to BLOB ends; adjacent ranges can share a normal output. assertThat(split.compactAfter().stream().map(DataFileMeta::nonNullRowIdRange)) - .containsExactly(new Range(0, 1249), new Range(1250, 2499), new Range(2500, 3749)); + .containsExactlyElementsOf( + estimatedRows == 1 + ? Arrays.asList( + new Range(0, 1249), + new Range(1250, 2499), + new Range(2500, 3749)) + : Arrays.asList(new Range(0, 2499), new Range(2500, 3749))); assertDedicatedFilesContained(table, dedicated); assertBlobValues(table); - // Each resulting normal range is entirely protected, so another size-only pass is useless. + // Completed outputs must not be rewritten even if estimates leave them oversized. assertThat( new DataEvolutionCompactCoordinator( table, false, false, table.snapshotManager().latestSnapshot()) + .withCompletedNormalFiles( + split.compactAfter().stream() + .map(DataFileMeta::fileName) + .collect(Collectors.toSet())) + .plan()) + .isEmpty(); + + options.put(CoreOptions.TARGET_FILE_SIZE.key(), "128 mb"); + options.put(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "2"); + table = table.copy(options); + Snapshot snapshot = table.snapshotManager().latestSnapshot(); + assertThat(new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan()) + .hasSize(1); + assertThat( + new DataEvolutionCompactCoordinator(table, false, false, snapshot) + .withCompletedNormalFiles( + split.compactAfter().stream() + .map(DataFileMeta::fileName) + .collect(Collectors.toSet())) .plan()) .isEmpty(); } @@ -492,7 +522,7 @@ public void testFullRangeVectorPreventsSplit() throws Exception { } @Test - public void testCompletedParquetOutputsDoNotRepeatSmallFileCompaction() throws Exception { + public void testEstimatedRangesIgnoreParquetBufferSize() throws Exception { catalog.createTable( identifier(), Schema.newBuilder() @@ -534,9 +564,9 @@ public void testCompletedParquetOutputsDoNotRepeatSmallFileCompaction() throws E DataEvolutionCompactTask compacted = compactSingleTask(table); assertThat(compacted.compactBefore()).hasSize(3); List outputs = compacted.compactAfter(); - // Parquet reaches the byte target using an uncompressed page buffer at each BLOB end. - // Closing compresses the page, so all three outputs remain small enough to merge again. - assertThat(outputs).extracting(DataFileMeta::rowCount).containsExactly(1000L, 1000L, 1000L); + // The input files fit in the target based on their compressed sizes, even though + // Parquet's uncompressed page buffer crosses that target every 1000 rows. + assertThat(outputs).extracting(DataFileMeta::rowCount).containsExactly(3000L); assertThat(outputs) .allSatisfy( file -> @@ -545,17 +575,7 @@ public void testCompletedParquetOutputsDoNotRepeatSmallFileCompaction() throws E assertDedicatedFilesContained(table, dedicated); Snapshot snapshot = table.snapshotManager().latestSnapshot(); - List repeated = - new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan(); - assertThat(repeated).hasSize(1); - assertThat(repeated.get(0).compactBefore()).containsExactlyInAnyOrderElementsOf(outputs); - assertThat( - new DataEvolutionCompactCoordinator(table, false, false, snapshot) - .withCompletedNormalFiles( - outputs.stream() - .map(DataFileMeta::fileName) - .collect(Collectors.toSet())) - .plan()) + assertThat(new DataEvolutionCompactCoordinator(table, false, false, snapshot).plan()) .isEmpty(); List ids = new ArrayList<>(); diff --git a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java index 5d5bb6e5da2e..1676d7ebd42b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/RollingFileWriterTest.java @@ -171,104 +171,6 @@ public void testRollingByRowsWithBundle() throws IOException { assertThat(files.get(2).rowCount()).isEqualTo(30); } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testRollingAtSafeBoundaries(boolean bundled) throws IOException { - RowDataRollingFileWriter writer = - createRowDataWriter(Long.MAX_VALUE, 3) - .withFileRollingPredicate(count -> count == 5 || count == 6 || count == 9); - writeRows(writer, 12, bundled); - writer.close(); - - // Boundary 6 must not roll a fresh file below target; boundary 9 uses the cumulative - // count, and close retains the final short file. - assertThat(writer.result()).extracting(DataFileMeta::rowCount).containsExactly(5L, 4L, 3L); - assertWrittenRows(writer.result(), 12); - } - - @Test - public void testSizeRollingAtSafeBoundariesBetweenChecks() throws IOException { - RowDataRollingFileWriter writer = - createRowDataWriter(TARGET_FILE_SIZE, Long.MAX_VALUE) - .withFileRollingPredicate(count -> count == 1250 || count == 2300); - writeRows(writer, 2500, false); - writer.close(); - - // Avro checks size at rows 1000 and 2000. A pending roll must honor the next safe - // boundary without waiting for another size check. - assertThat(writer.result()) - .extracting(DataFileMeta::rowCount) - .containsExactly(1250L, 1050L, 200L); - assertWrittenRows(writer.result(), 2500); - } - - @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testRowDataRollingWithoutPredicate(boolean bundled) throws IOException { - RowDataRollingFileWriter writer = createRowDataWriter(Long.MAX_VALUE, 3); - writeRows(writer, 12, bundled); - writer.close(); - - assertThat(writer.result()) - .extracting(DataFileMeta::rowCount) - .containsExactly(bundled ? new Long[] {12L} : new Long[] {3L, 3L, 3L, 3L}); - assertWrittenRows(writer.result(), 12); - } - - private RowDataRollingFileWriter createRowDataWriter( - long targetFileSize, long targetFileRowNum) { - return new RowDataRollingFileWriter( - LocalFileIO.create(), - 0L, - FileFormat.fromIdentifier("avro", new Options()), - targetFileSize, - SCHEMA, - new DataFilePathFactory( - new Path(tempDir + "/bucket-0"), - "avro", - CoreOptions.DATA_FILE_PREFIX.defaultValue(), - CoreOptions.CHANGELOG_FILE_PREFIX.defaultValue(), - CoreOptions.FILE_SUFFIX_INCLUDE_COMPRESSION.defaultValue(), - CoreOptions.FILE_COMPRESSION.defaultValue(), - null), - () -> new LongCounter(0), - CoreOptions.FILE_COMPRESSION.defaultValue(), - SimpleColStatsCollector.createFullStatsFactories(SCHEMA.getFieldCount()), - new FileIndexOptions(), - FileSource.APPEND, - true, - false, - null, - null, - targetFileRowNum); - } - - private static void writeRows(RowDataRollingFileWriter writer, int count, boolean bundled) - throws IOException { - if (bundled) { - writer.writeBundle(bundle(count)); - } else { - for (int i = 0; i < count; i++) { - writer.write(GenericRow.of(i)); - } - } - } - - private void assertWrittenRows(List files, int count) throws IOException { - List actual = new ArrayList<>(); - for (DataFileMeta file : files) { - actual.addAll( - readIntsFromRowFile( - FileFormat.fromIdentifier("avro", new Options()), - new Path(tempDir + "/bucket-0/" + file.fileName()))); - } - List expected = new ArrayList<>(); - for (int i = 0; i < count; i++) { - expected.add(i); - } - assertThat(actual).containsExactlyElementsOf(expected); - } - private static SingleUseBundleRecords bundle(int rowCount) { List rows = new ArrayList<>(); for (int i = 0; i < rowCount; i++) { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala index fac1bcfdadea..5f57058fee53 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CompactProcedureTestBase.scala @@ -1768,23 +1768,41 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT test("Paimon Procedure: split oversized files once per compact invocation across batches") { withTable("T") { sql(""" - |CREATE TABLE T (id INT, value STRING, pt STRING) + |CREATE TABLE T (id INT, value STRING, picture BINARY, pt STRING) |TBLPROPERTIES ( | 'bucket' = '-1', | 'file.format' = 'avro', | 'row-tracking.enabled' = 'true', | 'data-evolution.enabled' = 'true', - | 'compaction.min.file-num' = '10') + | 'blob-field' = 'picture', + | 'compaction.min.file-num' = '2') |PARTITIONED BY (pt) |""".stripMargin) - sql( - "INSERT INTO T SELECT /*+ REPARTITION(1) */ * FROM VALUES (1, 'a', 'p0'), (2, 'b', 'p0') AS S(id, value, pt)") - sql( - "INSERT INTO T SELECT /*+ REPARTITION(1) */ * FROM VALUES (3, 'c', 'p1'), (4, 'd', 'p1') AS S(id, value, pt)") - - val table = loadTable("T").copy( + for (pt <- Seq("p0", "p1")) { + sql(s""" + |INSERT INTO T SELECT /*+ REPARTITION(1) */ + |id, concat('value-', id), CAST('blob' AS BINARY), '$pt' FROM range(0, 1) + |""".stripMargin) + sql(s""" + |INSERT INTO T SELECT /*+ REPARTITION(1) */ + |id, concat('value-', id), CAST('blob' AS BINARY), '$pt' FROM range(1, 100) + |""".stripMargin) + } + val initial = loadTable("T") + CompactProcedure.executeDataEvolutionCompaction( + initial, + null, + null, + new JavaSparkContext(spark.sparkContext), + spark, + Int.box(2)) + val targetSize = normalDataFiles(initial).map(_.fileSize()).min / 4 + // Each normal file retains BLOB ranges for one row and 99 rows. The estimated + // cut lands inside the latter, so the safe output remains the full 100 rows. + val table = initial.copy( Map( - "target-file-size" -> "1 b", + "target-file-size" -> s"$targetSize b", + "compaction.min.file-num" -> "10", "data-evolution.compaction.split-large-files" -> "true").asJava) assert(normalDataFiles(table).size == 2) for (_ <- 0 until 2) { @@ -1806,12 +1824,13 @@ abstract class CompactProcedureTestBase extends PaimonSparkTestBase with StreamT assert(attempts.get() == 2) assert(lastSnapshotId(table) == beforeSnapshot + 2) assert(afterFiles.size == 2) - assert(afterFiles.forall(file => file.rowCount() == 2 && file.fileSize() > 2)) + assert( + afterFiles.forall(file => file.rowCount() == 100 && file.fileSize() > 2 * targetSize)) assert(afterFiles.forall(file => !beforeFiles.contains(file.fileName()))) } checkAnswer( - sql("SELECT id, value, pt FROM T ORDER BY id"), - Seq(Row(1, "a", "p0"), Row(2, "b", "p0"), Row(3, "c", "p1"), Row(4, "d", "p1"))) + sql("SELECT id, value, pt FROM T ORDER BY id, pt"), + (0 until 100).flatMap(id => Seq("p0", "p1").map(pt => Row(id, s"value-$id", pt)))) } }