From 444b80006e905913ce6638c5c7e6503cba929568 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 20 Oct 2026 01:26:55 -0700 Subject: [PATCH 1/5] feat(core): support structured-data local cache selection Cache Parquet, ORC and Avro data blocks through the existing local-cache whitelist without caching BLOB bodies. Preserve data, wildcard and default behavior, with metadata/index precedence and mutable-file bypass. --- crates/paimon/src/io/cache/file_type.rs | 108 ++++++++++++++++++++++-- crates/paimon/src/io/cache/mod.rs | 25 +++++- crates/paimon/src/io/file_io.rs | 97 ++++++++++++++++++++- docs/src/getting-started.md | 8 +- 4 files changed, 227 insertions(+), 11 deletions(-) diff --git a/crates/paimon/src/io/cache/file_type.rs b/crates/paimon/src/io/cache/file_type.rs index e01a6e15e..b2193ef36 100644 --- a/crates/paimon/src/io/cache/file_type.rs +++ b/crates/paimon/src/io/cache/file_type.rs @@ -21,25 +21,60 @@ use std::collections::HashSet; // Suffix `.{uuid}.tmp` appended by Java `Path.createTempPath()`: 1 dot + 36-char UUID + `.tmp`. const TEMP_FILE_SUFFIX_LEN: usize = 41; +// Keep this list aligned with the non-BLOB/non-video formats accepted by +// `arrow::format::create_format_reader_with_budget`. Text formats also support +// an optional compression suffix. +const STRUCTURED_DATA_SUFFIXES: [&str; 27] = [ + ".parquet", + ".orc", + ".avro", + ".row", + ".mosaic", + ".vortex", + ".csv", + ".json", + ".text", + ".csv.gz", + ".csv.bz2", + ".csv.deflate", + ".csv.snappy", + ".csv.lz4", + ".csv.zst", + ".json.gz", + ".json.bz2", + ".json.deflate", + ".json.snappy", + ".json.lz4", + ".json.zst", + ".text.gz", + ".text.bz2", + ".text.deflate", + ".text.snappy", + ".text.lz4", + ".text.zst", +]; + #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(super) enum FileType { Meta, Data, + StructuredData, BucketIndex, GlobalIndex, FileIndex, } impl FileType { - const ALL: [Self; 5] = [ + const ALL: [Self; 6] = [ Self::Meta, Self::Data, + Self::StructuredData, Self::BucketIndex, Self::GlobalIndex, Self::FileIndex, ]; - /// Mirrors Java `org.apache.paimon.utils.FileType#classify`, including its check order. + /// Keeps metadata/index classification ahead of structured data detection. pub(super) fn classify(path: &str) -> Self { let mut segments = path.rsplit('/'); let name = unwrap_temp_file_name(segments.next().unwrap_or(path)); @@ -78,7 +113,15 @@ impl FileType { return Self::Meta; } - Self::Data + let lower_name = name.to_ascii_lowercase(); + if STRUCTURED_DATA_SUFFIXES + .iter() + .any(|suffix| lower_name.len() > suffix.len() && lower_name.ends_with(suffix)) + { + Self::StructuredData + } else { + Self::Data + } } pub(super) fn is_mutable(path: &str) -> bool { @@ -107,13 +150,15 @@ impl FileType { "meta" => &[Self::Meta], "global-index" => &[Self::GlobalIndex], "bucket-index" => &[Self::BucketIndex], - "data" => &[Self::Data], + // Preserve the existing broad data selector. + "data" => &[Self::Data, Self::StructuredData], + "structured-data" => &[Self::StructuredData], "file-index" => &[Self::FileIndex], "" => &[], unknown => { log::warn!( "Unknown local-cache.whitelist value '{}'; supported values are \ - meta, global-index, bucket-index, data, file-index, \ + meta, global-index, bucket-index, data, structured-data, file-index, \ or * for all of them", unknown ); @@ -168,7 +213,7 @@ mod tests { ), ( "s3://bucket/table/bucket-0/data-abc.parquet", - FileType::Data, + FileType::StructuredData, ), ( "s3://bucket/table/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-0.avro.sidecar", @@ -182,7 +227,7 @@ mod tests { ("s3://bucket/table/changelog/changelog-5", FileType::Meta), ( "s3://bucket/table/bucket-0/changelog-123e4567-e89b-12d3-a456-426614174000-0.parquet", - FileType::Data, + FileType::StructuredData, ), ( "s3://bucket/table/snapshot/.snapshot-13.123e4567-e89b-12d3-a456-426614174000.tmp", @@ -194,7 +239,7 @@ mod tests { ), ( "s3://bucket/table/bucket-0/.data-abc.parquet.123e4567-e89b-12d3-a456-426614174000.tmp", - FileType::Data, + FileType::StructuredData, ), ]; @@ -248,6 +293,52 @@ mod tests { } } + #[test] + fn test_structured_data_classification_keeps_metadata_and_indexes_separate() { + for name in [ + "data.parquet", + "data.ORC", + "data.avro", + "data.row", + "data.mosaic", + "data.vortex", + "data.csv", + "data.json", + "data.text", + "data.csv.gz", + "data.JSON.ZST", + "data.text.snappy", + ] { + assert_eq!(FileType::classify(name), FileType::StructuredData); + } + for name in [ + "data.blob", + "data.BLOB", + "data.unknown", + "dir.parquet/data", + "data.parquet.blob", + "data.video", + ".parquet", + ".csv.gz", + ] { + assert_eq!(FileType::classify(name), FileType::Data); + } + assert_eq!(FileType::classify("manifest-1.avro"), FileType::Meta); + assert_eq!(FileType::classify("snapshot-1.parquet"), FileType::Meta); + assert_eq!( + FileType::classify("data.parquet.index"), + FileType::FileIndex + ); + assert_eq!( + FileType::parse_whitelist("structured-data"), + HashSet::from([FileType::StructuredData]) + ); + assert_eq!( + FileType::parse_whitelist("data"), + HashSet::from([FileType::Data, FileType::StructuredData]) + ); + } + #[test] fn test_file_type_parses_whitelist() { let all = HashSet::from([ @@ -255,6 +346,7 @@ mod tests { FileType::GlobalIndex, FileType::BucketIndex, FileType::Data, + FileType::StructuredData, FileType::FileIndex, ]); diff --git a/crates/paimon/src/io/cache/mod.rs b/crates/paimon/src/io/cache/mod.rs index 061cec191..8fc1ad631 100644 --- a/crates/paimon/src/io/cache/mod.rs +++ b/crates/paimon/src/io/cache/mod.rs @@ -409,7 +409,11 @@ mod tests { assert_eq!(config.block_size, 64 * 1024); assert_eq!( config.whitelist, - std::collections::HashSet::from([FileType::Meta, FileType::Data]) + std::collections::HashSet::from([ + FileType::Meta, + FileType::Data, + FileType::StructuredData + ]) ); } @@ -485,6 +489,25 @@ mod tests { assert_eq!(cache.file_size(path, &token).await, None); } + #[test] + fn test_structured_data_selection_excludes_mutable_files() { + let mut options = Options::new(); + options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); + options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "structured-data"); + let cache = create_local_cache(&options).unwrap().unwrap(); + assert!(cache.is_cacheable("s3://bucket/data.parquet")); + for path in [ + "s3://bucket/data.parquet.tmp", + "s3://bucket/.data.parquet.123e4567-e89b-12d3-a456-426614174000.tmp", + "s3://bucket/data.tmp.parquet", + "s3://bucket/manifest-1.avro", + "s3://bucket/data.parquet.index", + "s3://bucket/data.blob", + ] { + assert!(!cache.is_cacheable(path), "{path}"); + } + } + #[tokio::test] async fn test_local_cache_file_size_is_invalidated_across_shared_instances() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index e65ffadda..49ea1a34d 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -268,7 +268,7 @@ impl FileIO { /// `block_size` controls the aligned ranges presented to the cache. /// `whitelist` uses the same comma-separated values as /// `local-cache.whitelist`: `meta`, `global-index`, `bucket-index`, `data`, - /// and `file-index`, or `*` for all of them. + /// `structured-data`, and `file-index`, or `*` for all of them. pub fn with_file_block_cache( mut self, cache: Arc, @@ -2426,6 +2426,101 @@ mod input_output_test { } } + #[tokio::test] + async fn test_structured_data_cache_preserves_existing_selectors() { + for disk in [false, true] { + for whitelist in [ + "meta,global-index", + "data", + "structured-data", + "meta,global-index,structured-data", + "*", + ] { + let directory = tempfile::tempdir().unwrap(); + let mut options = Options::new(); + options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); + options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, whitelist); + options.set(CatalogOptions::LOCAL_CACHE_BLOCK_SIZE, "4"); + if disk { + options.set( + CatalogOptions::LOCAL_CACHE_DIR, + directory.path().to_string_lossy(), + ); + } + let cache = Arc::new( + LocalCache::new(LocalCacheConfig::from_options(&options).unwrap().unwrap()) + .unwrap(), + ); + let file_io = FileIOBuilder::new("memory") + .with_local_cache(cache) + .build() + .unwrap(); + for name in [ + "data.parquet", + "data.blob", + "data.orc", + "data.avro", + "data.row", + "data.mosaic", + "data.vortex", + "data.csv", + "data.json.zst", + "data.text.snappy", + "data.video", + "data.PARQUET", + "data.unknown", + "data.parquet.index", + "snapshot-1", + "snapshot-1.blob", + ] { + let path = format!("memory:/{name}"); + file_io + .new_output(&path) + .unwrap() + .write(Bytes::from_static(b"abcdefgh")) + .await + .unwrap(); + let input = file_io.new_input(&path).unwrap(); + let reader = input.reader().await.unwrap(); + assert_eq!(reader.read(1..7).await.unwrap(), b"bcdefg"[..]); + drop(reader); + let (op, relative_path, _) = input.source.resolve(&path).await.unwrap(); + op.delete(&relative_path).await.unwrap(); + let cached = match whitelist { + "*" => true, + "data" => name.starts_with("data.") && !name.ends_with(".index"), + "structured-data" => matches!( + name, + "data.parquet" + | "data.orc" + | "data.avro" + | "data.row" + | "data.mosaic" + | "data.vortex" + | "data.csv" + | "data.json.zst" + | "data.text.snappy" + | "data.PARQUET" + ), + "meta,global-index,structured-data" => !matches!( + name, + "data.blob" | "data.video" | "data.unknown" | "data.parquet.index" + ), + _ => name.starts_with("snapshot-"), + }; + let result = input.read().await; + if cached { + assert_eq!(result.unwrap(), b"abcdefgh"[..], "{whitelist}: {name}"); + let reader = input.reader().await.unwrap(); + assert_eq!(reader.read(1..7).await.unwrap(), b"bcdefg"[..]); + } else { + assert!(result.is_err(), "{whitelist}: {name}"); + } + } + } + } + } + async fn common_test_output_file_write_and_read(file_io: &FileIO, path: &str) { let output = file_io.new_output(path).unwrap(); let mut writer = output.writer().await.unwrap(); diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 40216050f..fd749b746 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -403,7 +403,13 @@ let catalog = CatalogFactory::create(options).await?; | `local-cache.dir` | none | Optional base directory. When set, Paimon uses a persistent disk cache in a private versioned child directory; otherwise it uses memory. | | `local-cache.max-size` | unlimited | Maximum cache size. Memory caches count payload bytes; disk caches count encoded bytes. Values accept byte units such as `512 MiB` or `20 GiB`. | | `local-cache.block-size` | `1 MiB` | Block size used for cached range reads. | -| `local-cache.whitelist` | `meta,global-index` | Comma-separated eligible types: `meta`, `global-index`, `bucket-index`, `data`, and `file-index`, or `*` for all of them. | +| `local-cache.whitelist` | `meta,global-index` | Comma-separated eligible types: `meta`, `global-index`, `bucket-index`, `data`, `structured-data`, and `file-index`, or `*` for all of them. | + +Use `meta,global-index,structured-data` to cache supported structured data +without BLOB or video bodies. It includes Parquet, ORC, Avro, ROW, Mosaic, +Vortex, CSV, JSON and TEXT files, including supported compressed text suffixes. +Matching is case-insensitive. `data` and `*` retain their broader behavior; the +default whitelist is unchanged. Each catalog owns its in-memory cache for the catalog's lifetime. Disk caches are reused after process restarts. Cache keys include a catalog-configuration From 315fdc9a483fadf465210625d0349adc28d2a4fe Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 20 Oct 2026 02:07:32 -0700 Subject: [PATCH 2/5] fix(core): exclude dedicated files from local cache --- crates/paimon/src/common/options.rs | 8 +++ crates/paimon/src/io/cache/file_type.rs | 87 ++++++++----------------- crates/paimon/src/io/cache/mod.rs | 55 ++++++++++++++-- crates/paimon/src/io/file_io.rs | 60 ++++++++--------- docs/src/getting-started.md | 11 ++-- 5 files changed, 121 insertions(+), 100 deletions(-) diff --git a/crates/paimon/src/common/options.rs b/crates/paimon/src/common/options.rs index 86391377c..a7c463166 100644 --- a/crates/paimon/src/common/options.rs +++ b/crates/paimon/src/common/options.rs @@ -91,6 +91,10 @@ impl CatalogOptions { /// Comma-separated file types eligible for local caching. pub const LOCAL_CACHE_WHITELIST: &'static str = "local-cache.whitelist"; + /// Exclude dedicated BLOB, video, and vector-store files from the local cache. + pub const LOCAL_CACHE_EXCLUDE_DEDICATED_FILES: &'static str = + "local-cache.exclude-dedicated-files"; + /// Maximum process-local cache size for parsed file-format metadata. pub const FILE_FORMAT_METADATA_CACHE_MAX_SIZE: &'static str = "file-format.metadata-cache.max-size"; @@ -301,6 +305,10 @@ mod tests { CatalogOptions::LOCAL_CACHE_WHITELIST, "local-cache.whitelist" ); + assert_eq!( + CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, + "local-cache.exclude-dedicated-files" + ); assert_eq!( CatalogOptions::FILE_FORMAT_METADATA_CACHE_MAX_SIZE, "file-format.metadata-cache.max-size" diff --git a/crates/paimon/src/io/cache/file_type.rs b/crates/paimon/src/io/cache/file_type.rs index b2193ef36..e12c4396f 100644 --- a/crates/paimon/src/io/cache/file_type.rs +++ b/crates/paimon/src/io/cache/file_type.rs @@ -21,44 +21,11 @@ use std::collections::HashSet; // Suffix `.{uuid}.tmp` appended by Java `Path.createTempPath()`: 1 dot + 36-char UUID + `.tmp`. const TEMP_FILE_SUFFIX_LEN: usize = 41; -// Keep this list aligned with the non-BLOB/non-video formats accepted by -// `arrow::format::create_format_reader_with_budget`. Text formats also support -// an optional compression suffix. -const STRUCTURED_DATA_SUFFIXES: [&str; 27] = [ - ".parquet", - ".orc", - ".avro", - ".row", - ".mosaic", - ".vortex", - ".csv", - ".json", - ".text", - ".csv.gz", - ".csv.bz2", - ".csv.deflate", - ".csv.snappy", - ".csv.lz4", - ".csv.zst", - ".json.gz", - ".json.bz2", - ".json.deflate", - ".json.snappy", - ".json.lz4", - ".json.zst", - ".text.gz", - ".text.bz2", - ".text.deflate", - ".text.snappy", - ".text.lz4", - ".text.zst", -]; - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(super) enum FileType { Meta, Data, - StructuredData, + DedicatedData, BucketIndex, GlobalIndex, FileIndex, @@ -68,13 +35,13 @@ impl FileType { const ALL: [Self; 6] = [ Self::Meta, Self::Data, - Self::StructuredData, + Self::DedicatedData, Self::BucketIndex, Self::GlobalIndex, Self::FileIndex, ]; - /// Keeps metadata/index classification ahead of structured data detection. + /// Keeps metadata/index classification ahead of dedicated data detection. pub(super) fn classify(path: &str) -> Self { let mut segments = path.rsplit('/'); let name = unwrap_temp_file_name(segments.next().unwrap_or(path)); @@ -113,17 +80,19 @@ impl FileType { return Self::Meta; } - let lower_name = name.to_ascii_lowercase(); - if STRUCTURED_DATA_SUFFIXES - .iter() - .any(|suffix| lower_name.len() > suffix.len() && lower_name.ends_with(suffix)) - { - Self::StructuredData + if Self::is_dedicated_file(name) { + Self::DedicatedData } else { Self::Data } } + /// Matches Java's BLOB/video and vector-store file naming rules. + fn is_dedicated_file(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + name.ends_with(".blob") || name.ends_with(".video") || name.contains(".vector.") + } + pub(super) fn is_mutable(path: &str) -> bool { let name = path.rsplit('/').next().unwrap_or(path); // Iceberg-compatible `version-hint.text`, `retire-pending` and `v{N}.metadata.json` @@ -151,14 +120,13 @@ impl FileType { "global-index" => &[Self::GlobalIndex], "bucket-index" => &[Self::BucketIndex], // Preserve the existing broad data selector. - "data" => &[Self::Data, Self::StructuredData], - "structured-data" => &[Self::StructuredData], + "data" => &[Self::Data, Self::DedicatedData], "file-index" => &[Self::FileIndex], "" => &[], unknown => { log::warn!( "Unknown local-cache.whitelist value '{}'; supported values are \ - meta, global-index, bucket-index, data, structured-data, file-index, \ + meta, global-index, bucket-index, data, file-index, \ or * for all of them", unknown ); @@ -213,7 +181,7 @@ mod tests { ), ( "s3://bucket/table/bucket-0/data-abc.parquet", - FileType::StructuredData, + FileType::Data, ), ( "s3://bucket/table/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-0.avro.sidecar", @@ -227,7 +195,7 @@ mod tests { ("s3://bucket/table/changelog/changelog-5", FileType::Meta), ( "s3://bucket/table/bucket-0/changelog-123e4567-e89b-12d3-a456-426614174000-0.parquet", - FileType::StructuredData, + FileType::Data, ), ( "s3://bucket/table/snapshot/.snapshot-13.123e4567-e89b-12d3-a456-426614174000.tmp", @@ -239,7 +207,7 @@ mod tests { ), ( "s3://bucket/table/bucket-0/.data-abc.parquet.123e4567-e89b-12d3-a456-426614174000.tmp", - FileType::StructuredData, + FileType::Data, ), ]; @@ -294,7 +262,7 @@ mod tests { } #[test] - fn test_structured_data_classification_keeps_metadata_and_indexes_separate() { + fn test_dedicated_file_classification_keeps_metadata_and_indexes_separate() { for name in [ "data.parquet", "data.ORC", @@ -308,20 +276,21 @@ mod tests { "data.csv.gz", "data.JSON.ZST", "data.text.snappy", + "data.unknown", + "dir.parquet/data", ] { - assert_eq!(FileType::classify(name), FileType::StructuredData); + assert_eq!(FileType::classify(name), FileType::Data); } for name in [ "data.blob", "data.BLOB", - "data.unknown", - "dir.parquet/data", "data.parquet.blob", "data.video", - ".parquet", - ".csv.gz", + "data.VIDEO", + "data.vector.vortex", + "data.VECTOR.index-data", ] { - assert_eq!(FileType::classify(name), FileType::Data); + assert_eq!(FileType::classify(name), FileType::DedicatedData); } assert_eq!(FileType::classify("manifest-1.avro"), FileType::Meta); assert_eq!(FileType::classify("snapshot-1.parquet"), FileType::Meta); @@ -329,13 +298,9 @@ mod tests { FileType::classify("data.parquet.index"), FileType::FileIndex ); - assert_eq!( - FileType::parse_whitelist("structured-data"), - HashSet::from([FileType::StructuredData]) - ); assert_eq!( FileType::parse_whitelist("data"), - HashSet::from([FileType::Data, FileType::StructuredData]) + HashSet::from([FileType::Data, FileType::DedicatedData]) ); } @@ -346,7 +311,7 @@ mod tests { FileType::GlobalIndex, FileType::BucketIndex, FileType::Data, - FileType::StructuredData, + FileType::DedicatedData, FileType::FileIndex, ]); diff --git a/crates/paimon/src/io/cache/mod.rs b/crates/paimon/src/io/cache/mod.rs index 8fc1ad631..ca380bd7e 100644 --- a/crates/paimon/src/io/cache/mod.rs +++ b/crates/paimon/src/io/cache/mod.rs @@ -306,13 +306,28 @@ impl LocalCacheConfig { .get(CatalogOptions::LOCAL_CACHE_WHITELIST) .map(String::as_str) .unwrap_or("meta,global-index"); + let mut whitelist = FileType::parse_whitelist(whitelist); + if let Some(value) = options.get(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES) { + let value = value.trim(); + if value.eq_ignore_ascii_case("true") { + whitelist.remove(&FileType::DedicatedData); + } else if !value.eq_ignore_ascii_case("false") { + return Err(crate::Error::ConfigInvalid { + message: format!( + "Invalid boolean for {}: '{}'", + CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, + value + ), + }); + } + } Ok(Some(Self { dir, namespace: catalog_namespace(namespace_options), max_size, block_size, - whitelist: FileType::parse_whitelist(whitelist), + whitelist, })) } } @@ -412,7 +427,7 @@ mod tests { std::collections::HashSet::from([ FileType::Meta, FileType::Data, - FileType::StructuredData + FileType::DedicatedData ]) ); } @@ -490,12 +505,14 @@ mod tests { } #[test] - fn test_structured_data_selection_excludes_mutable_files() { + fn test_dedicated_data_exclusion_preserves_mutable_bypass() { let mut options = Options::new(); options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); - options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "structured-data"); + options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "data"); + options.set(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, "true"); let cache = create_local_cache(&options).unwrap().unwrap(); assert!(cache.is_cacheable("s3://bucket/data.parquet")); + assert!(cache.is_cacheable("s3://bucket/data.unknown")); for path in [ "s3://bucket/data.parquet.tmp", "s3://bucket/.data.parquet.123e4567-e89b-12d3-a456-426614174000.tmp", @@ -503,11 +520,41 @@ mod tests { "s3://bucket/manifest-1.avro", "s3://bucket/data.parquet.index", "s3://bucket/data.blob", + "s3://bucket/data.video", + "s3://bucket/data.vector.vortex", ] { assert!(!cache.is_cacheable(path), "{path}"); } } + #[test] + fn test_dedicated_data_exclusion_option() { + let mut options = Options::new(); + options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); + options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "data"); + assert!(LocalCacheConfig::from_options(&options) + .unwrap() + .unwrap() + .whitelist + .contains(&FileType::DedicatedData)); + + options.set(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, "TrUe"); + assert!(!LocalCacheConfig::from_options(&options) + .unwrap() + .unwrap() + .whitelist + .contains(&FileType::DedicatedData)); + + options.set( + CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, + "invalid", + ); + let error = LocalCacheConfig::from_options(&options).unwrap_err(); + assert!(error + .to_string() + .contains(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES)); + } + #[tokio::test] async fn test_local_cache_file_size_is_invalidated_across_shared_instances() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 49ea1a34d..28c1fa15b 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -268,7 +268,7 @@ impl FileIO { /// `block_size` controls the aligned ranges presented to the cache. /// `whitelist` uses the same comma-separated values as /// `local-cache.whitelist`: `meta`, `global-index`, `bucket-index`, `data`, - /// `structured-data`, and `file-index`, or `*` for all of them. + /// and `file-index`, or `*` for all of them. pub fn with_file_block_cache( mut self, cache: Arc, @@ -2427,19 +2427,24 @@ mod input_output_test { } #[tokio::test] - async fn test_structured_data_cache_preserves_existing_selectors() { + async fn test_dedicated_file_exclusion_preserves_existing_selectors() { for disk in [false, true] { - for whitelist in [ - "meta,global-index", - "data", - "structured-data", - "meta,global-index,structured-data", - "*", + for (whitelist, exclude_dedicated) in [ + ("meta,global-index", false), + ("data", false), + ("data", true), + ("meta,global-index,data", true), + ("*", false), + ("*", true), ] { let directory = tempfile::tempdir().unwrap(); let mut options = Options::new(); options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, whitelist); + options.set( + CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, + exclude_dedicated.to_string(), + ); options.set(CatalogOptions::LOCAL_CACHE_BLOCK_SIZE, "4"); if disk { options.set( @@ -2467,6 +2472,7 @@ mod input_output_test { "data.json.zst", "data.text.snappy", "data.video", + "data.vector.vortex", "data.PARQUET", "data.unknown", "data.parquet.index", @@ -2486,35 +2492,31 @@ mod input_output_test { drop(reader); let (op, relative_path, _) = input.source.resolve(&path).await.unwrap(); op.delete(&relative_path).await.unwrap(); + let data_file = name.starts_with("data.") && !name.ends_with(".index"); + let dedicated = + matches!(name, "data.blob" | "data.video" | "data.vector.vortex"); let cached = match whitelist { - "*" => true, - "data" => name.starts_with("data.") && !name.ends_with(".index"), - "structured-data" => matches!( - name, - "data.parquet" - | "data.orc" - | "data.avro" - | "data.row" - | "data.mosaic" - | "data.vortex" - | "data.csv" - | "data.json.zst" - | "data.text.snappy" - | "data.PARQUET" - ), - "meta,global-index,structured-data" => !matches!( - name, - "data.blob" | "data.video" | "data.unknown" | "data.parquet.index" - ), + "*" => !exclude_dedicated || !dedicated, + "data" => data_file && (!exclude_dedicated || !dedicated), + "meta,global-index,data" => { + name.starts_with("snapshot-") || (data_file && !dedicated) + } _ => name.starts_with("snapshot-"), }; let result = input.read().await; if cached { - assert_eq!(result.unwrap(), b"abcdefgh"[..], "{whitelist}: {name}"); + assert_eq!( + result.unwrap(), + b"abcdefgh"[..], + "{whitelist} exclude={exclude_dedicated}: {name}" + ); let reader = input.reader().await.unwrap(); assert_eq!(reader.read(1..7).await.unwrap(), b"bcdefg"[..]); } else { - assert!(result.is_err(), "{whitelist}: {name}"); + assert!( + result.is_err(), + "{whitelist} exclude={exclude_dedicated}: {name}" + ); } } } diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index fd749b746..e1ee47ad6 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -403,13 +403,12 @@ let catalog = CatalogFactory::create(options).await?; | `local-cache.dir` | none | Optional base directory. When set, Paimon uses a persistent disk cache in a private versioned child directory; otherwise it uses memory. | | `local-cache.max-size` | unlimited | Maximum cache size. Memory caches count payload bytes; disk caches count encoded bytes. Values accept byte units such as `512 MiB` or `20 GiB`. | | `local-cache.block-size` | `1 MiB` | Block size used for cached range reads. | -| `local-cache.whitelist` | `meta,global-index` | Comma-separated eligible types: `meta`, `global-index`, `bucket-index`, `data`, `structured-data`, and `file-index`, or `*` for all of them. | +| `local-cache.whitelist` | `meta,global-index` | Comma-separated eligible types: `meta`, `global-index`, `bucket-index`, `data`, and `file-index`, or `*` for all of them. | +| `local-cache.exclude-dedicated-files` | `false` | Exclude BLOB, video, and vector-store files from `data` or `*`. | -Use `meta,global-index,structured-data` to cache supported structured data -without BLOB or video bodies. It includes Parquet, ORC, Avro, ROW, Mosaic, -Vortex, CSV, JSON and TEXT files, including supported compressed text suffixes. -Matching is case-insensitive. `data` and `*` retain their broader behavior; the -default whitelist is unchanged. +Use `local-cache.whitelist=meta,global-index,data` with +`local-cache.exclude-dedicated-files=true` to cache data blocks without dedicated +BLOB, video, or vector-store files. Other data files remain eligible. Each catalog owns its in-memory cache for the catalog's lifetime. Disk caches are reused after process restarts. Cache keys include a catalog-configuration From 0dd21d21a485ce5898c724f20f9900a5b28cc269 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 20 Oct 2026 02:14:46 -0700 Subject: [PATCH 3/5] fix(core): keep structured-data cache selector --- crates/paimon/src/common/options.rs | 8 ---- crates/paimon/src/io/cache/file_type.rs | 7 +++- crates/paimon/src/io/cache/mod.rs | 51 ++----------------------- crates/paimon/src/io/file_io.rs | 39 +++++++------------ docs/src/getting-started.md | 9 ++--- 5 files changed, 27 insertions(+), 87 deletions(-) diff --git a/crates/paimon/src/common/options.rs b/crates/paimon/src/common/options.rs index a7c463166..86391377c 100644 --- a/crates/paimon/src/common/options.rs +++ b/crates/paimon/src/common/options.rs @@ -91,10 +91,6 @@ impl CatalogOptions { /// Comma-separated file types eligible for local caching. pub const LOCAL_CACHE_WHITELIST: &'static str = "local-cache.whitelist"; - /// Exclude dedicated BLOB, video, and vector-store files from the local cache. - pub const LOCAL_CACHE_EXCLUDE_DEDICATED_FILES: &'static str = - "local-cache.exclude-dedicated-files"; - /// Maximum process-local cache size for parsed file-format metadata. pub const FILE_FORMAT_METADATA_CACHE_MAX_SIZE: &'static str = "file-format.metadata-cache.max-size"; @@ -305,10 +301,6 @@ mod tests { CatalogOptions::LOCAL_CACHE_WHITELIST, "local-cache.whitelist" ); - assert_eq!( - CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, - "local-cache.exclude-dedicated-files" - ); assert_eq!( CatalogOptions::FILE_FORMAT_METADATA_CACHE_MAX_SIZE, "file-format.metadata-cache.max-size" diff --git a/crates/paimon/src/io/cache/file_type.rs b/crates/paimon/src/io/cache/file_type.rs index e12c4396f..562a2345d 100644 --- a/crates/paimon/src/io/cache/file_type.rs +++ b/crates/paimon/src/io/cache/file_type.rs @@ -121,12 +121,13 @@ impl FileType { "bucket-index" => &[Self::BucketIndex], // Preserve the existing broad data selector. "data" => &[Self::Data, Self::DedicatedData], + "structured-data" => &[Self::Data], "file-index" => &[Self::FileIndex], "" => &[], unknown => { log::warn!( "Unknown local-cache.whitelist value '{}'; supported values are \ - meta, global-index, bucket-index, data, file-index, \ + meta, global-index, bucket-index, data, structured-data, file-index, \ or * for all of them", unknown ); @@ -302,6 +303,10 @@ mod tests { FileType::parse_whitelist("data"), HashSet::from([FileType::Data, FileType::DedicatedData]) ); + assert_eq!( + FileType::parse_whitelist("structured-data"), + HashSet::from([FileType::Data]) + ); } #[test] diff --git a/crates/paimon/src/io/cache/mod.rs b/crates/paimon/src/io/cache/mod.rs index ca380bd7e..d65af3857 100644 --- a/crates/paimon/src/io/cache/mod.rs +++ b/crates/paimon/src/io/cache/mod.rs @@ -306,28 +306,12 @@ impl LocalCacheConfig { .get(CatalogOptions::LOCAL_CACHE_WHITELIST) .map(String::as_str) .unwrap_or("meta,global-index"); - let mut whitelist = FileType::parse_whitelist(whitelist); - if let Some(value) = options.get(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES) { - let value = value.trim(); - if value.eq_ignore_ascii_case("true") { - whitelist.remove(&FileType::DedicatedData); - } else if !value.eq_ignore_ascii_case("false") { - return Err(crate::Error::ConfigInvalid { - message: format!( - "Invalid boolean for {}: '{}'", - CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, - value - ), - }); - } - } - Ok(Some(Self { dir, namespace: catalog_namespace(namespace_options), max_size, block_size, - whitelist, + whitelist: FileType::parse_whitelist(whitelist), })) } } @@ -505,11 +489,10 @@ mod tests { } #[test] - fn test_dedicated_data_exclusion_preserves_mutable_bypass() { + fn test_structured_data_excludes_dedicated_and_mutable_files() { let mut options = Options::new(); options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); - options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "data"); - options.set(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, "true"); + options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "structured-data"); let cache = create_local_cache(&options).unwrap().unwrap(); assert!(cache.is_cacheable("s3://bucket/data.parquet")); assert!(cache.is_cacheable("s3://bucket/data.unknown")); @@ -527,34 +510,6 @@ mod tests { } } - #[test] - fn test_dedicated_data_exclusion_option() { - let mut options = Options::new(); - options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); - options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, "data"); - assert!(LocalCacheConfig::from_options(&options) - .unwrap() - .unwrap() - .whitelist - .contains(&FileType::DedicatedData)); - - options.set(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, "TrUe"); - assert!(!LocalCacheConfig::from_options(&options) - .unwrap() - .unwrap() - .whitelist - .contains(&FileType::DedicatedData)); - - options.set( - CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, - "invalid", - ); - let error = LocalCacheConfig::from_options(&options).unwrap_err(); - assert!(error - .to_string() - .contains(CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES)); - } - #[tokio::test] async fn test_local_cache_file_size_is_invalidated_across_shared_instances() { let directory = tempfile::tempdir().unwrap(); diff --git a/crates/paimon/src/io/file_io.rs b/crates/paimon/src/io/file_io.rs index 28c1fa15b..41cc2adbc 100644 --- a/crates/paimon/src/io/file_io.rs +++ b/crates/paimon/src/io/file_io.rs @@ -268,7 +268,7 @@ impl FileIO { /// `block_size` controls the aligned ranges presented to the cache. /// `whitelist` uses the same comma-separated values as /// `local-cache.whitelist`: `meta`, `global-index`, `bucket-index`, `data`, - /// and `file-index`, or `*` for all of them. + /// `structured-data`, and `file-index`, or `*` for all of them. pub fn with_file_block_cache( mut self, cache: Arc, @@ -2427,24 +2427,19 @@ mod input_output_test { } #[tokio::test] - async fn test_dedicated_file_exclusion_preserves_existing_selectors() { + async fn test_structured_data_cache_preserves_existing_selectors() { for disk in [false, true] { - for (whitelist, exclude_dedicated) in [ - ("meta,global-index", false), - ("data", false), - ("data", true), - ("meta,global-index,data", true), - ("*", false), - ("*", true), + for whitelist in [ + "meta,global-index", + "data", + "structured-data", + "meta,global-index,structured-data", + "*", ] { let directory = tempfile::tempdir().unwrap(); let mut options = Options::new(); options.set(CatalogOptions::LOCAL_CACHE_ENABLED, "true"); options.set(CatalogOptions::LOCAL_CACHE_WHITELIST, whitelist); - options.set( - CatalogOptions::LOCAL_CACHE_EXCLUDE_DEDICATED_FILES, - exclude_dedicated.to_string(), - ); options.set(CatalogOptions::LOCAL_CACHE_BLOCK_SIZE, "4"); if disk { options.set( @@ -2496,27 +2491,21 @@ mod input_output_test { let dedicated = matches!(name, "data.blob" | "data.video" | "data.vector.vortex"); let cached = match whitelist { - "*" => !exclude_dedicated || !dedicated, - "data" => data_file && (!exclude_dedicated || !dedicated), - "meta,global-index,data" => { + "*" => true, + "data" => data_file, + "structured-data" => data_file && !dedicated, + "meta,global-index,structured-data" => { name.starts_with("snapshot-") || (data_file && !dedicated) } _ => name.starts_with("snapshot-"), }; let result = input.read().await; if cached { - assert_eq!( - result.unwrap(), - b"abcdefgh"[..], - "{whitelist} exclude={exclude_dedicated}: {name}" - ); + assert_eq!(result.unwrap(), b"abcdefgh"[..], "{whitelist}: {name}"); let reader = input.reader().await.unwrap(); assert_eq!(reader.read(1..7).await.unwrap(), b"bcdefg"[..]); } else { - assert!( - result.is_err(), - "{whitelist} exclude={exclude_dedicated}: {name}" - ); + assert!(result.is_err(), "{whitelist}: {name}"); } } } diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index e1ee47ad6..49fca6a18 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -403,12 +403,11 @@ let catalog = CatalogFactory::create(options).await?; | `local-cache.dir` | none | Optional base directory. When set, Paimon uses a persistent disk cache in a private versioned child directory; otherwise it uses memory. | | `local-cache.max-size` | unlimited | Maximum cache size. Memory caches count payload bytes; disk caches count encoded bytes. Values accept byte units such as `512 MiB` or `20 GiB`. | | `local-cache.block-size` | `1 MiB` | Block size used for cached range reads. | -| `local-cache.whitelist` | `meta,global-index` | Comma-separated eligible types: `meta`, `global-index`, `bucket-index`, `data`, and `file-index`, or `*` for all of them. | -| `local-cache.exclude-dedicated-files` | `false` | Exclude BLOB, video, and vector-store files from `data` or `*`. | +| `local-cache.whitelist` | `meta,global-index` | Comma-separated eligible types: `meta`, `global-index`, `bucket-index`, `data`, `structured-data`, and `file-index`, or `*` for all of them. | -Use `local-cache.whitelist=meta,global-index,data` with -`local-cache.exclude-dedicated-files=true` to cache data blocks without dedicated -BLOB, video, or vector-store files. Other data files remain eligible. +Use `local-cache.whitelist=meta,global-index,structured-data` to cache data +blocks without dedicated BLOB, video, or vector-store files. `data` and `*` +retain their broader behavior. Each catalog owns its in-memory cache for the catalog's lifetime. Disk caches are reused after process restarts. Cache keys include a catalog-configuration From d557712be0b9ca5581ba946417a6a9c20b8cea78 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 20 Oct 2026 02:35:06 -0700 Subject: [PATCH 4/5] refactor: name structured cache file classification --- crates/paimon/src/io/cache/file_type.rs | 28 ++++++++++++------------- crates/paimon/src/io/cache/mod.rs | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/crates/paimon/src/io/cache/file_type.rs b/crates/paimon/src/io/cache/file_type.rs index 562a2345d..b3d956c2d 100644 --- a/crates/paimon/src/io/cache/file_type.rs +++ b/crates/paimon/src/io/cache/file_type.rs @@ -25,7 +25,7 @@ const TEMP_FILE_SUFFIX_LEN: usize = 41; pub(super) enum FileType { Meta, Data, - DedicatedData, + StructuredData, BucketIndex, GlobalIndex, FileIndex, @@ -35,7 +35,7 @@ impl FileType { const ALL: [Self; 6] = [ Self::Meta, Self::Data, - Self::DedicatedData, + Self::StructuredData, Self::BucketIndex, Self::GlobalIndex, Self::FileIndex, @@ -81,9 +81,9 @@ impl FileType { } if Self::is_dedicated_file(name) { - Self::DedicatedData - } else { Self::Data + } else { + Self::StructuredData } } @@ -120,8 +120,8 @@ impl FileType { "global-index" => &[Self::GlobalIndex], "bucket-index" => &[Self::BucketIndex], // Preserve the existing broad data selector. - "data" => &[Self::Data, Self::DedicatedData], - "structured-data" => &[Self::Data], + "data" => &[Self::Data, Self::StructuredData], + "structured-data" => &[Self::StructuredData], "file-index" => &[Self::FileIndex], "" => &[], unknown => { @@ -182,7 +182,7 @@ mod tests { ), ( "s3://bucket/table/bucket-0/data-abc.parquet", - FileType::Data, + FileType::StructuredData, ), ( "s3://bucket/table/manifest/manifest-123e4567-e89b-12d3-a456-426614174000-0.avro.sidecar", @@ -196,7 +196,7 @@ mod tests { ("s3://bucket/table/changelog/changelog-5", FileType::Meta), ( "s3://bucket/table/bucket-0/changelog-123e4567-e89b-12d3-a456-426614174000-0.parquet", - FileType::Data, + FileType::StructuredData, ), ( "s3://bucket/table/snapshot/.snapshot-13.123e4567-e89b-12d3-a456-426614174000.tmp", @@ -208,7 +208,7 @@ mod tests { ), ( "s3://bucket/table/bucket-0/.data-abc.parquet.123e4567-e89b-12d3-a456-426614174000.tmp", - FileType::Data, + FileType::StructuredData, ), ]; @@ -280,7 +280,7 @@ mod tests { "data.unknown", "dir.parquet/data", ] { - assert_eq!(FileType::classify(name), FileType::Data); + assert_eq!(FileType::classify(name), FileType::StructuredData); } for name in [ "data.blob", @@ -291,7 +291,7 @@ mod tests { "data.vector.vortex", "data.VECTOR.index-data", ] { - assert_eq!(FileType::classify(name), FileType::DedicatedData); + assert_eq!(FileType::classify(name), FileType::Data); } assert_eq!(FileType::classify("manifest-1.avro"), FileType::Meta); assert_eq!(FileType::classify("snapshot-1.parquet"), FileType::Meta); @@ -301,11 +301,11 @@ mod tests { ); assert_eq!( FileType::parse_whitelist("data"), - HashSet::from([FileType::Data, FileType::DedicatedData]) + HashSet::from([FileType::Data, FileType::StructuredData]) ); assert_eq!( FileType::parse_whitelist("structured-data"), - HashSet::from([FileType::Data]) + HashSet::from([FileType::StructuredData]) ); } @@ -316,7 +316,7 @@ mod tests { FileType::GlobalIndex, FileType::BucketIndex, FileType::Data, - FileType::DedicatedData, + FileType::StructuredData, FileType::FileIndex, ]); diff --git a/crates/paimon/src/io/cache/mod.rs b/crates/paimon/src/io/cache/mod.rs index d65af3857..882f9fa90 100644 --- a/crates/paimon/src/io/cache/mod.rs +++ b/crates/paimon/src/io/cache/mod.rs @@ -411,7 +411,7 @@ mod tests { std::collections::HashSet::from([ FileType::Meta, FileType::Data, - FileType::DedicatedData + FileType::StructuredData ]) ); } From 616b008c7f47dcd90bdb351cbf4016e3dba1eca1 Mon Sep 17 00:00:00 2001 From: xiaohongbo Date: Tue, 20 Oct 2026 02:38:47 -0700 Subject: [PATCH 5/5] fix: keep dedicated files distinct from structured data --- crates/paimon/src/io/cache/file_type.rs | 14 +++++++------- crates/paimon/src/io/cache/mod.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/paimon/src/io/cache/file_type.rs b/crates/paimon/src/io/cache/file_type.rs index b3d956c2d..2e8b66e50 100644 --- a/crates/paimon/src/io/cache/file_type.rs +++ b/crates/paimon/src/io/cache/file_type.rs @@ -24,7 +24,7 @@ const TEMP_FILE_SUFFIX_LEN: usize = 41; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub(super) enum FileType { Meta, - Data, + DedicatedData, StructuredData, BucketIndex, GlobalIndex, @@ -34,7 +34,7 @@ pub(super) enum FileType { impl FileType { const ALL: [Self; 6] = [ Self::Meta, - Self::Data, + Self::DedicatedData, Self::StructuredData, Self::BucketIndex, Self::GlobalIndex, @@ -81,7 +81,7 @@ impl FileType { } if Self::is_dedicated_file(name) { - Self::Data + Self::DedicatedData } else { Self::StructuredData } @@ -120,7 +120,7 @@ impl FileType { "global-index" => &[Self::GlobalIndex], "bucket-index" => &[Self::BucketIndex], // Preserve the existing broad data selector. - "data" => &[Self::Data, Self::StructuredData], + "data" => &[Self::DedicatedData, Self::StructuredData], "structured-data" => &[Self::StructuredData], "file-index" => &[Self::FileIndex], "" => &[], @@ -291,7 +291,7 @@ mod tests { "data.vector.vortex", "data.VECTOR.index-data", ] { - assert_eq!(FileType::classify(name), FileType::Data); + assert_eq!(FileType::classify(name), FileType::DedicatedData); } assert_eq!(FileType::classify("manifest-1.avro"), FileType::Meta); assert_eq!(FileType::classify("snapshot-1.parquet"), FileType::Meta); @@ -301,7 +301,7 @@ mod tests { ); assert_eq!( FileType::parse_whitelist("data"), - HashSet::from([FileType::Data, FileType::StructuredData]) + HashSet::from([FileType::DedicatedData, FileType::StructuredData]) ); assert_eq!( FileType::parse_whitelist("structured-data"), @@ -315,7 +315,7 @@ mod tests { FileType::Meta, FileType::GlobalIndex, FileType::BucketIndex, - FileType::Data, + FileType::DedicatedData, FileType::StructuredData, FileType::FileIndex, ]); diff --git a/crates/paimon/src/io/cache/mod.rs b/crates/paimon/src/io/cache/mod.rs index 882f9fa90..74c636d01 100644 --- a/crates/paimon/src/io/cache/mod.rs +++ b/crates/paimon/src/io/cache/mod.rs @@ -410,7 +410,7 @@ mod tests { config.whitelist, std::collections::HashSet::from([ FileType::Meta, - FileType::Data, + FileType::DedicatedData, FileType::StructuredData ]) );