From cad2b4035119602e482d3eb70b904397b480237e Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Fri, 22 May 2026 14:21:27 +0800 Subject: [PATCH 01/16] feat(update): add MergingSnapshotUpdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Abstract base for merge-based snapshot operations (MergeAppend, OverwriteFiles, RowDelta, etc.), implementing the filter → write → merge pipeline consistent with Java's MergingSnapshotProducer. Also fixes SnapshotSummaryBuilder manifest count fields and a use-after-free bug in SnapshotUpdate::DeleteFile. --- src/iceberg/CMakeLists.txt | 1 + .../manifest/manifest_filter_manager.cc | 37 +- .../manifest/manifest_filter_manager.h | 34 + .../manifest/manifest_merge_manager.cc | 4 + src/iceberg/manifest/manifest_merge_manager.h | 5 + src/iceberg/meson.build | 1 + src/iceberg/snapshot.cc | 24 + src/iceberg/snapshot.h | 13 + src/iceberg/test/CMakeLists.txt | 1 + .../test/manifest_filter_manager_test.cc | 183 ++++ .../test/merging_snapshot_update_test.cc | 736 +++++++++++++++ src/iceberg/update/merging_snapshot_update.cc | 870 ++++++++++++++++++ src/iceberg/update/merging_snapshot_update.h | 350 +++++++ src/iceberg/update/snapshot_update.cc | 8 +- .../update/update_partition_statistics.h | 5 +- 15 files changed, 2262 insertions(+), 10 deletions(-) create mode 100644 src/iceberg/test/merging_snapshot_update_test.cc create mode 100644 src/iceberg/update/merging_snapshot_update.cc create mode 100644 src/iceberg/update/merging_snapshot_update.h diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 18cf70bdb..38d85534a 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -89,6 +89,7 @@ set(ICEBERG_SOURCES type.cc update/expire_snapshots.cc update/fast_append.cc + update/merging_snapshot_update.cc update/pending_update.cc update/set_snapshot.cc update/snapshot_manager.cc diff --git a/src/iceberg/manifest/manifest_filter_manager.cc b/src/iceberg/manifest/manifest_filter_manager.cc index 086c94a78..2b53158bc 100644 --- a/src/iceberg/manifest/manifest_filter_manager.cc +++ b/src/iceberg/manifest/manifest_filter_manager.cc @@ -117,12 +117,24 @@ bool ManifestFilterManager::ContainsDeletes() const { !drop_partitions_.empty(); } +void ManifestFilterManager::DropDeleteFilesOlderThan(int64_t sequence_number) { + min_sequence_number_ = sequence_number; +} + +void ManifestFilterManager::RemoveDanglingDeletesFor(const DataFileSet& deleted_files) { + for (const auto& file : deleted_files) { + removed_data_file_paths_.insert(file->file_path); + } +} + Result ManifestFilterManager::CanContainDroppedFiles(const ManifestFile&) const { // TODO(Guotao): Use the manifest descriptor to skip unrelated object-delete // manifests once object-delete partitions are tracked separately. // Currently, DeleteFile(std::shared_ptr) degrades to a path-based delete, // which forces scanning all manifests. - return !delete_paths_.empty(); + // Also open delete manifests when a minimum sequence number is set for cleanup. + return !delete_paths_.empty() || !removed_data_file_paths_.empty() || + (manifest_content_ == ManifestContent::kDeletes && min_sequence_number_ > 0); } Result ManifestFilterManager::CanContainDroppedPartitions( @@ -219,6 +231,25 @@ Result ManifestFilterManager::ShouldDelete(const ManifestEntry& entry, return true; } + // Delete-manifest-specific cleanup (only for ManifestContent::kDeletes). + if (manifest_content_ == ManifestContent::kDeletes) { + // Drop delete files whose data sequence number is older than the minimum + // retained by the table (they can no longer match any live data rows). + // seq == 0 (kInitialSequenceNumber / nullopt) is intentionally excluded: + // those entries predate sequence number assignment and must not be pruned. + int64_t seq = entry.sequence_number.value_or(0); + if (min_sequence_number_ > 0 && seq > 0 && seq < min_sequence_number_) { + return true; + } + + // Drop DVs that reference a data file that has been removed (dangling DV). + if (!removed_data_file_paths_.empty() && file.IsDeletionVector() && + file.referenced_data_file.has_value() && + removed_data_file_paths_.count(*file.referenced_data_file)) { + return true; + } + } + if (HasRowFilterExpression(delete_expr_)) { ICEBERG_ASSIGN_OR_RAISE(auto* residual_eval, GetResidualEvaluator(schema, specs_by_id, spec_id)); @@ -403,6 +434,7 @@ Result> ManifestFilterManager::FilterManifests( bool trust_manifest_references = CanTrustManifestReferences(manifests); manifest_evaluator_cache_.clear(); residual_evaluator_cache_.clear(); + replaced_manifests_count_ = 0; // TODO(Guotao): Parallelize manifest filtering with per-manifest results, then // merge found paths and deleted files after the loop. @@ -413,6 +445,9 @@ Result> ManifestFilterManager::FilterManifests( auto filtered_manifest, FilterManifest(schema, specs_by_id, *manifest_ptr, trust_manifest_references, writer_factory, found_paths)); + if (filtered_manifest.manifest_path != manifest_ptr->manifest_path) { + ++replaced_manifests_count_; + } filtered.push_back(std::move(filtered_manifest)); } diff --git a/src/iceberg/manifest/manifest_filter_manager.h b/src/iceberg/manifest/manifest_filter_manager.h index 55258b2b1..981b9ac3b 100644 --- a/src/iceberg/manifest/manifest_filter_manager.h +++ b/src/iceberg/manifest/manifest_filter_manager.h @@ -116,9 +116,36 @@ class ICEBERG_EXPORT ManifestFilterManager { /// manifest entry matches a delete condition. void FailAnyDelete(); + /// \brief Returns the number of manifests rewritten (replaced) by the last + /// FilterManifests() call. A manifest is replaced when it contained deleted entries + /// and was rewritten with those entries marked DELETED. + int32_t ReplacedManifestsCount() const { return replaced_manifests_count_; } + /// \brief Returns true if any delete condition has been registered. bool ContainsDeletes() const; + /// \brief Set the minimum data sequence number for delete files to retain. + /// + /// Only valid for ManifestContent::kDeletes managers. Delete entries whose + /// data_sequence_number is positive and less than \p sequence_number will be + /// marked DELETED. This continuously removes delete files that cannot match + /// any remaining data rows (i.e. all data written before that sequence number + /// has itself been deleted). + /// + /// \param sequence_number the inclusive lower bound; delete files older than + /// this value are dropped + void DropDeleteFilesOlderThan(int64_t sequence_number); + + /// \brief Register data files that have been removed so their dangling DVs + /// can be cleaned up. + /// + /// Only valid for ManifestContent::kDeletes managers. For each DV whose + /// referenced_data_file path appears in \p deleted_files, the DV entry is + /// marked DELETED because the data file it targets no longer exists. + /// + /// \param deleted_files set of data files that have been marked for deletion + void RemoveDanglingDeletesFor(const DataFileSet& deleted_files); + /// \brief Apply all accumulated delete conditions to the base snapshot's manifests. /// /// Manifests that cannot possibly contain deleted files are returned unchanged. @@ -220,6 +247,13 @@ class ICEBERG_EXPORT ManifestFilterManager { bool fail_any_delete_{false}; bool case_sensitive_{true}; + int32_t replaced_manifests_count_{0}; + + // minimum data sequence number; delete entries older than this are dropped + int64_t min_sequence_number_{0}; + // paths of data files that were removed; DVs referencing these are dangling + std::unordered_set removed_data_file_paths_; + std::unordered_map> manifest_evaluator_cache_; std::unordered_map> diff --git a/src/iceberg/manifest/manifest_merge_manager.cc b/src/iceberg/manifest/manifest_merge_manager.cc index 056dce3f5..aedcea735 100644 --- a/src/iceberg/manifest/manifest_merge_manager.cc +++ b/src/iceberg/manifest/manifest_merge_manager.cc @@ -57,6 +57,7 @@ Result> ManifestMergeManager::MergeManifests( std::ranges::copy(manifest_ranges | std::views::join, std::back_inserter(all)); if (all.empty() || !merge_enabled_) { + replaced_manifests_count_ = 0; return all | std::views::transform([](const ManifestFile* manifest) { return *manifest; }) | std::ranges::to>(); @@ -82,6 +83,7 @@ Result> ManifestMergeManager::MergeManifests( std::vector result; result.reserve(all.size()); + replaced_manifests_count_ = 0; for (auto& [key, group] : by_spec) { const auto* first = first_by_content.at(key.second); ICEBERG_ASSIGN_OR_RAISE(auto merged, MergeGroup(group, first, snapshot_id, metadata, @@ -140,6 +142,8 @@ Result> ManifestMergeManager::MergeGroup( } else { ICEBERG_ASSIGN_OR_RAISE( auto merged, FlushBin(bin, snapshot_id, metadata, file_io, writer_factory)); + // Each manifest consumed into the merged output (beyond the 1 output) is replaced. + replaced_manifests_count_ += static_cast(bin.size()) - 1; result.push_back(std::move(merged)); } } diff --git a/src/iceberg/manifest/manifest_merge_manager.h b/src/iceberg/manifest/manifest_merge_manager.h index 16cc8d987..614ab61c6 100644 --- a/src/iceberg/manifest/manifest_merge_manager.h +++ b/src/iceberg/manifest/manifest_merge_manager.h @@ -84,6 +84,10 @@ class ICEBERG_EXPORT ManifestMergeManager { const TableMetadata& metadata, std::shared_ptr file_io, const ManifestWriterFactory& writer_factory); + /// \brief Returns the number of manifests replaced (consumed into merged outputs) + /// by the last MergeManifests() call. + int32_t ReplacedManifestsCount() const { return replaced_manifests_count_; } + private: /// \brief Merge a group of manifests sharing the same spec_id. /// @@ -109,6 +113,7 @@ class ICEBERG_EXPORT ManifestMergeManager { const int64_t target_size_bytes_; const int32_t min_count_to_merge_; const bool merge_enabled_; + int32_t replaced_manifests_count_{0}; }; } // namespace iceberg diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index a5a60b605..678f30fbd 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -111,6 +111,7 @@ iceberg_sources = files( 'type.cc', 'update/expire_snapshots.cc', 'update/fast_append.cc', + 'update/merging_snapshot_update.cc', 'update/pending_update.cc', 'update/set_snapshot.cc', 'update/snapshot_manager.cc', diff --git a/src/iceberg/snapshot.cc b/src/iceberg/snapshot.cc index 1b3182fd9..d513e2be1 100644 --- a/src/iceberg/snapshot.cc +++ b/src/iceberg/snapshot.cc @@ -441,6 +441,10 @@ void SnapshotSummaryBuilder::Clear() { metrics_.Clear(); deleted_duplicate_files_ = 0; trust_partition_metrics_ = true; + manifests_counts_set_ = false; + manifests_created_ = 0; + manifests_kept_ = 0; + manifests_replaced_ = 0; } void SnapshotSummaryBuilder::SetPartitionSummaryLimit(int32_t max) { @@ -475,6 +479,14 @@ void SnapshotSummaryBuilder::Set(const std::string& property, const std::string& properties_[property] = value; } +void SnapshotSummaryBuilder::SetManifestCounts(int32_t created, int32_t kept, + int32_t replaced) { + manifests_counts_set_ = true; + manifests_created_ = created; + manifests_kept_ = kept; + manifests_replaced_ = replaced; +} + void SnapshotSummaryBuilder::Merge(const SnapshotSummaryBuilder& other) { for (const auto& [key, value] : other.properties_) { properties_[key] = value; @@ -491,6 +503,10 @@ void SnapshotSummaryBuilder::Merge(const SnapshotSummaryBuilder& other) { } deleted_duplicate_files_ += other.deleted_duplicate_files_; + // Manifest counts (manifests_counts_set_ / manifests_created_ / manifests_kept_ / + // manifests_replaced_) are intentionally not merged here. They are set directly + // on the root summary builder by Apply() after all manifests are finalized, and + // are never populated on sub-builders that get Merge()d in. } std::unordered_map SnapshotSummaryBuilder::Build() const { @@ -504,6 +520,14 @@ std::unordered_map SnapshotSummaryBuilder::Build() con SetIf(deleted_duplicate_files_ > 0, builder, SnapshotSummaryFields::kDeletedDuplicatedFiles, deleted_duplicate_files_); + // Always emit all three manifest count fields together when they have been set. + SetIf(manifests_counts_set_, builder, SnapshotSummaryFields::kManifestsCreated, + manifests_created_); + SetIf(manifests_counts_set_, builder, SnapshotSummaryFields::kManifestsKept, + manifests_kept_); + SetIf(manifests_counts_set_, builder, SnapshotSummaryFields::kManifestsReplaced, + manifests_replaced_); + SetIf(trust_partition_metrics_, builder, SnapshotSummaryFields::kChangedPartitionCountProp, partition_metrics_.size()); diff --git a/src/iceberg/snapshot.h b/src/iceberg/snapshot.h index f3e7ffb85..178c21dd7 100644 --- a/src/iceberg/snapshot.h +++ b/src/iceberg/snapshot.h @@ -338,6 +338,15 @@ class ICEBERG_EXPORT SnapshotSummaryBuilder { /// \param value Property value void Set(const std::string& property, const std::string& value); + /// \brief Set manifest count summary fields. + /// + /// Records how many manifests were created, kept, and replaced in this snapshot. + /// + /// \param created Manifests written by this snapshot + /// \param kept Manifests carried over unchanged from the previous snapshot + /// \param replaced Manifests rewritten or merged away + void SetManifestCounts(int32_t created, int32_t kept, int32_t replaced); + /// \brief Merge another builder's metrics into this one /// /// \param other The builder to merge from @@ -359,6 +368,10 @@ class ICEBERG_EXPORT SnapshotSummaryBuilder { int32_t max_changed_partitions_for_summaries_{0}; int64_t deleted_duplicate_files_{0}; bool trust_partition_metrics_{true}; + bool manifests_counts_set_{false}; + int32_t manifests_created_{0}; + int32_t manifests_kept_{0}; + int32_t manifests_replaced_{0}; }; /// \brief Data operation that produce snapshots. diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 997d18354..20babf19c 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -214,6 +214,7 @@ if(ICEBERG_BUILD_BUNDLE) expire_snapshots_test.cc fast_append_test.cc manifest_filter_manager_test.cc + merging_snapshot_update_test.cc name_mapping_update_test.cc snapshot_manager_test.cc transaction_test.cc diff --git a/src/iceberg/test/manifest_filter_manager_test.cc b/src/iceberg/test/manifest_filter_manager_test.cc index 7810509fa..aa1054fec 100644 --- a/src/iceberg/test/manifest_filter_manager_test.cc +++ b/src/iceberg/test/manifest_filter_manager_test.cc @@ -41,6 +41,7 @@ #include "iceberg/test/matchers.h" #include "iceberg/test/update_test_base.h" #include "iceberg/update/fast_append.h" +#include "iceberg/util/data_file_set.h" #include "iceberg/util/macros.h" namespace iceberg { @@ -379,4 +380,186 @@ TEST_F(ManifestFilterManagerTest, MultipleRowFiltersUseCombinedExpression) { EXPECT_EQ(entries[0].status, ManifestStatus::kDeleted); } +// Helper: write one or more delete-file entries to a new manifest. +// Each entry is (DataFile, data_sequence_number). +using DeleteManifestEntry = std::pair, int64_t>; +static Result WriteDeleteManifest( + const std::vector& files, std::shared_ptr file_io, + const TableMetadata& metadata, const std::string& path) { + if (files.empty()) { + return InvalidArgument("WriteDeleteManifest requires at least one entry"); + } + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + int32_t spec_id = files[0].first->partition_spec_id.value_or(0); + ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata.PartitionSpecById(spec_id)); + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + ManifestWriter::MakeWriter(metadata.format_version, /*snapshot_id=*/1L, path, + file_io, spec, schema, ManifestContent::kDeletes)); + for (auto& [file, seq] : files) { + ManifestEntry entry; + entry.status = ManifestStatus::kAdded; + entry.snapshot_id = 1L; + entry.sequence_number = seq; + entry.data_file = file; + ICEBERG_RETURN_UNEXPECTED(writer->WriteAddedEntry(entry)); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + return writer->ToManifestFile(); +} + +// Convenience overload for a single entry. +static Result WriteDeleteManifest(std::shared_ptr delete_file, + int64_t data_sequence_number, + std::shared_ptr file_io, + const TableMetadata& metadata, + const std::string& path) { + return WriteDeleteManifest({{delete_file, data_sequence_number}}, file_io, metadata, + path); +} + +TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThan) { + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + // Create a position-delete file with data_sequence_number = 2 (below threshold 5). + auto del_file = std::make_shared(); + del_file->content = DataFile::Content::kPositionDeletes; + del_file->file_path = table_location_ + "/delete/del_old.parquet"; + del_file->file_format = FileFormatType::kParquet; + del_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + del_file->file_size_in_bytes = 512; + del_file->record_count = 10; + del_file->partition_spec_id = spec_->spec_id(); + + auto manifest_path = std::format("{}/metadata/del-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto del_manifest, + WriteDeleteManifest(del_file, /*data_seq=*/2L, file_io_, *metadata, manifest_path)); + + ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); + // Drop delete files older than sequence number 5: entry (seq=2) should be dropped. + mgr.DropDeleteFilesOlderThan(5); + + std::vector manifests{&del_manifest}; + auto specs = SpecsById(*metadata); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr.FilterManifests(schema, specs, manifests, factory)); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].status, ManifestStatus::kDeleted); +} + +TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanKeepsNewerEntries) { + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + // Two entries in the same manifest: old (seq=2, below threshold) and new (seq=10, + // above). + auto make_del_file = [&](const std::string& path) { + auto f = std::make_shared(); + f->content = DataFile::Content::kPositionDeletes; + f->file_path = path; + f->file_format = FileFormatType::kParquet; + f->partition = PartitionValues(std::vector{Literal::Long(1L)}); + f->file_size_in_bytes = 512; + f->record_count = 10; + f->partition_spec_id = spec_->spec_id(); + return f; + }; + auto old_file = make_del_file(table_location_ + "/delete/del_old.parquet"); + auto new_file = make_del_file(table_location_ + "/delete/del_new.parquet"); + + auto manifest_path = std::format("{}/metadata/del-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL(auto del_manifest, + WriteDeleteManifest({{old_file, 2L}, {new_file, 10L}}, file_io_, + *metadata, manifest_path)); + + ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); + // Threshold=5: old entry dropped, new entry survives as kExisting. + mgr.DropDeleteFilesOlderThan(5); + + std::vector manifests{&del_manifest}; + auto specs = SpecsById(*metadata); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr.FilterManifests(schema, specs, manifests, factory)); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); + ASSERT_EQ(entries.size(), 2U); + // The old entry should be dropped; the new entry should survive. + auto deleted = std::count_if( + entries.begin(), entries.end(), + [](const ManifestEntry& e) { return e.status == ManifestStatus::kDeleted; }); + auto existing = std::count_if( + entries.begin(), entries.end(), + [](const ManifestEntry& e) { return e.status == ManifestStatus::kExisting; }); + EXPECT_EQ(deleted, 1); + EXPECT_EQ(existing, 1); + // Verify which entry survived. + for (const auto& e : entries) { + if (e.status == ManifestStatus::kExisting) { + EXPECT_EQ(e.data_file->file_path, new_file->file_path); + } else { + EXPECT_EQ(e.data_file->file_path, old_file->file_path); + } + } +} + +TEST_F(ManifestFilterManagerTest, RemoveDanglingDeletesForFiltersDanglingDV) { + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + const std::string data_file_path = table_location_ + "/data/referenced.parquet"; + + // Create a DV (position-delete, puffin format) referencing the data file. + auto dv_file = std::make_shared(); + dv_file->content = DataFile::Content::kPositionDeletes; + dv_file->file_path = table_location_ + "/delete/dv.puffin"; + dv_file->file_format = FileFormatType::kPuffin; + dv_file->referenced_data_file = data_file_path; + dv_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + dv_file->file_size_in_bytes = 256; + dv_file->record_count = 5; + dv_file->partition_spec_id = spec_->spec_id(); + + auto manifest_path = std::format("{}/metadata/dv-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto dv_manifest, + WriteDeleteManifest(dv_file, /*data_seq=*/3L, file_io_, *metadata, manifest_path)); + + // Register the referenced data file as deleted. + auto deleted_data_file = std::make_shared(); + deleted_data_file->content = DataFile::Content::kData; + deleted_data_file->file_path = data_file_path; + deleted_data_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + deleted_data_file->file_size_in_bytes = 1024; + deleted_data_file->record_count = 50; + deleted_data_file->partition_spec_id = spec_->spec_id(); + + DataFileSet deleted_files; + deleted_files.insert(deleted_data_file); + + ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); + mgr.RemoveDanglingDeletesFor(deleted_files); + + std::vector manifests{&dv_manifest}; + auto specs = SpecsById(*metadata); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr.FilterManifests(schema, specs, manifests, factory)); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].status, ManifestStatus::kDeleted); +} + } // namespace iceberg diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc new file mode 100644 index 000000000..366b09f25 --- /dev/null +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -0,0 +1,736 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/update/merging_snapshot_update.h" + +#include +#include +#include +#include + +#include +#include + +#include "iceberg/avro/avro_register.h" +#include "iceberg/constants.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/manifest/manifest_writer.h" +#include "iceberg/partition_spec.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/table_properties.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/update_test_base.h" +#include "iceberg/transaction.h" +#include "iceberg/update/fast_append.h" +#include "iceberg/update/update_properties.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +/// \brief Concrete subclass of MergingSnapshotUpdate for testing. +class TestMergeAppend : public MergingSnapshotUpdate { + public: + static Result> Make(std::string table_name, + std::shared_ptr table) { + ICEBERG_ASSIGN_OR_RAISE( + auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); + return std::unique_ptr( + new TestMergeAppend(std::move(table_name), std::move(ctx))); + } + + std::string operation() override { return "append"; } + + // Expose protected API for test access + Status AddFile(std::shared_ptr file) { return AddDataFile(std::move(file)); } + Status AddDelete(std::shared_ptr file) { + return AddDeleteFile(std::move(file)); + } + Status RemoveDataFile(std::shared_ptr file) { + return DeleteDataFile(std::move(file)); + } + Status RemoveDeleteFile(std::shared_ptr file) { + return DeleteDeleteFile(std::move(file)); + } + Status AppendManifest(ManifestFile manifest) { + return AddManifest(std::move(manifest)); + } + Result> DataSpec() const { + return MergingSnapshotUpdate::DataSpec(); + } + void SetDataSeqNumber(int64_t seq) { SetNewDataFilesDataSequenceNumber(seq); } + + bool HasDataFiles() const { return AddsDataFiles(); } + bool HasDeleteFiles() const { return AddsDeleteFiles(); } + bool HasDataDeletes() const { return DeletesDataFiles(); } + + private: + TestMergeAppend(std::string table_name, std::shared_ptr ctx) + : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {} +}; + +class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { + protected: + static void SetUpTestSuite() { avro::RegisterAll(); } + + void SetUp() override { + MinimalUpdateTestBase::SetUp(); + + ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); + ICEBERG_UNWRAP_OR_FAIL(schema_, table_->schema()); + + file_a_ = MakeDataFile("/data/file_a.parquet", /*partition_x=*/1L); + file_b_ = MakeDataFile("/data/file_b.parquet", /*partition_x=*/2L); + } + + std::shared_ptr MakeDataFile(const std::string& path, int64_t partition_x) { + auto f = std::make_shared(); + f->content = DataFile::Content::kData; + f->file_path = table_location_ + path; + f->file_format = FileFormatType::kParquet; + f->partition = PartitionValues(std::vector{Literal::Long(partition_x)}); + f->file_size_in_bytes = 1024; + f->record_count = 100; + f->partition_spec_id = spec_->spec_id(); + return f; + } + + std::shared_ptr MakeDeleteFile(const std::string& path, int64_t partition_x) { + auto f = MakeDataFile(path, partition_x); + f->content = DataFile::Content::kPositionDeletes; + return f; + } + + Result> NewMergeAppend() { + return TestMergeAppend::Make(TableName(), table_); + } + + // Commit file_a_ with FastAppend and refresh the table. + void CommitFileA() { + ICEBERG_UNWRAP_OR_FAIL(auto fa, table_->NewFastAppend()); + fa->AppendFile(file_a_); + EXPECT_THAT(fa->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + } + + // Read all entries from a list of ManifestFiles. + Result> ReadAllEntries( + const std::vector& manifests, const TableMetadata& metadata) { + std::vector result; + for (const auto& m : manifests) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata.PartitionSpecById(m.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto reader, + ManifestReader::Make(m, file_io_, schema, spec)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); + result.insert(result.end(), entries.begin(), entries.end()); + } + return result; + } + + // Write a manifest file containing the given data files. + // Returns a ManifestFile with added_snapshot_id = kInvalidSnapshotId so it + // is eligible for snapshot ID inheritance. + Result WriteManifest( + const std::string& path, const std::vector>& files) { + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + ManifestWriter::MakeWriter(/*format_version=*/2, kInvalidSnapshotId, path, + file_io_, spec_, schema_, ManifestContent::kData)); + for (const auto& f : files) { + ManifestEntry entry; + entry.status = ManifestStatus::kAdded; + entry.snapshot_id = std::nullopt; + entry.data_file = f; + ICEBERG_RETURN_UNEXPECTED(writer->WriteAddedEntry(entry)); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + return writer->ToManifestFile(); + } + + std::shared_ptr spec_; + std::shared_ptr schema_; + std::shared_ptr file_a_; + std::shared_ptr file_b_; +}; + +// ------------------------------------------------------------------------- +// State query tests +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, AddsDataFilesInitiallyFalse) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_FALSE(op->HasDataFiles()); + EXPECT_FALSE(op->HasDeleteFiles()); + EXPECT_FALSE(op->HasDataDeletes()); +} + +TEST_F(MergingSnapshotUpdateTest, AddsDataFilesTrueAfterAdd) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_TRUE(op->HasDataFiles()); + EXPECT_FALSE(op->HasDeleteFiles()); +} + +TEST_F(MergingSnapshotUpdateTest, AddsDeleteFilesTrueAfterAdd) { + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + EXPECT_FALSE(op->HasDataFiles()); + EXPECT_TRUE(op->HasDeleteFiles()); +} + +TEST_F(MergingSnapshotUpdateTest, DeletesDataFilesTrueAfterRegisterDelete) { + CommitFileA(); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); + EXPECT_TRUE(op->HasDataDeletes()); +} + +// ------------------------------------------------------------------------- +// Apply / Commit tests +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, CommitNewDataFile) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("added-data-files"), "1"); + EXPECT_EQ(snapshot->summary.at("added-records"), "100"); +} + +TEST_F(MergingSnapshotUpdateTest, CommitMultipleDataFiles) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("added-data-files"), "2"); + EXPECT_EQ(snapshot->summary.at("added-records"), "200"); +} + +TEST_F(MergingSnapshotUpdateTest, CommitDataFileAndDeleteFile) { + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + // Data file summary + EXPECT_EQ(snapshot->summary.at("added-data-files"), "1"); +} + +TEST_F(MergingSnapshotUpdateTest, CommitPreservesExistingManifests) { + // First append: file_a + CommitFileA(); + + // Second merge append: file_b + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + // Both data files should be visible — 1 existing + 1 new + EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); +} + +TEST_F(MergingSnapshotUpdateTest, CommitDeletesDataFile) { + CommitFileA(); + + // Remove file_a via merging snapshot update + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("total-data-files"), "0"); + EXPECT_EQ(snapshot->summary.at("deleted-data-files"), "1"); +} + +TEST_F(MergingSnapshotUpdateTest, SetNewDataFilesDataSequenceNumber) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + op->SetDataSeqNumber(42); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("added-data-files"), "1"); +} + +// ------------------------------------------------------------------------- +// CleanUncommitted test +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, CleanUncommittedAfterSuccessfulCommitDoesNotCrash) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + // Simulate a caller invoking CleanUncommitted after a commit (e.g. cleanup + // in an error handler that runs regardless of success). Passing an empty set + // means no manifests are considered committed, so CleanUncommitted attempts + // to delete all written manifests. This should not crash. + op->CleanUncommitted({}); +} + +// ------------------------------------------------------------------------- +// Delete file summary tests +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, CommitDeleteFileSummaryHasAddedDeleteFiles) { + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDeleteFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedPosDeleteFiles), "1"); + EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kRemovedDeleteFiles), 0); +} + +// Covers the bug where deleted delete files were not tracked in the snapshot summary. +TEST_F(MergingSnapshotUpdateTest, CommitDeletesDeleteFileSummaryHasRemovedDeleteFiles) { + // Step 1: commit a delete file. + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + } + + // Step 2: commit a new snapshot that removes the delete file. + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->RemoveDeleteFile(del_file), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kRemovedDeleteFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kRemovedPosDeleteFiles), "1"); + EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kAddedDeleteFiles), 0); +} + +// ------------------------------------------------------------------------- +// Deduplication test +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, DuplicateDataFileOnlyCountedOnce) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); // duplicate — should be ignored + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "1"); +} + +// ------------------------------------------------------------------------- +// ValidateNewDeleteFile format version tests +// ------------------------------------------------------------------------- + +/// \brief V1-table test fixture — deletes are not supported in format v1. +class MergingSnapshotUpdateV1Test : public UpdateTestBase { + protected: + std::string MetadataResource() const override { return "TableMetadataV1Valid.json"; } + std::string TableName() const override { return "v1_test_table"; } + + void SetUp() override { + UpdateTestBase::SetUp(); + ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec()); + } + + std::shared_ptr MakeDeleteFile(const std::string& path) { + auto f = std::make_shared(); + f->content = DataFile::Content::kPositionDeletes; + f->file_path = table_location_ + path; + f->file_format = FileFormatType::kParquet; + f->partition = PartitionValues(std::vector{Literal::Long(1L)}); + f->file_size_in_bytes = 512; + f->record_count = 10; + f->partition_spec_id = spec_->spec_id(); + return f; + } + + Result> NewMergeAppend() { + return TestMergeAppend::Make(TableName(), table_); + } + + std::shared_ptr spec_; +}; + +TEST_F(MergingSnapshotUpdateV1Test, ValidateNewDeleteFileV1Rejected) { + auto del_file = MakeDeleteFile("/delete/del_a.parquet"); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV2RejectsDeletionVector) { + // Position delete with referenced_data_file set = deletion vector, not allowed in v2. + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + del_file->referenced_data_file = table_location_ + "/data/file_a.parquet"; + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV2AllowsEqualityDelete) { + auto eq_del = MakeDeleteFile("/delete/eq_del.parquet", 1L); + eq_del->content = DataFile::Content::kEqualityDeletes; + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(eq_del), IsOk()); +} + +// ------------------------------------------------------------------------- +// AddManifest — invalid manifest rejection +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsDeleteManifest) { + // Build a ManifestFile with content = kDeletes + ManifestFile del_manifest; + del_manifest.manifest_path = table_location_ + "/metadata/del.avro"; + del_manifest.content = ManifestContent::kDeletes; + del_manifest.added_snapshot_id = kInvalidSnapshotId; + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AppendManifest(del_manifest), IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithExistingFiles) { + // Construct a ManifestFile that reports existing files without writing to disk. + ManifestFile manifest; + manifest.manifest_path = table_location_ + "/metadata/existing.avro"; + manifest.content = ManifestContent::kData; + manifest.added_snapshot_id = kInvalidSnapshotId; + manifest.existing_files_count = 1; // has_existing_files() returns true + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithDeletedFiles) { + ManifestFile manifest; + manifest.manifest_path = table_location_ + "/metadata/deleted.avro"; + manifest.content = ManifestContent::kData; + manifest.added_snapshot_id = kInvalidSnapshotId; + manifest.deleted_files_count = 1; // has_deleted_files() returns true + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithAssignedSnapshotId) { + ManifestFile manifest; + manifest.manifest_path = table_location_ + "/metadata/snap.avro"; + manifest.content = ManifestContent::kData; + manifest.added_snapshot_id = 12345; // already assigned + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithFirstRowId) { + ManifestFile manifest; + manifest.manifest_path = table_location_ + "/metadata/rowid.avro"; + manifest.content = ManifestContent::kData; + manifest.added_snapshot_id = kInvalidSnapshotId; + manifest.first_row_id = 0; // assigned first_row_id + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); +} + +// ------------------------------------------------------------------------- +// AddManifest — basic commit (inherit path: v2 with can_inherit_snapshot_id) +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, AppendManifestEmptyTable) { + auto path = table_location_ + "/metadata/input.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_, file_b_})); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AppendManifest(manifest), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + + // In v2 with snapshot ID inheritance, the manifest path is reused directly. + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + ASSERT_EQ(data_manifests.size(), 1); + + EXPECT_EQ(snapshot->summary.at("added-data-files"), "2"); + EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); +} + +TEST_F(MergingSnapshotUpdateTest, AppendManifestWithDataFiles) { + // Mix AddDataFile + AddManifest — should produce 2 manifests. + auto path = table_location_ + "/metadata/input.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_, file_b_})); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); // file_b_ staged directly + EXPECT_THAT(op->AppendManifest(manifest), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + // Written manifest (file_b_) + appended manifest (file_a_, file_b_) + EXPECT_EQ(data_manifests.size(), 2); + EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); +} + +// ------------------------------------------------------------------------- +// AddManifest — merge behavior +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, AppendManifestMergeWithMinCountOne) { + // Set min-count-to-merge = 1 so all manifests are merged. + ICEBERG_UNWRAP_OR_FAIL(auto props, table_->NewUpdateProperties()); + props->Set(std::string(TableProperties::kManifestMinMergeCount.key()), "1"); + EXPECT_THAT(props->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + auto path = table_location_ + "/metadata/input.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_, file_b_})); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->AppendManifest(manifest), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + // Both manifests merged into one. + EXPECT_EQ(data_manifests.size(), 1); + EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); +} + +TEST_F(MergingSnapshotUpdateTest, AppendManifestDoNotMergeMinCount) { + // Set min-count-to-merge = 4 so 3 manifests are not merged. + ICEBERG_UNWRAP_OR_FAIL(auto props, table_->NewUpdateProperties()); + props->Set(std::string(TableProperties::kManifestMinMergeCount.key()), "4"); + EXPECT_THAT(props->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + auto path1 = table_location_ + "/metadata/m1.avro"; + auto path2 = table_location_ + "/metadata/m2.avro"; + auto path3 = table_location_ + "/metadata/m3.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(path1, {file_a_})); + ICEBERG_UNWRAP_OR_FAIL(auto m2, WriteManifest(path2, {file_b_})); + ICEBERG_UNWRAP_OR_FAIL( + auto m3, WriteManifest(path3, {MakeDataFile("/data/file_c.parquet", 3L)})); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AppendManifest(m1), IsOk()); + EXPECT_THAT(op->AppendManifest(m2), IsOk()); + EXPECT_THAT(op->AppendManifest(m3), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + // Below min-count-to-merge threshold — all 3 pass through unchanged. + EXPECT_EQ(data_manifests.size(), 3); + EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); +} + +// ------------------------------------------------------------------------- +// Manifest merge — data files only +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, ManifestMergeMergesIntoOne) { + // Set min-count-to-merge = 1 so every append triggers a merge. + ICEBERG_UNWRAP_OR_FAIL(auto props, table_->NewUpdateProperties()); + props->Set(std::string(TableProperties::kManifestMinMergeCount.key()), "1"); + EXPECT_THAT(props->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + // Snapshot 1: file_a_ + CommitFileA(); + + // Snapshot 2: file_b_ — should merge with existing manifest. + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + EXPECT_EQ(data_manifests.size(), 1); + EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsReplaced), "1"); +} + +TEST_F(MergingSnapshotUpdateTest, ManifestMergeDoesNotMergeWhenBelowMinCount) { + // Default min-count-to-merge = 100, so manifests are not merged. + CommitFileA(); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + EXPECT_EQ(data_manifests.size(), 2); + EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); +} + +TEST_F(MergingSnapshotUpdateTest, ManifestMergeDoesNotMergeWhenSizeTargetTooSmall) { + // Set a tiny size target so manifests never merge. + ICEBERG_UNWRAP_OR_FAIL(auto props, table_->NewUpdateProperties()); + props->Set(std::string(TableProperties::kManifestTargetSizeBytes.key()), "10"); + EXPECT_THAT(props->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + CommitFileA(); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + EXPECT_EQ(data_manifests.size(), 2); +} + +// ------------------------------------------------------------------------- +// Manifest count summary +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, SummaryManifestCountsOnFirstCommit) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsCreated), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsReplaced), "0"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsKept), "0"); +} + +TEST_F(MergingSnapshotUpdateTest, SummaryManifestCountsOnSecondCommitNoMerge) { + CommitFileA(); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + // 1 new manifest created, 1 existing manifest kept, 0 replaced. + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsCreated), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsReplaced), "0"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsKept), "1"); +} + +TEST_F(MergingSnapshotUpdateTest, SummaryManifestCountsAfterMerge) { + ICEBERG_UNWRAP_OR_FAIL(auto props, table_->NewUpdateProperties()); + props->Set(std::string(TableProperties::kManifestMinMergeCount.key()), "1"); + EXPECT_THAT(props->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + CommitFileA(); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + // 1 merged output created, 1 existing manifest replaced, 0 kept. + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsCreated), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsReplaced), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsKept), "0"); +} + +TEST_F(MergingSnapshotUpdateTest, SummaryManifestCountsAfterDelete) { + ICEBERG_UNWRAP_OR_FAIL(auto props, table_->NewUpdateProperties()); + props->Set(std::string(TableProperties::kManifestMinMergeCount.key()), "1"); + EXPECT_THAT(props->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + CommitFileA(); + + // Delete file_a_ — filter manager rewrites the manifest. + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + // Filter rewrites 1 manifest (replaced), merge produces 1 output (created). + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsReplaced), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsCreated), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsKept), "0"); +} + +// ------------------------------------------------------------------------- +// DataSpec — multiple partition specs +// ------------------------------------------------------------------------- + +TEST_F(MergingSnapshotUpdateTest, DataSpecThrowsWithMultipleSpecs) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + // file_a_ and file_b_ both use spec_id 0 — DataSpec() should succeed. + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->DataSpec(), IsOk()); +} + +TEST_F(MergingSnapshotUpdateTest, DataSpecThrowsWhenEmpty) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + // No files added — DataSpec() should fail. + EXPECT_THAT(op->DataSpec(), IsError(ErrorKind::kInvalidArgument)); +} + +} // namespace iceberg diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc new file mode 100644 index 000000000..f2d0c93e0 --- /dev/null +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -0,0 +1,870 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/update/merging_snapshot_update.h" + +#include +#include +#include +#include + +#include "iceberg/constants.h" +#include "iceberg/delete_file_index.h" +#include "iceberg/expression/expressions.h" +#include "iceberg/expression/inclusive_metrics_evaluator.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_list.h" +#include "iceberg/manifest/manifest_reader.h" +#include "iceberg/manifest/manifest_util_internal.h" +#include "iceberg/manifest/manifest_writer.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/table.h" +#include "iceberg/table_metadata.h" +#include "iceberg/table_properties.h" +#include "iceberg/transaction.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/snapshot_util_internal.h" + +namespace iceberg { + +MergingSnapshotUpdate::MergingSnapshotUpdate(std::string table_name, + std::shared_ptr ctx) + : SnapshotUpdate(std::move(ctx)), + table_name_(std::move(table_name)), + delete_expression_(Expressions::AlwaysFalse()), + data_filter_manager_(ManifestContent::kData, ctx_->table->io()), + delete_filter_manager_(ManifestContent::kDeletes, ctx_->table->io()), + data_merge_manager_( + base().properties.Get(TableProperties::kManifestTargetSizeBytes), + base().properties.Get(TableProperties::kManifestMinMergeCount), + base().properties.Get(TableProperties::kManifestMergeEnabled)), + delete_merge_manager_( + base().properties.Get(TableProperties::kManifestTargetSizeBytes), + base().properties.Get(TableProperties::kManifestMinMergeCount), + base().properties.Get(TableProperties::kManifestMergeEnabled)) {} + +// ------------------------------------------------------------------------- +// Primitive API +// ------------------------------------------------------------------------- + +Status MergingSnapshotUpdate::AddDataFile(std::shared_ptr file) { + if (!file) { + return InvalidArgument("Cannot add a null data file"); + } + if (!file->partition_spec_id.has_value()) { + return InvalidArgument("Data file must have a partition spec ID"); + } + + int32_t spec_id = file->partition_spec_id.value(); + ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(spec_id)); + + // Suppress first_row_id — it will be assigned by the commit, not inherited from the + // source file. + file->first_row_id = std::nullopt; + + auto& data_files = new_data_files_by_spec_[spec_id]; + auto [it, inserted] = data_files.insert(file); + if (inserted) { + has_new_data_files_ = true; + ICEBERG_RETURN_UNEXPECTED(added_data_files_summary_.AddedFile(*spec, *file)); + } + return {}; +} + +Status MergingSnapshotUpdate::ValidateNewDeleteFile(const DataFile& file) { + if (file.content == DataFile::Content::kData) { + return InvalidArgument("Expected a delete file but got a data file: {}", + file.file_path); + } + const int8_t format_version = base().format_version; + const bool is_dv = file.referenced_data_file.has_value(); + switch (format_version) { + case 1: + return InvalidArgument("Deletes are supported in V2 and above"); + case 2: + // Position deletes must NOT be DVs in v2. + if (file.content == DataFile::Content::kPositionDeletes && is_dv) { + return InvalidArgument("Must not use DVs for position deletes in V2: {}", + file.file_path); + } + break; + default: + if (format_version >= 3) { + // Position deletes MUST be DVs in v3+. + if (file.content == DataFile::Content::kPositionDeletes && !is_dv) { + return InvalidArgument("Must use DVs for position deletes in V{}: {}", + format_version, file.file_path); + } + } else { + return InvalidArgument("Unsupported format version: {}", format_version); + } + break; + } + return {}; +} + +Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file) { + if (!file) { + return InvalidArgument("Cannot add a null delete file"); + } + ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(*file)); + if (!file->partition_spec_id.has_value()) { + return InvalidArgument("Delete file must have a partition spec ID"); + } + ICEBERG_ASSIGN_OR_RAISE(auto spec, + base().PartitionSpecById(file->partition_spec_id.value())); + ICEBERG_RETURN_UNEXPECTED(added_delete_files_summary_.AddedFile(*spec, *file)); + has_new_delete_files_ = true; + new_delete_files_.push_back(std::move(file)); + return {}; +} + +Status MergingSnapshotUpdate::DeleteDataFile(std::shared_ptr file) { + if (!file) { + return InvalidArgument("Cannot delete a null data file"); + } + return data_filter_manager_.DeleteFile(std::move(file)); +} + +Status MergingSnapshotUpdate::DeleteDeleteFile(std::shared_ptr file) { + if (!file) { + return InvalidArgument("Cannot delete a null delete file"); + } + return delete_filter_manager_.DeleteFile(std::move(file)); +} + +void MergingSnapshotUpdate::DeleteByPath(std::string_view path) { + data_filter_manager_.DeleteFile(path); +} + +Status MergingSnapshotUpdate::DeleteByRowFilter(std::shared_ptr expr) { + // If a delete file matches the row filter, it can also be removed because the rows + // it references will also be deleted. Both filter managers receive the expression. + delete_expression_ = expr; + ICEBERG_RETURN_UNEXPECTED(data_filter_manager_.DeleteByRowFilter(expr)); + return delete_filter_manager_.DeleteByRowFilter(std::move(expr)); +} + +void MergingSnapshotUpdate::DropPartition(int32_t spec_id, PartitionValues partition) { + // Dropping data in a partition also drops all delete files in that partition. + data_filter_manager_.DropPartition(spec_id, partition); + delete_filter_manager_.DropPartition(spec_id, std::move(partition)); +} + +void MergingSnapshotUpdate::FailMissingDeletePaths() { + data_filter_manager_.FailMissingDeletePaths(); + delete_filter_manager_.FailMissingDeletePaths(); +} + +void MergingSnapshotUpdate::FailAnyDelete() { + data_filter_manager_.FailAnyDelete(); + delete_filter_manager_.FailAnyDelete(); +} + +void MergingSnapshotUpdate::SetNewDataFilesDataSequenceNumber(int64_t sequence_number) { + new_data_files_data_seq_number_ = sequence_number; +} + +void MergingSnapshotUpdate::CaseSensitive(bool case_sensitive) { + case_sensitive_ = case_sensitive; + data_filter_manager_.CaseSensitive(case_sensitive); + delete_filter_manager_.CaseSensitive(case_sensitive); +} + +void MergingSnapshotUpdate::Set(const std::string& property, const std::string& value) { + summary_builder().Set(property, value); +} + +Result> MergingSnapshotUpdate::DataSpec() const { + if (new_data_files_by_spec_.empty()) { + return InvalidArgument("DataSpec() called before any data file was added"); + } + if (new_data_files_by_spec_.size() > 1) { + return InvalidArgument( + "DataSpec() requires exactly one partition spec; got {} different specs", + new_data_files_by_spec_.size()); + } + return base().PartitionSpecById(new_data_files_by_spec_.begin()->first); +} + +std::vector> MergingSnapshotUpdate::AddedDataFiles() const { + std::vector> result; + for (const auto& [spec_id, files] : new_data_files_by_spec_) { + for (const auto& file : files) { + result.push_back(file); + } + } + return result; +} + +Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr /*file*/, + int64_t /*data_sequence_number*/) { + return NotImplemented( + "AddDeleteFile with explicit data sequence number is not yet implemented"); +} + +Status MergingSnapshotUpdate::AddManifest(ManifestFile manifest) { + if (manifest.content != ManifestContent::kData) { + return InvalidArgument("Cannot append delete manifest: {}", manifest.manifest_path); + } + if (manifest.has_existing_files()) { + return InvalidArgument("Cannot append manifest with existing files: {}", + manifest.manifest_path); + } + if (manifest.has_deleted_files()) { + return InvalidArgument("Cannot append manifest with deleted files: {}", + manifest.manifest_path); + } + if (manifest.added_snapshot_id != kInvalidSnapshotId) { + return InvalidArgument("Snapshot id must be assigned during commit: {}", + manifest.manifest_path); + } + if (manifest.first_row_id.has_value()) { + return InvalidArgument("Cannot append manifest with assigned first_row_id: {}", + manifest.manifest_path); + } + + if (can_inherit_snapshot_id()) { + appended_manifests_summary_.AddedManifest(manifest); + append_manifests_.push_back(std::move(manifest)); + } else { + ICEBERG_ASSIGN_OR_RAISE(auto copied, CopyManifest(manifest)); + rewritten_append_manifests_.push_back(std::move(copied)); + } + return {}; +} + +Result MergingSnapshotUpdate::CopyManifest(const ManifestFile& manifest) { + const TableMetadata& current = base(); + ICEBERG_ASSIGN_OR_RAISE(auto schema, current.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto spec, + current.PartitionSpecById(manifest.partition_spec_id)); + std::string path = ManifestPath(); + all_written_manifests_.insert(path); + return CopyAppendManifest(manifest, ctx_->table->io(), schema, spec, SnapshotId(), path, + current.format_version, &appended_manifests_summary_); +} + +// ------------------------------------------------------------------------- +// State queries +// ------------------------------------------------------------------------- + +bool MergingSnapshotUpdate::AddsDataFiles() const { + return !new_data_files_by_spec_.empty(); +} + +bool MergingSnapshotUpdate::AddsDeleteFiles() const { return !new_delete_files_.empty(); } + +bool MergingSnapshotUpdate::DeletesDataFiles() const { + return data_filter_manager_.ContainsDeletes(); +} + +bool MergingSnapshotUpdate::DeletesDeleteFiles() const { + return delete_filter_manager_.ContainsDeletes(); +} + +// ------------------------------------------------------------------------- +// Apply pipeline +// ------------------------------------------------------------------------- + +ManifestWriterFactory MergingSnapshotUpdate::MakeTrackedWriterFactory() { + return [this](int32_t spec_id, + ManifestContent content) -> Result> { + const TableMetadata& meta = base(); + ICEBERG_ASSIGN_OR_RAISE(auto schema, meta.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto spec, meta.PartitionSpecById(spec_id)); + std::string path = ManifestPath(); + all_written_manifests_.insert(path); + return ManifestWriter::MakeWriter(meta.format_version, SnapshotId(), std::move(path), + ctx_->table->io(), std::move(spec), + std::move(schema), content); + }; +} + +Result> MergingSnapshotUpdate::WriteNewDataManifests() { + // If new files were staged after the cache was populated (commit retry), invalidate. + if (has_new_data_files_ && cached_new_data_manifests_.has_value()) { + for (const auto& m : *cached_new_data_manifests_) { + std::ignore = DeleteFile(m.manifest_path); + } + cached_new_data_manifests_.reset(); + } + + if (cached_new_data_manifests_.has_value()) { + return *cached_new_data_manifests_; + } + + std::vector result; + for (const auto& [spec_id, data_files] : new_data_files_by_spec_) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(spec_id)); + ICEBERG_ASSIGN_OR_RAISE( + auto written, + WriteDataManifests(data_files.as_span(), spec, new_data_files_data_seq_number_)); + for (const auto& m : written) { + all_written_manifests_.insert(m.manifest_path); + } + result.insert(result.end(), std::make_move_iterator(written.begin()), + std::make_move_iterator(written.end())); + } + + cached_new_data_manifests_ = result; + has_new_data_files_ = false; + return result; +} + +Result> MergingSnapshotUpdate::WriteNewDeleteManifests() { + // If new files were staged after the cache was populated (commit retry), invalidate. + if (has_new_delete_files_ && cached_new_delete_manifests_.has_value()) { + for (const auto& m : *cached_new_delete_manifests_) { + std::ignore = DeleteFile(m.manifest_path); + } + cached_new_delete_manifests_.reset(); + } + + if (cached_new_delete_manifests_.has_value()) { + return *cached_new_delete_manifests_; + } + + // Group delete files by partition spec ID, mirroring WriteNewDataManifests(). + std::unordered_map>> + delete_files_by_spec; + for (const auto& file : new_delete_files_) { + delete_files_by_spec[file->partition_spec_id.value()].push_back(file); + } + + std::vector result; + for (const auto& [spec_id, delete_files] : delete_files_by_spec) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto written, + WriteDeleteManifests(std::span(delete_files), spec)); + for (const auto& m : written) { + all_written_manifests_.insert(m.manifest_path); + } + result.insert(result.end(), std::make_move_iterator(written.begin()), + std::make_move_iterator(written.end())); + } + + cached_new_delete_manifests_ = result; + has_new_delete_files_ = false; + return result; +} + +Result> MergingSnapshotUpdate::Apply( + const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) { + // Re-validate buffered delete files against the current format version. A format + // upgrade between staging and commit could make previously-valid files invalid. + for (const auto& file : new_delete_files_) { + ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(*file)); + } + + // Rebuild summary from stable sub-builders so that commit retries don't double-count. + summary_builder().Clear(); + summary_builder().Merge(added_data_files_summary_); + summary_builder().Merge(added_delete_files_summary_); + summary_builder().Merge(appended_manifests_summary_); + + auto tracked_factory = MakeTrackedWriterFactory(); + + // Step 1: Filter data manifests. + ICEBERG_ASSIGN_OR_RAISE(auto filtered_data, + data_filter_manager_.FilterManifests( + metadata_to_update, snapshot, tracked_factory)); + + // Track deleted data files in the summary builder. + for (const auto& file : data_filter_manager_.FilesToBeDeleted()) { + if (!file->partition_spec_id.has_value()) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE( + auto spec, metadata_to_update.PartitionSpecById(*file->partition_spec_id)); + ICEBERG_RETURN_UNEXPECTED(summary_builder().DeletedFile(*spec, *file)); + } + + // Step 2: Compute min data sequence number; set up delete filter cleanup. + // Use last_sequence_number as the initial value so that an empty filtered list + // produces a sensible minimum. Skip manifests with kUnassignedSequenceNumber — + // those are manifests written in the current Apply() call whose sequence number + // hasn't been assigned yet. If all filtered manifests are unassigned (e.g. the + // table has no pre-existing data manifests), the fallback to last_sequence_number + // is safe: any delete file with seq > 0 and seq <= last_sequence_number can no + // longer match live data rows, so cleaning them up is correct. + int64_t min_data_seq = metadata_to_update.last_sequence_number; + for (const auto& manifest : filtered_data) { + if (manifest.min_sequence_number != kUnassignedSequenceNumber) { + min_data_seq = std::min(min_data_seq, manifest.min_sequence_number); + } + } + delete_filter_manager_.DropDeleteFilesOlderThan(min_data_seq); + delete_filter_manager_.RemoveDanglingDeletesFor( + data_filter_manager_.FilesToBeDeleted()); + + // Step 3: Filter delete manifests. + ICEBERG_ASSIGN_OR_RAISE(auto filtered_deletes, + delete_filter_manager_.FilterManifests( + metadata_to_update, snapshot, tracked_factory)); + + // Track deleted delete files in the summary builder. + for (const auto& file : delete_filter_manager_.FilesToBeDeleted()) { + if (!file->partition_spec_id.has_value()) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE( + auto spec, metadata_to_update.PartitionSpecById(*file->partition_spec_id)); + ICEBERG_RETURN_UNEXPECTED(summary_builder().DeletedFile(*spec, *file)); + } + + // Drop manifests with no live files — they carry no data and should not be merged + // into the new snapshot. Manifests written by the current snapshot are always kept + // regardless of live-file counts; the merge stage handles any that are empty. + int64_t snapshot_id = SnapshotId(); + auto should_keep = [snapshot_id](const ManifestFile& m) { + return m.has_added_files() || m.has_existing_files() || + m.added_snapshot_id == snapshot_id; + }; + std::erase_if(filtered_data, [&](const ManifestFile& m) { return !should_keep(m); }); + std::erase_if(filtered_deletes, [&](const ManifestFile& m) { return !should_keep(m); }); + + // Step 4: Write (or retrieve cached) new data manifests. + ICEBERG_ASSIGN_OR_RAISE(auto written_data_manifests, WriteNewDataManifests()); + + // Incorporate append manifests (from AddManifest), stamping each with the + // current snapshot ID. append_manifests_ are used directly (inherit path); + // rewritten_append_manifests_ were already copied with the snapshot ID. + std::vector new_data_manifests = std::move(written_data_manifests); + for (const auto& src : append_manifests_) { + ManifestFile m = src; + m.added_snapshot_id = snapshot_id; + new_data_manifests.push_back(std::move(m)); + } + for (const auto& src : rewritten_append_manifests_) { + ManifestFile m = src; + m.added_snapshot_id = snapshot_id; + new_data_manifests.push_back(std::move(m)); + } + + // Step 5: Write (or retrieve cached) new delete manifests. + ICEBERG_ASSIGN_OR_RAISE(auto new_delete_manifests, WriteNewDeleteManifests()); + + // Step 6: Merge data manifests. + ICEBERG_ASSIGN_OR_RAISE(auto merged_data, + data_merge_manager_.MergeManifests( + filtered_data, new_data_manifests, SnapshotId(), + metadata_to_update, ctx_->table->io(), tracked_factory)); + + // Step 7: Merge delete manifests. + ICEBERG_ASSIGN_OR_RAISE(auto merged_deletes, + delete_merge_manager_.MergeManifests( + filtered_deletes, new_delete_manifests, SnapshotId(), + metadata_to_update, ctx_->table->io(), tracked_factory)); + + std::vector result; + result.reserve(merged_data.size() + merged_deletes.size()); + result.insert(result.end(), std::make_move_iterator(merged_data.begin()), + std::make_move_iterator(merged_data.end())); + result.insert(result.end(), std::make_move_iterator(merged_deletes.begin()), + std::make_move_iterator(merged_deletes.end())); + + // Manifest count summary. + int32_t manifests_created = 0; + int32_t manifests_kept = 0; + for (const auto& m : result) { + if (m.added_snapshot_id == snapshot_id) { + ++manifests_created; + } else { + ++manifests_kept; + } + } + int32_t replaced_manifests_count = data_filter_manager_.ReplacedManifestsCount() + + delete_filter_manager_.ReplacedManifestsCount() + + data_merge_manager_.ReplacedManifestsCount() + + delete_merge_manager_.ReplacedManifestsCount(); + summary_builder().SetManifestCounts(manifests_created, manifests_kept, + replaced_manifests_count); + + return result; +} + +void MergingSnapshotUpdate::CleanUncommitted( + const std::unordered_set& committed) { + for (const auto& path : all_written_manifests_) { + if (!committed.contains(path)) { + std::ignore = DeleteFile(path); + } + } + all_written_manifests_.clear(); + cached_new_data_manifests_.reset(); + cached_new_delete_manifests_.reset(); + has_new_data_files_ = false; + has_new_delete_files_ = false; + + // rewritten_append_manifests_ are always owned by the table (copied by us), + // so delete any that were not committed. + for (const auto& m : rewritten_append_manifests_) { + if (!committed.contains(m.manifest_path)) { + std::ignore = DeleteFile(m.manifest_path); + } + } + + // append_manifests_ are only owned by the table if the commit succeeded + // (i.e., at least one manifest was committed). + if (!committed.empty()) { + for (const auto& m : append_manifests_) { + if (!committed.contains(m.manifest_path)) { + std::ignore = DeleteFile(m.manifest_path); + } + } + } +} + +std::unordered_map MergingSnapshotUpdate::Summary() { + summary_builder().SetPartitionSummaryLimit( + base().properties.Get(TableProperties::kWritePartitionSummaryLimit)); + return summary_builder().Build(); +} + +// ------------------------------------------------------------------------- +// Conflict-detection helpers +// ------------------------------------------------------------------------- + +Status MergingSnapshotUpdate::ValidateAddedDataFiles( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr filter, const std::shared_ptr& parent, + std::shared_ptr io, bool case_sensitive) { + if (parent == nullptr) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto ancestors, + SnapshotUtil::AncestorsBetween(metadata, parent->snapshot_id, + starting_snapshot_id)); + + // Build the full set of matching snapshot IDs first, then scan their manifests. + // The full set must be known before filtering manifests, since a manifest may have + // been written by a different snapshot in the ancestry range. + std::unordered_set matching_snapshot_ids; + for (const auto& snap : ancestors) { + auto op = snap->Operation(); + if (op == DataOperation::kAppend || op == DataOperation::kOverwrite) { + matching_snapshot_ids.insert(snap->snapshot_id); + } + } + + std::unique_ptr evaluator; + if (filter != nullptr) { + ICEBERG_ASSIGN_OR_RAISE( + evaluator, InclusiveMetricsEvaluator::Make(filter, *schema, case_sensitive)); + } + + for (const auto& snapshot : ancestors) { + if (!matching_snapshot_ids.contains(snapshot->snapshot_id)) { + continue; + } + auto cached = SnapshotCache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto data_manifests, cached.DataManifests(io)); + + for (const auto& manifest : data_manifests) { + if (!matching_snapshot_ids.contains(manifest.added_snapshot_id)) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE(auto spec, + metadata.PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto reader, + ManifestReader::Make(manifest, io, schema, spec)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); + + for (const auto& entry : entries) { + if (entry.status != ManifestStatus::kAdded) { + continue; + } + if (entry.data_file == nullptr) { + continue; + } + if (evaluator != nullptr) { + ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*entry.data_file)); + if (!matches) { + continue; + } + } + return InvalidArgument( + "Found conflicting files that can contain rows matching {}:" + " {} in snapshot {}", + filter != nullptr ? filter->ToString() : "any expression", + entry.data_file->file_path, snapshot->snapshot_id); + } + } + } + return {}; +} + +Status MergingSnapshotUpdate::ValidateDataFilesExist( + const TableMetadata& metadata, int64_t starting_snapshot_id, + const std::unordered_set& file_paths, bool allow_deletes, + std::shared_ptr filter, const std::shared_ptr& parent, + std::shared_ptr io, bool case_sensitive) { + if (parent == nullptr || file_paths.empty()) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto ancestors, + SnapshotUtil::AncestorsBetween(metadata, parent->snapshot_id, + starting_snapshot_id)); + + // Build the full set of matching snapshot IDs first, then scan their manifests. + // The full set must be known before filtering manifests, since a manifest may have + // been written by a different snapshot in the ancestry range. + // Included operations: OVERWRITE and REPLACE always; DELETE when allow_deletes is + // false. + std::unordered_set matching_snapshot_ids; + for (const auto& snap : ancestors) { + auto op = snap->Operation(); + if (op == DataOperation::kOverwrite || op == DataOperation::kReplace) { + matching_snapshot_ids.insert(snap->snapshot_id); + } else if (!allow_deletes && op == DataOperation::kDelete) { + matching_snapshot_ids.insert(snap->snapshot_id); + } + } + + // Build a metrics evaluator for the conflict-detection filter, if provided. + std::unique_ptr evaluator; + if (filter != nullptr) { + ICEBERG_ASSIGN_OR_RAISE( + evaluator, InclusiveMetricsEvaluator::Make(filter, *schema, case_sensitive)); + } + + for (const auto& snapshot : ancestors) { + if (!matching_snapshot_ids.contains(snapshot->snapshot_id)) { + continue; + } + auto cached = SnapshotCache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto data_manifests, cached.DataManifests(io)); + + for (const auto& manifest : data_manifests) { + if (!matching_snapshot_ids.contains(manifest.added_snapshot_id)) { + continue; + } + ICEBERG_ASSIGN_OR_RAISE(auto spec, + metadata.PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto reader, + ManifestReader::Make(manifest, io, schema, spec)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); + + for (const auto& entry : entries) { + if (entry.status != ManifestStatus::kDeleted) { + continue; + } + if (entry.data_file == nullptr) { + continue; + } + if (!file_paths.contains(entry.data_file->file_path)) { + continue; + } + if (evaluator != nullptr) { + ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*entry.data_file)); + if (!matches) { + continue; + } + } + return InvalidArgument("Cannot commit, missing data files: {} in snapshot {}", + entry.data_file->file_path, snapshot->snapshot_id); + } + } + } + return {}; +} + +Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( + const TableMetadata& metadata, int64_t starting_snapshot_id, + const DataFileSet& replaced_files, const std::shared_ptr& parent, + std::shared_ptr io, bool ignore_equality_deletes) { + if (parent == nullptr || replaced_files.empty() || metadata.format_version < 2) { + return {}; + } + + // Build an index of delete files added since starting_snapshot_id. + // Covers both position and equality deletes; the caller controls whether + // equality deletes are ignored. + ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, + nullptr, nullptr, parent, io)); + + if (deletes->empty()) { + return {}; + } + + // Compute the starting sequence number for the data file check. + int64_t starting_seq = TableMetadata::kInitialSequenceNumber; + if (auto snap_result = metadata.SnapshotById(starting_snapshot_id); + snap_result.has_value()) { + starting_seq = snap_result.value()->sequence_number; + } + + for (const auto& data_file : replaced_files) { + ICEBERG_ASSIGN_OR_RAISE(auto delete_files, + deletes->ForDataFile(starting_seq, *data_file)); + if (ignore_equality_deletes) { + // Only fail on position deletes — equality deletes at higher sequence numbers + // still apply to the rewritten files and are not a conflict. + for (const auto& df : delete_files) { + if (df->content == DataFile::Content::kPositionDeletes) { + return InvalidArgument( + "Cannot commit, found new position delete for replaced data file: {}", + data_file->file_path); + } + } + } else { + if (!delete_files.empty()) { + return InvalidArgument( + "Cannot commit, found new delete for replaced data file: {}", + data_file->file_path); + } + } + } + return {}; +} + +Status MergingSnapshotUpdate::ValidateAddedDataFiles( + const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, + const PartitionSet& /*partition_set*/, const std::shared_ptr& /*parent*/, + std::shared_ptr /*io*/) { + return NotImplemented( + "ValidateAddedDataFiles with PartitionSet is not yet implemented"); +} + +Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( + const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, + std::shared_ptr /*data_filter*/, const DataFileSet& /*replaced_files*/, + const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { + return NotImplemented( + "ValidateNoNewDeletesForDataFiles with data filter is not yet implemented"); +} + +Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( + const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, + std::shared_ptr /*data_filter*/, + const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { + return NotImplemented( + "ValidateNoNewDeleteFiles with Expression is not yet implemented"); +} + +Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( + const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, + const PartitionSet& /*partition_set*/, const std::shared_ptr& /*parent*/, + std::shared_ptr /*io*/) { + return NotImplemented( + "ValidateNoNewDeleteFiles with PartitionSet is not yet implemented"); +} + +Status MergingSnapshotUpdate::ValidateDeletedDataFiles( + const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, + std::shared_ptr /*data_filter*/, + const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { + return NotImplemented( + "ValidateDeletedDataFiles with Expression is not yet implemented"); +} + +Status MergingSnapshotUpdate::ValidateDeletedDataFiles( + const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, + const PartitionSet& /*partition_set*/, const std::shared_ptr& /*parent*/, + std::shared_ptr /*io*/) { + return NotImplemented( + "ValidateDeletedDataFiles with PartitionSet is not yet implemented"); +} + +Result> MergingSnapshotUpdate::AddedDeleteFiles( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, std::shared_ptr partition_set, + const std::shared_ptr& parent, std::shared_ptr io, + bool case_sensitive) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + + if (parent == nullptr || metadata.format_version < 2) { + ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, + TableMetadataCache(&metadata).GetPartitionSpecsById()); + std::unordered_map> specs_by_id( + specs_ref.get().begin(), specs_ref.get().end()); + ICEBERG_ASSIGN_OR_RAISE(auto builder, DeleteFileIndex::BuilderFor( + io, schema, std::move(specs_by_id), {})); + return builder.Build(); + } + + ICEBERG_ASSIGN_OR_RAISE(auto ancestors, + SnapshotUtil::AncestorsBetween(metadata, parent->snapshot_id, + starting_snapshot_id)); + + // Collect delete manifests from OVERWRITE and DELETE snapshots only. + std::unordered_set matching_snapshot_ids; + for (const auto& snap : ancestors) { + auto op = snap->Operation(); + if (op == DataOperation::kOverwrite || op == DataOperation::kDelete) { + matching_snapshot_ids.insert(snap->snapshot_id); + } + } + + std::vector delete_manifests; + for (const auto& snapshot : ancestors) { + if (!matching_snapshot_ids.contains(snapshot->snapshot_id)) { + continue; + } + auto cached = SnapshotCache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, cached.DeleteManifests(io)); + for (const auto& m : manifests) { + if (matching_snapshot_ids.contains(m.added_snapshot_id)) { + delete_manifests.push_back(m); + } + } + } + + // Compute the starting sequence number from the starting snapshot. + int64_t starting_seq = TableMetadata::kInitialSequenceNumber; + if (auto snap_result = metadata.SnapshotById(starting_snapshot_id); + snap_result.has_value()) { + starting_seq = snap_result.value()->sequence_number; + } + + ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, + TableMetadataCache(&metadata).GetPartitionSpecsById()); + std::unordered_map> specs_by_id( + specs_ref.get().begin(), specs_ref.get().end()); + + ICEBERG_ASSIGN_OR_RAISE(auto builder, + DeleteFileIndex::BuilderFor(io, schema, std::move(specs_by_id), + std::move(delete_manifests))); + builder.AfterSequenceNumber(starting_seq); + builder.CaseSensitive(case_sensitive); + if (data_filter != nullptr) { + builder.DataFilter(std::move(data_filter)); + } + if (partition_set != nullptr) { + builder.FilterPartitions(std::move(partition_set)); + } + return builder.Build(); +} + +Status MergingSnapshotUpdate::ValidateAddedDVs( + const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, + std::shared_ptr /*conflict_filter*/, + const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { + return NotImplemented( + "ValidateAddedDVs is not yet supported (deletion vectors require format v3)"); +} + +} // namespace iceberg diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h new file mode 100644 index 000000000..a4030bb4d --- /dev/null +++ b/src/iceberg/update/merging_snapshot_update.h @@ -0,0 +1,350 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/update/merging_snapshot_update.h + +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/delete_file_index.h" +#include "iceberg/iceberg_export.h" +#include "iceberg/manifest/manifest_filter_manager.h" +#include "iceberg/manifest/manifest_merge_manager.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" +#include "iceberg/update/snapshot_update.h" +#include "iceberg/util/data_file_set.h" + +namespace iceberg { + +/// \brief Abstract base class for all merge-based snapshot write operations. +/// +/// Provides the complete filter → write → merge pipeline that all merge-based +/// operations (MergeAppend, OverwriteFiles, RowDelta, ReplacePartitions, +/// RewriteFiles) share. Subclasses only need to implement `operation()` and +/// call the protected primitive API to describe what changes to make. +/// +/// The Apply() pipeline: +/// 1. Filter data manifests (via data_filter_manager_) +/// 2. Compute min data sequence number and set up delete filter cleanup +/// 3. Filter delete manifests (via delete_filter_manager_) +/// 4. Write new data manifests (cached for commit retry) +/// 5. Write new delete manifests (cached for commit retry) +/// 6. Merge data manifests (via data_merge_manager_) +/// 7. Merge delete manifests (via delete_merge_manager_) +/// +class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { + public: + ~MergingSnapshotUpdate() override = default; + + // SnapshotUpdate overrides + Result> Apply( + const TableMetadata& metadata_to_update, + const std::shared_ptr& snapshot) override; + + void CleanUncommitted(const std::unordered_set& committed) override; + + std::unordered_map Summary() override; + + /// \brief Set a custom property in the snapshot summary. + void Set(const std::string& property, const std::string& value); + + protected: + /// \brief Constructor; reads merge configuration from table properties. + explicit MergingSnapshotUpdate(std::string table_name, + std::shared_ptr ctx); + + /// \brief Stage a data file to be added to the table. + Status AddDataFile(std::shared_ptr file); + + /// \brief Stage a delete file to be added to the table. + Status AddDeleteFile(std::shared_ptr file); + + /// \brief Validate a delete file against the table format version rules. + /// + /// - Format v1: deletes are not supported. + /// - Format v2: position deletes must NOT be deletion vectors (DVs). + /// - Format v3+: position deletes MUST be deletion vectors (DVs). + Status ValidateNewDeleteFile(const DataFile& file); + + /// \brief Stage a delete file with an explicit data sequence number. + /// + /// \note Not yet implemented; returns NotImplemented error. + Status AddDeleteFile(std::shared_ptr file, int64_t data_sequence_number); + + /// \brief Add all files in a pre-existing data manifest to the new snapshot. + /// + /// The manifest must contain only DATA content and only ADDED entries (no + /// existing or deleted files). If snapshot ID inheritance is enabled and the + /// manifest has no snapshot ID assigned, it is used directly; otherwise it is + /// copied with the current snapshot ID. + Status AddManifest(ManifestFile manifest); + + /// \brief Register a data file (by object) to be deleted from the table. + Status DeleteDataFile(std::shared_ptr file); + + /// \brief Register a delete file (by object) to be removed from the table. + Status DeleteDeleteFile(std::shared_ptr file); + + /// \brief Register a data file path to be deleted from the table. + /// + /// \note Only applies to data files. To remove delete files, use DeleteDeleteFile(). + void DeleteByPath(std::string_view path); + + /// \brief Register an expression to delete matching rows. + /// + /// Both data and delete filter managers receive the expression: delete files that + /// match the row filter can also be removed because those rows will be deleted. + Status DeleteByRowFilter(std::shared_ptr expr); + + /// \brief Register a partition to be dropped. + /// + /// Both data and delete filter managers receive the partition drop, since dropping + /// data in a partition also drops all delete files in that partition. + void DropPartition(int32_t spec_id, PartitionValues partition); + + /// \brief Fail if any registered delete path is not found in any manifest. + void FailMissingDeletePaths(); + + /// \brief Fail if any manifest entry matches a delete condition. + void FailAnyDelete(); + + /// \brief Override the data sequence number assigned to all newly-added data files. + void SetNewDataFilesDataSequenceNumber(int64_t sequence_number); + + /// \brief Set case sensitivity for row filter and expression evaluation. + void CaseSensitive(bool case_sensitive); + + /// \brief Returns true if case-sensitive matching is enabled (default: true). + bool IsCaseSensitive() const { return case_sensitive_; } + + /// \brief Returns true if any data files have been staged for addition. + bool AddsDataFiles() const; + + /// \brief Returns true if any delete files have been staged for addition. + bool AddsDeleteFiles() const; + + /// \brief Returns true if any data files have been registered for deletion. + bool DeletesDataFiles() const; + + /// \brief Returns true if any delete files have been registered for removal. + bool DeletesDeleteFiles() const; + + /// \brief Returns the row-filter expression set via DeleteByRowFilter, or nullptr. + const std::shared_ptr& RowFilter() const { return delete_expression_; } + + /// \brief Returns the single partition spec for all staged data files. + /// + /// Precondition: exactly one partition spec ID must be represented among staged + /// data files. + Result> DataSpec() const; + + /// \brief Returns all data files staged for addition. + std::vector> AddedDataFiles() const; + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a data file matching the given filter expression. + static Status ValidateAddedDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr filter, + const std::shared_ptr& parent, + std::shared_ptr io, + bool case_sensitive = true); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a data file in any partition of the given partition set. + /// + /// \note Not yet implemented; returns NotImplemented error. + static Status ValidateAddedDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const PartitionSet& partition_set, + const std::shared_ptr& parent, + std::shared_ptr io); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// removed a file whose path is in file_paths (and allow_deletes is false). + static Status ValidateDataFilesExist( + const TableMetadata& metadata, int64_t starting_snapshot_id, + const std::unordered_set& file_paths, bool allow_deletes, + std::shared_ptr filter, const std::shared_ptr& parent, + std::shared_ptr io, bool case_sensitive = true); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a delete file that covers a file in replaced_files. + /// + /// Whether equality deletes are checked is derived automatically from whether + /// a custom data sequence number was set via SetNewDataFilesDataSequenceNumber(): + /// if set, equality deletes are ignored because they still apply to the rewritten + /// files and are not a conflict. + /// + /// Subclasses should prefer this overload over the static one. + Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const DataFileSet& replaced_files, + const std::shared_ptr& parent, + std::shared_ptr io) const { + return ValidateNoNewDeletesForDataFiles(metadata, starting_snapshot_id, + replaced_files, parent, io, + new_data_files_data_seq_number_.has_value()); + } + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a delete file that covers a file in replaced_files. + /// + /// \param ignore_equality_deletes If true, only position deletes are checked. + /// Set to true when replaced data files have the same sequence number as the + /// new files (e.g. RewriteFiles), so equality deletes at higher sequence numbers + /// still apply and are not a conflict. + static Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const DataFileSet& replaced_files, + const std::shared_ptr& parent, + std::shared_ptr io, + bool ignore_equality_deletes = false); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a delete file matching the data filter that covers a file in replaced_files. + /// + /// \note Not yet implemented; returns NotImplemented error. + static Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const DataFileSet& replaced_files, + const std::shared_ptr& parent, + std::shared_ptr io); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a delete file matching the given row filter. + /// + /// \note Not yet implemented; returns NotImplemented error. + static Status ValidateNoNewDeleteFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const std::shared_ptr& parent, + std::shared_ptr io); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a delete file matching any partition in the given partition set. + /// + /// \note Not yet implemented; returns NotImplemented error. + static Status ValidateNoNewDeleteFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const PartitionSet& partition_set, + const std::shared_ptr& parent, + std::shared_ptr io); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// deleted a data file matching the given row filter. + /// + /// \note Not yet implemented; returns NotImplemented error. + static Status ValidateDeletedDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const std::shared_ptr& parent, + std::shared_ptr io); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// deleted a data file in any partition of the given partition set. + /// + /// \note Not yet implemented; returns NotImplemented error. + static Status ValidateDeletedDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const PartitionSet& partition_set, + const std::shared_ptr& parent, + std::shared_ptr io); + + /// \brief Build a DeleteFileIndex of delete files added since starting_snapshot_id. + static Result> AddedDeleteFiles( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, + std::shared_ptr partition_set, + const std::shared_ptr& parent, std::shared_ptr io, + bool case_sensitive = true); + + /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] + /// added a deletion vector that conflicts with DVs being written. + /// + /// \note Deletion vectors (format v3) are not yet supported; returns NotImplemented. + static Status ValidateAddedDVs(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr conflict_filter, + const std::shared_ptr& parent, + std::shared_ptr io); + + private: + /// \brief Create a ManifestWriterFactory that records every path it creates in + /// all_written_manifests_. + ManifestWriterFactory MakeTrackedWriterFactory(); + + /// \brief Copy a manifest with the current snapshot ID, for use when snapshot + /// ID inheritance is not possible. + Result CopyManifest(const ManifestFile& manifest); + + /// \brief Write new data manifests for staged data files; caches the result. + Result> WriteNewDataManifests(); + + /// \brief Write new delete manifests for staged delete files; caches the result. + Result> WriteNewDeleteManifests(); + + // Used for commit event notifications and diagnostic log messages. + std::string table_name_; + std::shared_ptr delete_expression_; + bool case_sensitive_ = true; + + // Stable sub-builders for added files — accumulated across retries and merged + // into summary_builder_ at the start of each Apply() call. + SnapshotSummaryBuilder added_data_files_summary_; + SnapshotSummaryBuilder added_delete_files_summary_; + SnapshotSummaryBuilder appended_manifests_summary_; + + ManifestFilterManager data_filter_manager_; + ManifestFilterManager delete_filter_manager_; + ManifestMergeManager data_merge_manager_; + ManifestMergeManager delete_merge_manager_; + + std::unordered_map new_data_files_by_spec_; + std::vector> new_delete_files_; + std::optional new_data_files_data_seq_number_; + + // Manifests passed via AddManifest(): inherit path (no copy needed) and + // rewrite path (must be copied with the current snapshot ID). + std::vector append_manifests_; + std::vector rewritten_append_manifests_; + + // Set to true when new files are staged after the cache was populated, so the + // cache is invalidated and re-written on the next Apply() call (commit retry). + bool has_new_data_files_ = false; + bool has_new_delete_files_ = false; + + std::optional> cached_new_data_manifests_; + std::optional> cached_new_delete_manifests_; + + /// Tracks every manifest path created via MakeTrackedWriterFactory, plus the + /// paths in cached_new_*_manifests_. Used by CleanUncommitted(). + std::unordered_set all_written_manifests_; +}; + +} // namespace iceberg diff --git a/src/iceberg/update/snapshot_update.cc b/src/iceberg/update/snapshot_update.cc index a59ebdc72..fc5359e42 100644 --- a/src/iceberg/update/snapshot_update.cc +++ b/src/iceberg/update/snapshot_update.cc @@ -41,7 +41,7 @@ namespace iceberg { namespace { -// The Java impl skips updating total if parsing fails. Here we choose to be strict. +// Skips updating total if parsing fails would be lenient; here we choose to be strict. Status UpdateTotal(std::unordered_map& summary, const std::unordered_map& previous_summary, const std::string& total_property, const std::string& added_property, @@ -398,14 +398,10 @@ void SnapshotUpdate::CleanAll() { } Status SnapshotUpdate::DeleteFile(const std::string& path) { - static const auto kDefaultDeleteFunc = [this](const std::string& path) { - return this->ctx_->table->io()->DeleteFile(path); - }; if (delete_func_) { return delete_func_(path); - } else { - return kDefaultDeleteFunc(path); } + return ctx_->table->io()->DeleteFile(path); } std::string SnapshotUpdate::ManifestListPath() { diff --git a/src/iceberg/update/update_partition_statistics.h b/src/iceberg/update/update_partition_statistics.h index 982b1bd39..86bf1b081 100644 --- a/src/iceberg/update/update_partition_statistics.h +++ b/src/iceberg/update/update_partition_statistics.h @@ -65,9 +65,8 @@ class ICEBERG_EXPORT UpdatePartitionStatistics : public PendingUpdate { /// \brief Partition statistics updates are intentionally not retried today. /// - /// This matches the current Java `SetPartitionStatistics` behavior, which commits - /// directly without a retry loop. Keep this conservative until we add explicit replay - /// coverage for this update type. + /// Commits directly without a retry loop. Keep this conservative until we add + /// explicit replay coverage for this update type. bool IsRetryable() const override { return false; } struct ApplyResult { From 3c5d108bf739b3e472304c31e620cbefb80f33b8 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Wed, 27 May 2026 14:54:03 +0800 Subject: [PATCH 02/16] Align merging snapshot update with Java parity Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../manifest/manifest_filter_manager.cc | 18 +- .../manifest/manifest_filter_manager.h | 20 +- .../manifest/manifest_merge_manager.cc | 8 +- .../test/manifest_filter_manager_test.cc | 36 +- .../test/manifest_merge_manager_test.cc | 28 + .../test/merging_snapshot_update_test.cc | 391 +++++++++++++- src/iceberg/test/snapshot_util_test.cc | 14 +- src/iceberg/update/merging_snapshot_update.cc | 482 ++++++++++++------ src/iceberg/update/merging_snapshot_update.h | 26 +- src/iceberg/update/snapshot_update.cc | 20 +- src/iceberg/update/snapshot_update.h | 9 + src/iceberg/util/snapshot_util.cc | 4 +- src/iceberg/util/snapshot_util_internal.h | 12 +- 13 files changed, 827 insertions(+), 241 deletions(-) diff --git a/src/iceberg/manifest/manifest_filter_manager.cc b/src/iceberg/manifest/manifest_filter_manager.cc index 2b53158bc..6630b9fd4 100644 --- a/src/iceberg/manifest/manifest_filter_manager.cc +++ b/src/iceberg/manifest/manifest_filter_manager.cc @@ -94,7 +94,6 @@ void ManifestFilterManager::DeleteFile(std::string_view path) { Status ManifestFilterManager::DeleteFile(std::shared_ptr file) { ICEBERG_PRECHECK(file != nullptr, "Cannot delete file: null"); delete_paths_.insert(file->file_path); - delete_files_.insert(std::move(file)); return {}; } @@ -132,9 +131,7 @@ Result ManifestFilterManager::CanContainDroppedFiles(const ManifestFile&) // manifests once object-delete partitions are tracked separately. // Currently, DeleteFile(std::shared_ptr) degrades to a path-based delete, // which forces scanning all manifests. - // Also open delete manifests when a minimum sequence number is set for cleanup. - return !delete_paths_.empty() || !removed_data_file_paths_.empty() || - (manifest_content_ == ManifestContent::kDeletes && min_sequence_number_ > 0); + return !delete_paths_.empty() || !removed_data_file_paths_.empty(); } Result ManifestFilterManager::CanContainDroppedPartitions( @@ -385,6 +382,16 @@ Status ManifestFilterManager::ValidateRequiredDeletes( Result> ManifestFilterManager::FilterManifests( const TableMetadata& metadata, const std::shared_ptr& base_snapshot, const ManifestWriterFactory& writer_factory) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + return FilterManifests(schema, metadata, base_snapshot, writer_factory); +} + +Result> ManifestFilterManager::FilterManifests( + const std::shared_ptr& schema, const TableMetadata& metadata, + const std::shared_ptr& base_snapshot, + const ManifestWriterFactory& writer_factory) { + delete_files_.clear(); + replaced_manifests_count_ = 0; if (!base_snapshot) { ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes({})); return std::vector{}; @@ -402,7 +409,6 @@ Result> ManifestFilterManager::FilterManifests( manifests.push_back(&manifest); } - ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); TableMetadataCache metadata_cache(&metadata); ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, metadata_cache.GetPartitionSpecsById()); @@ -426,7 +432,9 @@ Result> ManifestFilterManager::FilterManifests( } std::unordered_set found_paths; + delete_files_.clear(); if (manifests.empty()) { + replaced_manifests_count_ = 0; ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(found_paths)); return std::vector{}; } diff --git a/src/iceberg/manifest/manifest_filter_manager.h b/src/iceberg/manifest/manifest_filter_manager.h index 981b9ac3b..e319623b0 100644 --- a/src/iceberg/manifest/manifest_filter_manager.h +++ b/src/iceberg/manifest/manifest_filter_manager.h @@ -85,19 +85,16 @@ class ICEBERG_EXPORT ManifestFilterManager { /// \brief Register a file object for deletion. /// /// Any manifest entry whose file_path matches file->file_path will be marked - /// DELETED. The file object is retained in FilesToBeDeleted(), allowing callers - /// to enumerate deleted file objects for follow-up delete-file cleanup. - /// Duplicate registrations (same path) are silently ignored. + /// DELETED. Duplicate registrations (same path) are silently ignored. /// /// \param file The data/delete file to delete (must not be null) Status DeleteFile(std::shared_ptr file); /// \brief Returns the set of file objects marked for deletion by this manager. /// - /// This includes files registered via DeleteFile(DataFile) and files discovered - /// during FilterManifests() that were deleted by path, partition, or row-filter - /// matching. Used by higher-level operations (e.g. RowDelta) to enumerate the - /// deleted data files for delete-file cleanup. + /// This is populated by the most recent FilterManifests() call and contains only + /// files that were actually deleted from filtered manifests. Used by higher-level + /// operations (e.g. RowDelta) to enumerate deleted data files for follow-up cleanup. const DataFileSet& FilesToBeDeleted() const; /// \brief Register a partition for dropping. @@ -159,6 +156,15 @@ class ICEBERG_EXPORT ManifestFilterManager { const TableMetadata& metadata, const std::shared_ptr& base_snapshot, const ManifestWriterFactory& writer_factory); + /// \brief Apply all accumulated delete conditions using an explicit schema. + /// + /// This overload is used when callers need row-filter evaluation bound against a + /// schema other than metadata.Schema(), such as the schema at a branch head. + Result> FilterManifests( + const std::shared_ptr& schema, const TableMetadata& metadata, + const std::shared_ptr& base_snapshot, + const ManifestWriterFactory& writer_factory); + /// \brief Apply all accumulated delete conditions to the provided manifests. /// /// This overload accepts only the context needed for filtering. It is intended for diff --git a/src/iceberg/manifest/manifest_merge_manager.cc b/src/iceberg/manifest/manifest_merge_manager.cc index aedcea735..e29a96846 100644 --- a/src/iceberg/manifest/manifest_merge_manager.cc +++ b/src/iceberg/manifest/manifest_merge_manager.cc @@ -142,8 +142,12 @@ Result> ManifestMergeManager::MergeGroup( } else { ICEBERG_ASSIGN_OR_RAISE( auto merged, FlushBin(bin, snapshot_id, metadata, file_io, writer_factory)); - // Each manifest consumed into the merged output (beyond the 1 output) is replaced. - replaced_manifests_count_ += static_cast(bin.size()) - 1; + if (bin.size() > 1) { + replaced_manifests_count_ += static_cast(std::ranges::count_if( + bin, [snapshot_id](const ManifestFile* manifest) { + return manifest->added_snapshot_id != snapshot_id; + })); + } result.push_back(std::move(merged)); } } diff --git a/src/iceberg/test/manifest_filter_manager_test.cc b/src/iceberg/test/manifest_filter_manager_test.cc index aa1054fec..7dc4ee2d9 100644 --- a/src/iceberg/test/manifest_filter_manager_test.cc +++ b/src/iceberg/test/manifest_filter_manager_test.cc @@ -418,7 +418,7 @@ static Result WriteDeleteManifest(std::shared_ptr delete path); } -TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThan) { +TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanDoesNotRewriteOnItsOwn) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); @@ -439,7 +439,6 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThan) { WriteDeleteManifest(del_file, /*data_seq=*/2L, file_io_, *metadata, manifest_path)); ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); - // Drop delete files older than sequence number 5: entry (seq=2) should be dropped. mgr.DropDeleteFilesOlderThan(5); std::vector manifests{&del_manifest}; @@ -449,17 +448,16 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThan) { ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(schema, specs, manifests, factory)); - ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); - ASSERT_EQ(entries.size(), 1U); - EXPECT_EQ(entries[0].status, ManifestStatus::kDeleted); + ASSERT_EQ(result.size(), 1U); + EXPECT_EQ(result[0].manifest_path, del_manifest.manifest_path); } -TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanKeepsNewerEntries) { +TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanDuringDeleteManifestRewrite) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - // Two entries in the same manifest: old (seq=2, below threshold) and new (seq=10, - // above). + // Three entries in the same manifest: old (seq=2, below threshold), targeted + // (explicit path delete), and keep (survives the rewrite). auto make_del_file = [&](const std::string& path) { auto f = std::make_shared(); f->content = DataFile::Content::kPositionDeletes; @@ -472,16 +470,18 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanKeepsNewerEntries) { return f; }; auto old_file = make_del_file(table_location_ + "/delete/del_old.parquet"); - auto new_file = make_del_file(table_location_ + "/delete/del_new.parquet"); + auto targeted_file = make_del_file(table_location_ + "/delete/del_targeted.parquet"); + auto keep_file = make_del_file(table_location_ + "/delete/del_keep.parquet"); auto manifest_path = std::format("{}/metadata/del-manifest-{}.avro", table_location_, manifest_counter_++); ICEBERG_UNWRAP_OR_FAIL(auto del_manifest, - WriteDeleteManifest({{old_file, 2L}, {new_file, 10L}}, file_io_, - *metadata, manifest_path)); + WriteDeleteManifest( + {{old_file, 2L}, {targeted_file, 10L}, {keep_file, 10L}}, + file_io_, *metadata, manifest_path)); ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); - // Threshold=5: old entry dropped, new entry survives as kExisting. + mgr.DeleteFile(targeted_file->file_path); mgr.DropDeleteFilesOlderThan(5); std::vector manifests{&del_manifest}; @@ -492,22 +492,22 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanKeepsNewerEntries) { mgr.FilterManifests(schema, specs, manifests, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); - ASSERT_EQ(entries.size(), 2U); - // The old entry should be dropped; the new entry should survive. + ASSERT_EQ(entries.size(), 3U); auto deleted = std::count_if( entries.begin(), entries.end(), [](const ManifestEntry& e) { return e.status == ManifestStatus::kDeleted; }); auto existing = std::count_if( entries.begin(), entries.end(), [](const ManifestEntry& e) { return e.status == ManifestStatus::kExisting; }); - EXPECT_EQ(deleted, 1); + EXPECT_EQ(deleted, 2); EXPECT_EQ(existing, 1); - // Verify which entry survived. for (const auto& e : entries) { if (e.status == ManifestStatus::kExisting) { - EXPECT_EQ(e.data_file->file_path, new_file->file_path); + EXPECT_EQ(e.data_file->file_path, keep_file->file_path); } else { - EXPECT_EQ(e.data_file->file_path, old_file->file_path); + EXPECT_THAT(e.data_file->file_path, + ::testing::AnyOf(old_file->file_path, targeted_file->file_path)); + EXPECT_EQ(e.status, ManifestStatus::kDeleted); } } } diff --git a/src/iceberg/test/manifest_merge_manager_test.cc b/src/iceberg/test/manifest_merge_manager_test.cc index b19eace86..b5dd60f9e 100644 --- a/src/iceberg/test/manifest_merge_manager_test.cc +++ b/src/iceberg/test/manifest_merge_manager_test.cc @@ -192,6 +192,34 @@ TEST_F(ManifestMergeManagerTest, MergeOccursAtThreshold) { EXPECT_EQ(count1, 3); } +TEST_F(ManifestMergeManagerTest, ReplacedManifestCountTracksPreviousSnapshotInputs) { + ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); + m0.added_snapshot_id = kSnapshotId - 1; + m1.added_snapshot_id = kSnapshotId - 2; + + ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr.MergeManifests({m0, m1}, {}, kSnapshotId, *metadata_, file_io_, + MakeWriterFactory())); + + EXPECT_EQ(result.size(), 1U); + EXPECT_EQ(mgr.ReplacedManifestsCount(), 2); +} + +TEST_F(ManifestMergeManagerTest, ReplacedManifestCountIgnoresCurrentSnapshotInputs) { + ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); + + ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr.MergeManifests({}, {m0, m1}, kSnapshotId, *metadata_, file_io_, + MakeWriterFactory())); + + EXPECT_EQ(result.size(), 1U); + EXPECT_EQ(mgr.ReplacedManifestsCount(), 0); +} + TEST_F(ManifestMergeManagerTest, OversizedManifestPassedThrough) { // m_large exceeds target → must not be merged; m_small fits ICEBERG_UNWRAP_OR_FAIL(auto m_large, WriteManifest(kSpecId0, 2, /*size=*/2000)); diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 366b09f25..d436b7c53 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -29,6 +29,7 @@ #include "iceberg/avro/avro_register.h" #include "iceberg/constants.h" +#include "iceberg/expression/expressions.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_reader.h" #include "iceberg/manifest/manifest_writer.h" @@ -66,6 +67,9 @@ class TestMergeAppend : public MergingSnapshotUpdate { Status AddDelete(std::shared_ptr file) { return AddDeleteFile(std::move(file)); } + Status AddDelete(std::shared_ptr file, int64_t data_sequence_number) { + return AddDeleteFile(std::move(file), data_sequence_number); + } Status RemoveDataFile(std::shared_ptr file) { return DeleteDataFile(std::move(file)); } @@ -78,7 +82,59 @@ class TestMergeAppend : public MergingSnapshotUpdate { Result> DataSpec() const { return MergingSnapshotUpdate::DataSpec(); } + int64_t GeneratedSnapshotId() { return SnapshotId(); } void SetDataSeqNumber(int64_t seq) { SetNewDataFilesDataSequenceNumber(seq); } + static Status ValidateAddedDataFilesForTest(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const std::shared_ptr& parent, + std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateAddedDataFiles(metadata, starting_snapshot_id, + nullptr, parent, std::move(io)); + } + static Status ValidateAddedDataFilesForTest(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const PartitionSet& partition_set, + const std::shared_ptr& parent, + std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateAddedDataFiles( + metadata, starting_snapshot_id, partition_set, parent, std::move(io)); + } + static Status ValidateNoNewDeletesForDataFilesForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, const DataFileSet& replaced_files, + const std::shared_ptr& parent, std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( + metadata, starting_snapshot_id, std::move(data_filter), replaced_files, parent, + std::move(io)); + } + static Status ValidateNoNewDeleteFilesForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, const std::shared_ptr& parent, + std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateNoNewDeleteFiles( + metadata, starting_snapshot_id, std::move(data_filter), parent, std::move(io)); + } + static Status ValidateNoNewDeleteFilesForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + const PartitionSet& partition_set, const std::shared_ptr& parent, + std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateNoNewDeleteFiles( + metadata, starting_snapshot_id, partition_set, parent, std::move(io)); + } + static Status ValidateDeletedDataFilesForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, const std::shared_ptr& parent, + std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateDeletedDataFiles( + metadata, starting_snapshot_id, std::move(data_filter), parent, std::move(io)); + } + static Status ValidateDeletedDataFilesForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + const PartitionSet& partition_set, const std::shared_ptr& parent, + std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateDeletedDataFiles( + metadata, starting_snapshot_id, partition_set, parent, std::move(io)); + } bool HasDataFiles() const { return AddsDataFiles(); } bool HasDeleteFiles() const { return AddsDeleteFiles(); } @@ -89,6 +145,34 @@ class TestMergeAppend : public MergingSnapshotUpdate { : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {} }; +class TestOverwriteUpdate : public MergingSnapshotUpdate { + public: + static Result> Make( + std::string table_name, std::shared_ptr
table) { + ICEBERG_ASSIGN_OR_RAISE( + auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); + return std::unique_ptr( + new TestOverwriteUpdate(std::move(table_name), std::move(ctx))); + } + + std::string operation() override { return DataOperation::kOverwrite; } + int64_t GeneratedSnapshotId() { return SnapshotId(); } + + Status AddDelete(std::shared_ptr file) { + return AddDeleteFile(std::move(file)); + } + Status AddDelete(std::shared_ptr file, int64_t data_sequence_number) { + return AddDeleteFile(std::move(file), data_sequence_number); + } + Status RemoveDataFile(std::shared_ptr file) { + return DeleteDataFile(std::move(file)); + } + + private: + TestOverwriteUpdate(std::string table_name, std::shared_ptr ctx) + : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {} +}; + class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { protected: static void SetUpTestSuite() { avro::RegisterAll(); } @@ -121,10 +205,22 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return f; } + std::shared_ptr MakeEqualityDeleteFile(const std::string& path, + int64_t partition_x) { + auto f = MakeDeleteFile(path, partition_x); + f->content = DataFile::Content::kEqualityDeletes; + f->equality_ids = {1}; + return f; + } + Result> NewMergeAppend() { return TestMergeAppend::Make(TableName(), table_); } + Result> NewOverwriteUpdate() { + return TestOverwriteUpdate::Make(TableName(), table_); + } + // Commit file_a_ with FastAppend and refresh the table. void CommitFileA() { ICEBERG_UNWRAP_OR_FAIL(auto fa, table_->NewFastAppend()); @@ -168,6 +264,28 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return writer->ToManifestFile(); } + Result> MakeSyntheticSnapshot( + std::string operation, int64_t snapshot_id, + std::optional parent_snapshot_id, int64_t sequence_number, + const std::vector& manifests) { + auto manifest_list_path = + table_location_ + "/metadata/manifest-list-" + std::to_string(snapshot_id) + ".avro"; + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + ManifestListWriter::MakeWriter(table_->metadata()->format_version, snapshot_id, + parent_snapshot_id, manifest_list_path, file_io_, + sequence_number)); + ICEBERG_RETURN_UNEXPECTED(writer->AddAll(manifests)); + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + + ICEBERG_ASSIGN_OR_RAISE( + auto snapshot, + Snapshot::Make(sequence_number, snapshot_id, parent_snapshot_id, TimePointMs{}, + std::move(operation), {}, table_->metadata()->current_schema_id, + manifest_list_path)); + return std::shared_ptr(std::move(snapshot)); + } + std::shared_ptr spec_; std::shared_ptr schema_; std::shared_ptr file_a_; @@ -193,7 +311,7 @@ TEST_F(MergingSnapshotUpdateTest, AddsDataFilesTrueAfterAdd) { } TEST_F(MergingSnapshotUpdateTest, AddsDeleteFilesTrueAfterAdd) { - auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file), IsOk()); EXPECT_FALSE(op->HasDataFiles()); @@ -236,7 +354,7 @@ TEST_F(MergingSnapshotUpdateTest, CommitMultipleDataFiles) { } TEST_F(MergingSnapshotUpdateTest, CommitDataFileAndDeleteFile) { - auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddFile(file_a_), IsOk()); @@ -323,6 +441,25 @@ TEST_F(MergingSnapshotUpdateTest, CommitDeleteFileSummaryHasAddedDeleteFiles) { EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kRemovedDeleteFiles), 0); } +TEST_F(MergingSnapshotUpdateTest, AddDeleteFileWithExplicitSequenceWritesSequenceNumber) { + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file, 17), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); + auto delete_manifest_it = + std::find_if(manifests.begin(), manifests.end(), [](const ManifestFile& manifest) { + return manifest.content == ManifestContent::kDeletes; + }); + ASSERT_NE(delete_manifest_it, manifests.end()); + ICEBERG_UNWRAP_OR_FAIL( + auto entries, ReadAllEntries(std::vector{*delete_manifest_it}, + *table_->metadata())); + ASSERT_EQ(entries.size(), 1U); + ASSERT_TRUE(entries[0].sequence_number.has_value()); + EXPECT_EQ(entries[0].sequence_number.value(), 17); +} + // Covers the bug where deleted delete files were not tracked in the snapshot summary. TEST_F(MergingSnapshotUpdateTest, CommitDeletesDeleteFileSummaryHasRemovedDeleteFiles) { // Step 1: commit a delete file. @@ -434,48 +571,60 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsDeleteManifest) { EXPECT_THAT(op->AppendManifest(del_manifest), IsError(ErrorKind::kInvalidArgument)); } -TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithExistingFiles) { - // Construct a ManifestFile that reports existing files without writing to disk. +TEST_F(MergingSnapshotUpdateTest, AddManifestAllowsManifestWithExistingFilesCount) { ManifestFile manifest; manifest.manifest_path = table_location_ + "/metadata/existing.avro"; manifest.content = ManifestContent::kData; manifest.added_snapshot_id = kInvalidSnapshotId; - manifest.existing_files_count = 1; // has_existing_files() returns true + manifest.existing_files_count = 1; ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); - EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(op->AppendManifest(manifest), IsOk()); } -TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithDeletedFiles) { +TEST_F(MergingSnapshotUpdateTest, AddManifestAllowsManifestWithDeletedFilesCount) { ManifestFile manifest; manifest.manifest_path = table_location_ + "/metadata/deleted.avro"; manifest.content = ManifestContent::kData; manifest.added_snapshot_id = kInvalidSnapshotId; - manifest.deleted_files_count = 1; // has_deleted_files() returns true + manifest.deleted_files_count = 1; ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); - EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(op->AppendManifest(manifest), IsOk()); } -TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithAssignedSnapshotId) { - ManifestFile manifest; - manifest.manifest_path = table_location_ + "/metadata/snap.avro"; - manifest.content = ManifestContent::kData; - manifest.added_snapshot_id = 12345; // already assigned +TEST_F(MergingSnapshotUpdateTest, AddManifestCopiesManifestWithAssignedSnapshotId) { + auto path = table_location_ + "/metadata/snap.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_})); + manifest.added_snapshot_id = 12345; ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); - EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(op->AppendManifest(manifest), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + ASSERT_EQ(data_manifests.size(), 1U); + EXPECT_NE(data_manifests[0].manifest_path, path); } -TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithFirstRowId) { - ManifestFile manifest; - manifest.manifest_path = table_location_ + "/metadata/rowid.avro"; - manifest.content = ManifestContent::kData; - manifest.added_snapshot_id = kInvalidSnapshotId; - manifest.first_row_id = 0; // assigned first_row_id +TEST_F(MergingSnapshotUpdateTest, AddManifestCopiesManifestWithFirstRowId) { + auto path = table_location_ + "/metadata/rowid.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_})); + manifest.first_row_id = 0; ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); - EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(op->AppendManifest(manifest), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, + SnapshotCache(snapshot.get()).DataManifests(file_io_)); + ASSERT_EQ(data_manifests.size(), 1U); + EXPECT_NE(data_manifests[0].manifest_path, path); } // ------------------------------------------------------------------------- @@ -715,6 +864,204 @@ TEST_F(MergingSnapshotUpdateTest, SummaryManifestCountsAfterDelete) { EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsKept), "0"); } +TEST_F(MergingSnapshotUpdateTest, MissingRequestedDeleteDoesNotAffectSummary) { + CommitFileA(); + + auto missing = MakeDataFile("/data/missing.parquet", 3L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->RemoveDataFile(missing), IsOk()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kDeletedDataFiles), 0U); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "2"); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesFailsForTruncatedHistory) { + auto metadata = std::make_shared(); + metadata->format_version = 2; + metadata->location = table_location_; + metadata->current_schema_id = 0; + metadata->schemas.push_back(schema_); + + auto make_snapshot = [](int64_t snapshot_id, std::optional parent_snapshot_id) { + return std::make_shared(Snapshot{ + .snapshot_id = snapshot_id, + .parent_snapshot_id = parent_snapshot_id, + .sequence_number = snapshot_id, + .timestamp_ms = TimePointMs{}, + .manifest_list = "", + .summary = {}, + .schema_id = 0, + }); + }; + + auto base_snapshot = make_snapshot(1, std::nullopt); + auto main_snapshot = make_snapshot(2, 1); + auto branch_snapshot = make_snapshot(3, 1); + metadata->snapshots = {base_snapshot, main_snapshot, branch_snapshot}; + + EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest(*metadata, /*starting=*/2, + branch_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithPartitionSetDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + ICEBERG_UNWRAP_OR_FAIL(auto fast_append, table_->NewFastAppend()); + fast_append->AppendFile(file_b_); + EXPECT_THAT(fast_append->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto second_snapshot, table_->current_snapshot()); + + PartitionSet partition_set; + ASSERT_TRUE(partition_set.add(spec_->spec_id(), file_b_->partition)); + EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest( + *table_->metadata(), first_snapshot->snapshot_id, partition_set, + second_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesWithFilterDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + DataFileSet replaced_files; + replaced_files.insert(file_a_); + EXPECT_THAT(TestMergeAppend::ValidateNoNewDeletesForDataFilesForTest( + *metadata, first_snapshot->snapshot_id, + Expressions::AlwaysTrue(), replaced_files, second_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithExpressionDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + EXPECT_THAT(TestMergeAppend::ValidateNoNewDeleteFilesForTest( + *metadata, first_snapshot->snapshot_id, + Expressions::AlwaysTrue(), second_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithPartitionSetDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + PartitionSet partition_set; + ASSERT_TRUE(partition_set.add(spec_->spec_id(), del_file->partition)); + EXPECT_THAT(TestMergeAppend::ValidateNoNewDeleteFilesForTest( + *metadata, first_snapshot->snapshot_id, partition_set, + second_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + EXPECT_THAT(TestMergeAppend::ValidateDeletedDataFilesForTest( + *metadata, first_snapshot->snapshot_id, + Expressions::AlwaysTrue(), second_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithPartitionSetDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + PartitionSet partition_set; + ASSERT_TRUE(partition_set.add(spec_->spec_id(), file_a_->partition)); + EXPECT_THAT(TestMergeAppend::ValidateDeletedDataFilesForTest( + *metadata, first_snapshot->snapshot_id, partition_set, + second_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + // ------------------------------------------------------------------------- // DataSpec — multiple partition specs // ------------------------------------------------------------------------- diff --git a/src/iceberg/test/snapshot_util_test.cc b/src/iceberg/test/snapshot_util_test.cc index a47b403da..83ef09f4b 100644 --- a/src/iceberg/test/snapshot_util_test.cc +++ b/src/iceberg/test/snapshot_util_test.cc @@ -301,10 +301,20 @@ TEST_F(SnapshotUtilTest, SchemaForBranch) { ICEBERG_UNWRAP_OR_FAIL(auto initial_schema, table_->schema()); ASSERT_NE(initial_schema, nullptr); + auto branch_schema = std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32()), + SchemaField::MakeRequired(2, "data", string()), + SchemaField::MakeOptional(3, "branch_only", string())}, + 1); + table_->metadata()->schemas.push_back(branch_schema); + ICEBERG_UNWRAP_OR_FAIL(auto branch_snapshot, table_->SnapshotById(branch_snapshot_id_)); + branch_snapshot->schema_id = branch_schema->schema_id(); + std::string branch = "b1"; ICEBERG_UNWRAP_OR_FAIL(auto schema, SnapshotUtil::SchemaFor(*table_, branch)); - // Branch should return current schema (not snapshot schema) - EXPECT_EQ(schema->fields().size(), initial_schema->fields().size()); + EXPECT_EQ(schema->schema_id(), branch_schema->schema_id()); + EXPECT_EQ(schema->fields().size(), branch_schema->fields().size()); + EXPECT_NE(schema->fields().size(), initial_schema->fields().size()); } TEST_F(SnapshotUtilTest, SchemaForTag) { diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index f2d0c93e0..738dc2dcd 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -45,6 +45,110 @@ namespace iceberg { +namespace { + +bool MatchesOperation(std::optional operation, + std::initializer_list expected) { + return operation.has_value() && + std::find(expected.begin(), expected.end(), operation.value()) != + expected.end(); +} + +struct ValidationHistoryResult { + std::vector manifests; + std::unordered_set snapshot_ids; +}; + +Result>> ValidationAncestorsBetween( + const TableMetadata& metadata, int64_t latest_snapshot_id, + int64_t starting_snapshot_id) { + ICEBERG_ASSIGN_OR_RAISE( + auto ancestors, + SnapshotUtil::AncestorsBetween(metadata, latest_snapshot_id, starting_snapshot_id)); + if (latest_snapshot_id == starting_snapshot_id) { + return ancestors; + } + if (ancestors.empty()) { + return InvalidArgument("Cannot validate history: starting snapshot {} is not an ancestor " + "of snapshot {}", + starting_snapshot_id, latest_snapshot_id); + } + + const auto& oldest_checked = ancestors.back(); + if (oldest_checked == nullptr || !oldest_checked->parent_snapshot_id.has_value() || + oldest_checked->parent_snapshot_id.value() != starting_snapshot_id) { + return InvalidArgument("Cannot validate history: starting snapshot {} is not an ancestor " + "of snapshot {}", + starting_snapshot_id, latest_snapshot_id); + } + return ancestors; +} + +Result ValidationHistory( + const TableMetadata& metadata, int64_t latest_snapshot_id, + int64_t starting_snapshot_id, + std::initializer_list matching_operations, + ManifestContent content, const std::shared_ptr& io) { + ICEBERG_ASSIGN_OR_RAISE( + auto ancestors, + ValidationAncestorsBetween(metadata, latest_snapshot_id, starting_snapshot_id)); + + ValidationHistoryResult result; + for (const auto& snapshot : ancestors) { + if (!MatchesOperation(snapshot->Operation(), matching_operations)) { + continue; + } + + result.snapshot_ids.insert(snapshot->snapshot_id); + auto cached = SnapshotCache(snapshot.get()); + ICEBERG_ASSIGN_OR_RAISE( + auto manifests, content == ManifestContent::kData ? cached.DataManifests(io) + : cached.DeleteManifests(io)); + for (const auto& manifest : manifests) { + if (manifest.added_snapshot_id == snapshot->snapshot_id) { + result.manifests.push_back(manifest); + } + } + } + + return result; +} + +Result> FindMatchingDataFile( + const TableMetadata& metadata, const std::vector& manifests, + ManifestStatus status, std::shared_ptr filter, + const PartitionSet* partition_set, const std::shared_ptr& io, + bool case_sensitive) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + auto partition_filter = partition_set != nullptr + ? std::make_shared(*partition_set) + : std::shared_ptr{}; + + for (const auto& manifest : manifests) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, + metadata.PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto reader, ManifestReader::Make(manifest, io, schema, spec)); + reader->CaseSensitive(case_sensitive); + if (filter != nullptr) { + reader->FilterRows(filter); + } + if (partition_filter != nullptr) { + reader->FilterPartitions(partition_filter); + } + + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); + for (const auto& entry : entries) { + if (entry.status == status && entry.data_file != nullptr) { + return entry.data_file->file_path; + } + } + } + + return std::optional{}; +} + +} // namespace + MergingSnapshotUpdate::MergingSnapshotUpdate(std::string table_name, std::shared_ptr ctx) : SnapshotUpdate(std::move(ctx)), @@ -122,6 +226,16 @@ Status MergingSnapshotUpdate::ValidateNewDeleteFile(const DataFile& file) { } Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file) { + return AddDeleteFile(std::move(file), std::nullopt); +} + +Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, + int64_t data_sequence_number) { + return AddDeleteFile(std::move(file), std::optional(data_sequence_number)); +} + +Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, + std::optional data_sequence_number) { if (!file) { return InvalidArgument("Cannot add a null delete file"); } @@ -133,7 +247,9 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file) { base().PartitionSpecById(file->partition_spec_id.value())); ICEBERG_RETURN_UNEXPECTED(added_delete_files_summary_.AddedFile(*spec, *file)); has_new_delete_files_ = true; - new_delete_files_.push_back(std::move(file)); + new_delete_files_.push_back( + PendingDeleteFile{.file = std::move(file), + .data_sequence_number = std::move(data_sequence_number)}); return {}; } @@ -215,34 +331,12 @@ std::vector> MergingSnapshotUpdate::AddedDataFiles() c return result; } -Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr /*file*/, - int64_t /*data_sequence_number*/) { - return NotImplemented( - "AddDeleteFile with explicit data sequence number is not yet implemented"); -} - Status MergingSnapshotUpdate::AddManifest(ManifestFile manifest) { if (manifest.content != ManifestContent::kData) { return InvalidArgument("Cannot append delete manifest: {}", manifest.manifest_path); } - if (manifest.has_existing_files()) { - return InvalidArgument("Cannot append manifest with existing files: {}", - manifest.manifest_path); - } - if (manifest.has_deleted_files()) { - return InvalidArgument("Cannot append manifest with deleted files: {}", - manifest.manifest_path); - } - if (manifest.added_snapshot_id != kInvalidSnapshotId) { - return InvalidArgument("Snapshot id must be assigned during commit: {}", - manifest.manifest_path); - } - if (manifest.first_row_id.has_value()) { - return InvalidArgument("Cannot append manifest with assigned first_row_id: {}", - manifest.manifest_path); - } - - if (can_inherit_snapshot_id()) { + if (can_inherit_snapshot_id() && manifest.added_snapshot_id == kInvalidSnapshotId && + !manifest.first_row_id.has_value()) { appended_manifests_summary_.AddedManifest(manifest); append_manifests_.push_back(std::move(manifest)); } else { @@ -254,7 +348,7 @@ Status MergingSnapshotUpdate::AddManifest(ManifestFile manifest) { Result MergingSnapshotUpdate::CopyManifest(const ManifestFile& manifest) { const TableMetadata& current = base(); - ICEBERG_ASSIGN_OR_RAISE(auto schema, current.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto schema, SnapshotUtil::SchemaFor(current, target_branch())); ICEBERG_ASSIGN_OR_RAISE(auto spec, current.PartitionSpecById(manifest.partition_spec_id)); std::string path = ManifestPath(); @@ -285,17 +379,16 @@ bool MergingSnapshotUpdate::DeletesDeleteFiles() const { // Apply pipeline // ------------------------------------------------------------------------- -ManifestWriterFactory MergingSnapshotUpdate::MakeTrackedWriterFactory() { - return [this](int32_t spec_id, +ManifestWriterFactory MergingSnapshotUpdate::MakeTrackedWriterFactory( + const std::shared_ptr& schema) { + return [this, schema](int32_t spec_id, ManifestContent content) -> Result> { const TableMetadata& meta = base(); - ICEBERG_ASSIGN_OR_RAISE(auto schema, meta.Schema()); ICEBERG_ASSIGN_OR_RAISE(auto spec, meta.PartitionSpecById(spec_id)); std::string path = ManifestPath(); all_written_manifests_.insert(path); return ManifestWriter::MakeWriter(meta.format_version, SnapshotId(), std::move(path), - ctx_->table->io(), std::move(spec), - std::move(schema), content); + ctx_->table->io(), std::move(spec), schema, content); }; } @@ -344,17 +437,25 @@ Result> MergingSnapshotUpdate::WriteNewDeleteManifests } // Group delete files by partition spec ID, mirroring WriteNewDataManifests(). - std::unordered_map>> - delete_files_by_spec; - for (const auto& file : new_delete_files_) { - delete_files_by_spec[file->partition_spec_id.value()].push_back(file); + std::unordered_map> delete_files_by_spec; + for (const auto& pending_file : new_delete_files_) { + delete_files_by_spec[pending_file.file->partition_spec_id.value()].push_back( + pending_file); } std::vector result; - for (const auto& [spec_id, delete_files] : delete_files_by_spec) { + for (auto& [spec_id, delete_files] : delete_files_by_spec) { ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(spec_id)); + std::vector delete_entries; + delete_entries.reserve(delete_files.size()); + for (const auto& pending_file : delete_files) { + delete_entries.push_back(DeleteManifestEntry{ + .file = pending_file.file, + .data_sequence_number = pending_file.data_sequence_number, + }); + } ICEBERG_ASSIGN_OR_RAISE(auto written, - WriteDeleteManifests(std::span(delete_files), spec)); + WriteDeleteManifests(delete_entries, spec)); for (const auto& m : written) { all_written_manifests_.insert(m.manifest_path); } @@ -371,8 +472,8 @@ Result> MergingSnapshotUpdate::Apply( const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) { // Re-validate buffered delete files against the current format version. A format // upgrade between staging and commit could make previously-valid files invalid. - for (const auto& file : new_delete_files_) { - ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(*file)); + for (const auto& pending_file : new_delete_files_) { + ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(*pending_file.file)); } // Rebuild summary from stable sub-builders so that commit retries don't double-count. @@ -381,12 +482,14 @@ Result> MergingSnapshotUpdate::Apply( summary_builder().Merge(added_delete_files_summary_); summary_builder().Merge(appended_manifests_summary_); - auto tracked_factory = MakeTrackedWriterFactory(); + ICEBERG_ASSIGN_OR_RAISE(auto target_schema, + SnapshotUtil::SchemaFor(metadata_to_update, target_branch())); + auto tracked_factory = MakeTrackedWriterFactory(target_schema); // Step 1: Filter data manifests. ICEBERG_ASSIGN_OR_RAISE(auto filtered_data, data_filter_manager_.FilterManifests( - metadata_to_update, snapshot, tracked_factory)); + target_schema, metadata_to_update, snapshot, tracked_factory)); // Track deleted data files in the summary builder. for (const auto& file : data_filter_manager_.FilesToBeDeleted()) { @@ -419,7 +522,7 @@ Result> MergingSnapshotUpdate::Apply( // Step 3: Filter delete manifests. ICEBERG_ASSIGN_OR_RAISE(auto filtered_deletes, delete_filter_manager_.FilterManifests( - metadata_to_update, snapshot, tracked_factory)); + target_schema, metadata_to_update, snapshot, tracked_factory)); // Track deleted delete files in the summary builder. for (const auto& file : delete_filter_manager_.FilesToBeDeleted()) { @@ -552,65 +655,19 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( return {}; } - ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); - ICEBERG_ASSIGN_OR_RAISE(auto ancestors, - SnapshotUtil::AncestorsBetween(metadata, parent->snapshot_id, - starting_snapshot_id)); - - // Build the full set of matching snapshot IDs first, then scan their manifests. - // The full set must be known before filtering manifests, since a manifest may have - // been written by a different snapshot in the ancestry range. - std::unordered_set matching_snapshot_ids; - for (const auto& snap : ancestors) { - auto op = snap->Operation(); - if (op == DataOperation::kAppend || op == DataOperation::kOverwrite) { - matching_snapshot_ids.insert(snap->snapshot_id); - } - } - - std::unique_ptr evaluator; - if (filter != nullptr) { - ICEBERG_ASSIGN_OR_RAISE( - evaluator, InclusiveMetricsEvaluator::Make(filter, *schema, case_sensitive)); - } - - for (const auto& snapshot : ancestors) { - if (!matching_snapshot_ids.contains(snapshot->snapshot_id)) { - continue; - } - auto cached = SnapshotCache(snapshot.get()); - ICEBERG_ASSIGN_OR_RAISE(auto data_manifests, cached.DataManifests(io)); - - for (const auto& manifest : data_manifests) { - if (!matching_snapshot_ids.contains(manifest.added_snapshot_id)) { - continue; - } - ICEBERG_ASSIGN_OR_RAISE(auto spec, - metadata.PartitionSpecById(manifest.partition_spec_id)); - ICEBERG_ASSIGN_OR_RAISE(auto reader, - ManifestReader::Make(manifest, io, schema, spec)); - ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); - - for (const auto& entry : entries) { - if (entry.status != ManifestStatus::kAdded) { - continue; - } - if (entry.data_file == nullptr) { - continue; - } - if (evaluator != nullptr) { - ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*entry.data_file)); - if (!matches) { - continue; - } - } - return InvalidArgument( - "Found conflicting files that can contain rows matching {}:" - " {} in snapshot {}", - filter != nullptr ? filter->ToString() : "any expression", - entry.data_file->file_path, snapshot->snapshot_id); - } - } + ICEBERG_ASSIGN_OR_RAISE( + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kAppend, DataOperation::kOverwrite}, + ManifestContent::kData, io)); + ICEBERG_ASSIGN_OR_RAISE(auto conflict_path, + FindMatchingDataFile(metadata, history.manifests, + ManifestStatus::kAdded, filter, nullptr, io, + case_sensitive)); + if (conflict_path.has_value()) { + return InvalidArgument("Found conflicting files that can contain rows matching {}: {}", + filter != nullptr ? filter->ToString() : "any expression", + conflict_path.value()); } return {}; } @@ -626,8 +683,8 @@ Status MergingSnapshotUpdate::ValidateDataFilesExist( ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); ICEBERG_ASSIGN_OR_RAISE(auto ancestors, - SnapshotUtil::AncestorsBetween(metadata, parent->snapshot_id, - starting_snapshot_id)); + ValidationAncestorsBetween(metadata, parent->snapshot_id, + starting_snapshot_id)); // Build the full set of matching snapshot IDs first, then scan their manifests. // The full set must be known before filtering manifests, since a manifest may have @@ -742,51 +799,177 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( } Status MergingSnapshotUpdate::ValidateAddedDataFiles( - const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, - const PartitionSet& /*partition_set*/, const std::shared_ptr& /*parent*/, - std::shared_ptr /*io*/) { - return NotImplemented( - "ValidateAddedDataFiles with PartitionSet is not yet implemented"); + const TableMetadata& metadata, int64_t starting_snapshot_id, + const PartitionSet& partition_set, const std::shared_ptr& parent, + std::shared_ptr io) { + if (parent == nullptr) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kAppend, DataOperation::kOverwrite}, + ManifestContent::kData, io)); + ICEBERG_ASSIGN_OR_RAISE( + auto conflict_path, + FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kAdded, nullptr, + &partition_set, io, /*case_sensitive=*/true)); + if (conflict_path.has_value()) { + return InvalidArgument( + "Found conflicting files that can contain rows in validated partitions: {}", + conflict_path.value()); + } + return {}; } Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( - const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, - std::shared_ptr /*data_filter*/, const DataFileSet& /*replaced_files*/, - const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { - return NotImplemented( - "ValidateNoNewDeletesForDataFiles with data filter is not yet implemented"); + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, const DataFileSet& replaced_files, + const std::shared_ptr& parent, std::shared_ptr io) { + if (parent == nullptr || replaced_files.empty() || metadata.format_version < 2) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, nullptr, parent, io)); + if (deletes->empty()) { + return {}; + } + + int64_t starting_seq = TableMetadata::kInitialSequenceNumber; + if (auto snap_result = metadata.SnapshotById(starting_snapshot_id); + snap_result.has_value()) { + starting_seq = snap_result.value()->sequence_number; + } + + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + std::unique_ptr evaluator; + if (data_filter != nullptr) { + ICEBERG_ASSIGN_OR_RAISE( + evaluator, InclusiveMetricsEvaluator::Make(data_filter, *schema, + /*case_sensitive=*/true)); + } + + for (const auto& data_file : replaced_files) { + ICEBERG_ASSIGN_OR_RAISE(auto delete_files, + deletes->ForDataFile(starting_seq, *data_file)); + for (const auto& delete_file : delete_files) { + if (evaluator != nullptr) { + ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*delete_file)); + if (!matches) { + continue; + } + } + return InvalidArgument("Cannot commit, found new delete for replaced data file: {}", + data_file->file_path); + } + } + return {}; } Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( - const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, - std::shared_ptr /*data_filter*/, - const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { - return NotImplemented( - "ValidateNoNewDeleteFiles with Expression is not yet implemented"); + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const std::shared_ptr& parent, std::shared_ptr io) { + ICEBERG_ASSIGN_OR_RAISE( + auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, nullptr, parent, io)); + auto referenced_delete_files = deletes->ReferencedDeleteFiles(); + + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + std::unique_ptr evaluator; + if (data_filter != nullptr) { + ICEBERG_ASSIGN_OR_RAISE( + evaluator, InclusiveMetricsEvaluator::Make(data_filter, *schema, + /*case_sensitive=*/true)); + } + + for (const auto& delete_file : referenced_delete_files) { + if (evaluator != nullptr) { + ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*delete_file)); + if (!matches) { + continue; + } + } + return InvalidArgument("Found new conflicting delete files: {}", + delete_file->file_path); + } + return {}; } Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( - const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, - const PartitionSet& /*partition_set*/, const std::shared_ptr& /*parent*/, - std::shared_ptr /*io*/) { - return NotImplemented( - "ValidateNoNewDeleteFiles with PartitionSet is not yet implemented"); + const TableMetadata& metadata, int64_t starting_snapshot_id, + const PartitionSet& partition_set, const std::shared_ptr& parent, + std::shared_ptr io) { + ICEBERG_ASSIGN_OR_RAISE( + auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, nullptr, parent, io)); + auto referenced_delete_files = deletes->ReferencedDeleteFiles(); + for (const auto& delete_file : referenced_delete_files) { + if (!delete_file->partition_spec_id.has_value() || + !partition_set.contains(delete_file->partition_spec_id.value(), + delete_file->partition)) { + continue; + } + return InvalidArgument("Found new conflicting delete files in validated partitions: {}", + delete_file->file_path); + } + return {}; } Status MergingSnapshotUpdate::ValidateDeletedDataFiles( - const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, - std::shared_ptr /*data_filter*/, - const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { - return NotImplemented( - "ValidateDeletedDataFiles with Expression is not yet implemented"); + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const std::shared_ptr& parent, std::shared_ptr io) { + if (parent == nullptr) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kOverwrite, DataOperation::kReplace, + DataOperation::kDelete}, + ManifestContent::kData, io)); + ICEBERG_ASSIGN_OR_RAISE(auto conflict_path, + FindMatchingDataFile(metadata, history.manifests, + ManifestStatus::kDeleted, data_filter, + nullptr, io, /*case_sensitive=*/true)); + if (conflict_path.has_value()) { + return InvalidArgument( + "Found conflicting deleted files that can contain rows matching {}: {}", + data_filter != nullptr ? data_filter->ToString() : "any expression", + conflict_path.value()); + } + return {}; } Status MergingSnapshotUpdate::ValidateDeletedDataFiles( - const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, - const PartitionSet& /*partition_set*/, const std::shared_ptr& /*parent*/, - std::shared_ptr /*io*/) { - return NotImplemented( - "ValidateDeletedDataFiles with PartitionSet is not yet implemented"); + const TableMetadata& metadata, int64_t starting_snapshot_id, + const PartitionSet& partition_set, const std::shared_ptr& parent, + std::shared_ptr io) { + if (parent == nullptr) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kOverwrite, DataOperation::kReplace, + DataOperation::kDelete}, + ManifestContent::kData, io)); + ICEBERG_ASSIGN_OR_RAISE( + auto conflict_path, + FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kDeleted, nullptr, + &partition_set, io, /*case_sensitive=*/true)); + if (conflict_path.has_value()) { + return InvalidArgument( + "Found conflicting deleted files in validated partitions: {}", + conflict_path.value()); + } + return {}; } Result> MergingSnapshotUpdate::AddedDeleteFiles( @@ -806,32 +989,11 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles return builder.Build(); } - ICEBERG_ASSIGN_OR_RAISE(auto ancestors, - SnapshotUtil::AncestorsBetween(metadata, parent->snapshot_id, - starting_snapshot_id)); - - // Collect delete manifests from OVERWRITE and DELETE snapshots only. - std::unordered_set matching_snapshot_ids; - for (const auto& snap : ancestors) { - auto op = snap->Operation(); - if (op == DataOperation::kOverwrite || op == DataOperation::kDelete) { - matching_snapshot_ids.insert(snap->snapshot_id); - } - } - - std::vector delete_manifests; - for (const auto& snapshot : ancestors) { - if (!matching_snapshot_ids.contains(snapshot->snapshot_id)) { - continue; - } - auto cached = SnapshotCache(snapshot.get()); - ICEBERG_ASSIGN_OR_RAISE(auto manifests, cached.DeleteManifests(io)); - for (const auto& m : manifests) { - if (matching_snapshot_ids.contains(m.added_snapshot_id)) { - delete_manifests.push_back(m); - } - } - } + ICEBERG_ASSIGN_OR_RAISE( + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kOverwrite, DataOperation::kDelete}, + ManifestContent::kDeletes, io)); // Compute the starting sequence number from the starting snapshot. int64_t starting_seq = TableMetadata::kInitialSequenceNumber; @@ -847,7 +1009,7 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles ICEBERG_ASSIGN_OR_RAISE(auto builder, DeleteFileIndex::BuilderFor(io, schema, std::move(specs_by_id), - std::move(delete_manifests))); + std::move(history.manifests))); builder.AfterSequenceNumber(starting_seq); builder.CaseSensitive(case_sensitive); if (data_filter != nullptr) { diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h index a4030bb4d..54e259c23 100644 --- a/src/iceberg/update/merging_snapshot_update.h +++ b/src/iceberg/update/merging_snapshot_update.h @@ -92,15 +92,13 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Stage a delete file with an explicit data sequence number. /// - /// \note Not yet implemented; returns NotImplemented error. Status AddDeleteFile(std::shared_ptr file, int64_t data_sequence_number); /// \brief Add all files in a pre-existing data manifest to the new snapshot. /// - /// The manifest must contain only DATA content and only ADDED entries (no - /// existing or deleted files). If snapshot ID inheritance is enabled and the - /// manifest has no snapshot ID assigned, it is used directly; otherwise it is - /// copied with the current snapshot ID. + /// The manifest must contain DATA content. If snapshot ID inheritance is + /// enabled and the manifest has no snapshot ID assigned, it is used directly; + /// otherwise it is copied with the current snapshot ID. Status AddManifest(ManifestFile manifest); /// \brief Register a data file (by object) to be deleted from the table. @@ -177,7 +175,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// added a data file in any partition of the given partition set. /// - /// \note Not yet implemented; returns NotImplemented error. static Status ValidateAddedDataFiles(const TableMetadata& metadata, int64_t starting_snapshot_id, const PartitionSet& partition_set, @@ -228,7 +225,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// added a delete file matching the data filter that covers a file in replaced_files. /// - /// \note Not yet implemented; returns NotImplemented error. static Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, int64_t starting_snapshot_id, std::shared_ptr data_filter, @@ -239,7 +235,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// added a delete file matching the given row filter. /// - /// \note Not yet implemented; returns NotImplemented error. static Status ValidateNoNewDeleteFiles(const TableMetadata& metadata, int64_t starting_snapshot_id, std::shared_ptr data_filter, @@ -249,7 +244,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// added a delete file matching any partition in the given partition set. /// - /// \note Not yet implemented; returns NotImplemented error. static Status ValidateNoNewDeleteFiles(const TableMetadata& metadata, int64_t starting_snapshot_id, const PartitionSet& partition_set, @@ -259,7 +253,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// deleted a data file matching the given row filter. /// - /// \note Not yet implemented; returns NotImplemented error. static Status ValidateDeletedDataFiles(const TableMetadata& metadata, int64_t starting_snapshot_id, std::shared_ptr data_filter, @@ -269,7 +262,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// deleted a data file in any partition of the given partition set. /// - /// \note Not yet implemented; returns NotImplemented error. static Status ValidateDeletedDataFiles(const TableMetadata& metadata, int64_t starting_snapshot_id, const PartitionSet& partition_set, @@ -295,14 +287,22 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { std::shared_ptr io); private: + struct PendingDeleteFile { + std::shared_ptr file; + std::optional data_sequence_number; + }; + /// \brief Create a ManifestWriterFactory that records every path it creates in /// all_written_manifests_. - ManifestWriterFactory MakeTrackedWriterFactory(); + ManifestWriterFactory MakeTrackedWriterFactory(const std::shared_ptr& schema); /// \brief Copy a manifest with the current snapshot ID, for use when snapshot /// ID inheritance is not possible. Result CopyManifest(const ManifestFile& manifest); + Status AddDeleteFile(std::shared_ptr file, + std::optional data_sequence_number); + /// \brief Write new data manifests for staged data files; caches the result. Result> WriteNewDataManifests(); @@ -326,7 +326,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { ManifestMergeManager delete_merge_manager_; std::unordered_map new_data_files_by_spec_; - std::vector> new_delete_files_; + std::vector new_delete_files_; std::optional new_data_files_data_seq_number_; // Manifests passed via AddManifest(): inherit path (no copy needed) and diff --git a/src/iceberg/update/snapshot_update.cc b/src/iceberg/update/snapshot_update.cc index fc5359e42..bff12c912 100644 --- a/src/iceberg/update/snapshot_update.cc +++ b/src/iceberg/update/snapshot_update.cc @@ -194,6 +194,19 @@ Result> SnapshotUpdate::WriteDataManifests( Result> SnapshotUpdate::WriteDeleteManifests( std::span> files, const std::shared_ptr& spec) { + std::vector delete_entries; + delete_entries.reserve(files.size()); + for (const auto& file : files) { + delete_entries.push_back( + DeleteManifestEntry{.file = file, .data_sequence_number = std::nullopt}); + } + return WriteDeleteManifests(delete_entries, spec); +} + +// TODO(xxx): write manifests in parallel +Result> SnapshotUpdate::WriteDeleteManifests( + std::span files, + const std::shared_ptr& spec) { if (files.empty()) { return std::vector{}; } @@ -208,10 +221,9 @@ Result> SnapshotUpdate::WriteDeleteManifests( }, target_manifest_size_bytes_); - for (const auto& file : files) { - // FIXME: Java impl wrap it with `PendingDeleteFile` and deals with - // file->data_sequence_number - ICEBERG_RETURN_UNEXPECTED(rolling_writer.WriteAddedEntry(file)); + for (const auto& entry : files) { + ICEBERG_RETURN_UNEXPECTED( + rolling_writer.WriteAddedEntry(entry.file, entry.data_sequence_number)); } ICEBERG_RETURN_UNEXPECTED(rolling_writer.Close()); return rolling_writer.ToManifestFiles(); diff --git a/src/iceberg/update/snapshot_update.h b/src/iceberg/update/snapshot_update.h index f48e5f44d..42df70d61 100644 --- a/src/iceberg/update/snapshot_update.h +++ b/src/iceberg/update/snapshot_update.h @@ -122,6 +122,11 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { Status Finalize(Result commit_result) override; protected: + struct DeleteManifestEntry { + std::shared_ptr file; + std::optional data_sequence_number; + }; + explicit SnapshotUpdate(std::shared_ptr ctx); /// \brief Write data manifests for the given data files @@ -144,6 +149,10 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { std::span> files, const std::shared_ptr& spec); + Result> WriteDeleteManifests( + std::span files, + const std::shared_ptr& spec); + const std::string& target_branch() const { return target_branch_; } bool can_inherit_snapshot_id() const { return can_inherit_snapshot_id_; } const std::string& commit_uuid() const { return commit_uuid_; } diff --git a/src/iceberg/util/snapshot_util.cc b/src/iceberg/util/snapshot_util.cc index 49019408b..1bbd6ae8c 100644 --- a/src/iceberg/util/snapshot_util.cc +++ b/src/iceberg/util/snapshot_util.cc @@ -375,7 +375,7 @@ Result> SnapshotUtil::SchemaFor(const Table& table, const auto& metadata = table.metadata(); auto it = metadata->refs.find(ref); - if (it == metadata->refs.cend() || it->second->type() == SnapshotRefType::kBranch) { + if (it == metadata->refs.cend()) { return table.schema(); } @@ -389,7 +389,7 @@ Result> SnapshotUtil::SchemaFor(const TableMetadata& met } auto it = metadata.refs.find(ref); - if (it == metadata.refs.end() || it->second->type() == SnapshotRefType::kBranch) { + if (it == metadata.refs.end()) { return metadata.Schema(); } diff --git a/src/iceberg/util/snapshot_util_internal.h b/src/iceberg/util/snapshot_util_internal.h index 8a3158185..66a99a3b4 100644 --- a/src/iceberg/util/snapshot_util_internal.h +++ b/src/iceberg/util/snapshot_util_internal.h @@ -306,9 +306,9 @@ class ICEBERG_EXPORT SnapshotUtil { /// \brief Return the schema of the snapshot at a given ref. /// - /// If the ref does not exist or the ref is a branch, the table schema is returned - /// because it will be the schema when the new branch is created. If the ref is a tag, - /// then the snapshot schema is returned. + /// If the ref does not exist, the current table schema is returned. If the ref exists + /// and points to a snapshot (branch or tag), the schema recorded for that snapshot is + /// returned. /// /// \param table The table /// \param ref Ref name of the table (empty string means main branch) @@ -318,9 +318,9 @@ class ICEBERG_EXPORT SnapshotUtil { /// \brief Return the schema of the snapshot at a given ref. /// - /// If the ref does not exist or the ref is a branch, the table schema is returned - /// because it will be the schema when the new branch is created. If the ref is a tag, - /// then the snapshot schema is returned. + /// If the ref does not exist, the current table schema is returned. If the ref exists + /// and points to a snapshot (branch or tag), the schema recorded for that snapshot is + /// returned. /// /// \param metadata The table metadata /// \param ref Ref name of the table (empty string means main branch) From e3621d428fc1190c35c6e0715b3fa4383693fc95 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Wed, 27 May 2026 15:04:39 +0800 Subject: [PATCH 03/16] Apply clang-format fixes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../manifest/manifest_merge_manager.cc | 4 +- .../test/manifest_filter_manager_test.cc | 8 +- .../test/manifest_merge_manager_test.cc | 12 +- .../test/merging_snapshot_update_test.cc | 122 ++++++------ src/iceberg/update/merging_snapshot_update.cc | 182 +++++++++--------- 5 files changed, 165 insertions(+), 163 deletions(-) diff --git a/src/iceberg/manifest/manifest_merge_manager.cc b/src/iceberg/manifest/manifest_merge_manager.cc index e29a96846..b924450cf 100644 --- a/src/iceberg/manifest/manifest_merge_manager.cc +++ b/src/iceberg/manifest/manifest_merge_manager.cc @@ -143,8 +143,8 @@ Result> ManifestMergeManager::MergeGroup( ICEBERG_ASSIGN_OR_RAISE( auto merged, FlushBin(bin, snapshot_id, metadata, file_io, writer_factory)); if (bin.size() > 1) { - replaced_manifests_count_ += static_cast(std::ranges::count_if( - bin, [snapshot_id](const ManifestFile* manifest) { + replaced_manifests_count_ += static_cast( + std::ranges::count_if(bin, [snapshot_id](const ManifestFile* manifest) { return manifest->added_snapshot_id != snapshot_id; })); } diff --git a/src/iceberg/test/manifest_filter_manager_test.cc b/src/iceberg/test/manifest_filter_manager_test.cc index 7dc4ee2d9..a49445ba3 100644 --- a/src/iceberg/test/manifest_filter_manager_test.cc +++ b/src/iceberg/test/manifest_filter_manager_test.cc @@ -475,10 +475,10 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanDuringDeleteManifestRe auto manifest_path = std::format("{}/metadata/del-manifest-{}.avro", table_location_, manifest_counter_++); - ICEBERG_UNWRAP_OR_FAIL(auto del_manifest, - WriteDeleteManifest( - {{old_file, 2L}, {targeted_file, 10L}, {keep_file, 10L}}, - file_io_, *metadata, manifest_path)); + ICEBERG_UNWRAP_OR_FAIL( + auto del_manifest, + WriteDeleteManifest({{old_file, 2L}, {targeted_file, 10L}, {keep_file, 10L}}, + file_io_, *metadata, manifest_path)); ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); mgr.DeleteFile(targeted_file->file_path); diff --git a/src/iceberg/test/manifest_merge_manager_test.cc b/src/iceberg/test/manifest_merge_manager_test.cc index b5dd60f9e..d06ff0842 100644 --- a/src/iceberg/test/manifest_merge_manager_test.cc +++ b/src/iceberg/test/manifest_merge_manager_test.cc @@ -199,9 +199,9 @@ TEST_F(ManifestMergeManagerTest, ReplacedManifestCountTracksPreviousSnapshotInpu m1.added_snapshot_id = kSnapshotId - 2; ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.MergeManifests({m0, m1}, {}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); + ICEBERG_UNWRAP_OR_FAIL( + auto result, mgr.MergeManifests({m0, m1}, {}, kSnapshotId, *metadata_, file_io_, + MakeWriterFactory())); EXPECT_EQ(result.size(), 1U); EXPECT_EQ(mgr.ReplacedManifestsCount(), 2); @@ -212,9 +212,9 @@ TEST_F(ManifestMergeManagerTest, ReplacedManifestCountIgnoresCurrentSnapshotInpu ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.MergeManifests({}, {m0, m1}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); + ICEBERG_UNWRAP_OR_FAIL( + auto result, mgr.MergeManifests({}, {m0, m1}, kSnapshotId, *metadata_, file_io_, + MakeWriterFactory())); EXPECT_EQ(result.size(), 1U); EXPECT_EQ(mgr.ReplacedManifestsCount(), 0); diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index d436b7c53..a8317ddc3 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -107,31 +107,35 @@ class TestMergeAppend : public MergingSnapshotUpdate { metadata, starting_snapshot_id, std::move(data_filter), replaced_files, parent, std::move(io)); } - static Status ValidateNoNewDeleteFilesForTest( - const TableMetadata& metadata, int64_t starting_snapshot_id, - std::shared_ptr data_filter, const std::shared_ptr& parent, - std::shared_ptr io) { + static Status ValidateNoNewDeleteFilesForTest(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const std::shared_ptr& parent, + std::shared_ptr io) { return MergingSnapshotUpdate::ValidateNoNewDeleteFiles( metadata, starting_snapshot_id, std::move(data_filter), parent, std::move(io)); } - static Status ValidateNoNewDeleteFilesForTest( - const TableMetadata& metadata, int64_t starting_snapshot_id, - const PartitionSet& partition_set, const std::shared_ptr& parent, - std::shared_ptr io) { + static Status ValidateNoNewDeleteFilesForTest(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const PartitionSet& partition_set, + const std::shared_ptr& parent, + std::shared_ptr io) { return MergingSnapshotUpdate::ValidateNoNewDeleteFiles( metadata, starting_snapshot_id, partition_set, parent, std::move(io)); } - static Status ValidateDeletedDataFilesForTest( - const TableMetadata& metadata, int64_t starting_snapshot_id, - std::shared_ptr data_filter, const std::shared_ptr& parent, - std::shared_ptr io) { + static Status ValidateDeletedDataFilesForTest(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const std::shared_ptr& parent, + std::shared_ptr io) { return MergingSnapshotUpdate::ValidateDeletedDataFiles( metadata, starting_snapshot_id, std::move(data_filter), parent, std::move(io)); } - static Status ValidateDeletedDataFilesForTest( - const TableMetadata& metadata, int64_t starting_snapshot_id, - const PartitionSet& partition_set, const std::shared_ptr& parent, - std::shared_ptr io) { + static Status ValidateDeletedDataFilesForTest(const TableMetadata& metadata, + int64_t starting_snapshot_id, + const PartitionSet& partition_set, + const std::shared_ptr& parent, + std::shared_ptr io) { return MergingSnapshotUpdate::ValidateDeletedDataFiles( metadata, starting_snapshot_id, partition_set, parent, std::move(io)); } @@ -147,30 +151,30 @@ class TestMergeAppend : public MergingSnapshotUpdate { class TestOverwriteUpdate : public MergingSnapshotUpdate { public: - static Result> Make( - std::string table_name, std::shared_ptr
table) { - ICEBERG_ASSIGN_OR_RAISE( - auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); - return std::unique_ptr( - new TestOverwriteUpdate(std::move(table_name), std::move(ctx))); - } - - std::string operation() override { return DataOperation::kOverwrite; } - int64_t GeneratedSnapshotId() { return SnapshotId(); } - - Status AddDelete(std::shared_ptr file) { - return AddDeleteFile(std::move(file)); - } - Status AddDelete(std::shared_ptr file, int64_t data_sequence_number) { - return AddDeleteFile(std::move(file), data_sequence_number); - } - Status RemoveDataFile(std::shared_ptr file) { - return DeleteDataFile(std::move(file)); - } + static Result> Make(std::string table_name, + std::shared_ptr
table) { + ICEBERG_ASSIGN_OR_RAISE( + auto ctx, TransactionContext::Make(std::move(table), TransactionKind::kUpdate)); + return std::unique_ptr( + new TestOverwriteUpdate(std::move(table_name), std::move(ctx))); + } + + std::string operation() override { return DataOperation::kOverwrite; } + int64_t GeneratedSnapshotId() { return SnapshotId(); } + + Status AddDelete(std::shared_ptr file) { + return AddDeleteFile(std::move(file)); + } + Status AddDelete(std::shared_ptr file, int64_t data_sequence_number) { + return AddDeleteFile(std::move(file), data_sequence_number); + } + Status RemoveDataFile(std::shared_ptr file) { + return DeleteDataFile(std::move(file)); + } private: - TestOverwriteUpdate(std::string table_name, std::shared_ptr ctx) - : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {} + TestOverwriteUpdate(std::string table_name, std::shared_ptr ctx) + : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {} }; class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { @@ -268,8 +272,8 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { std::string operation, int64_t snapshot_id, std::optional parent_snapshot_id, int64_t sequence_number, const std::vector& manifests) { - auto manifest_list_path = - table_location_ + "/metadata/manifest-list-" + std::to_string(snapshot_id) + ".avro"; + auto manifest_list_path = table_location_ + "/metadata/manifest-list-" + + std::to_string(snapshot_id) + ".avro"; ICEBERG_ASSIGN_OR_RAISE( auto writer, ManifestListWriter::MakeWriter(table_->metadata()->format_version, snapshot_id, @@ -452,9 +456,9 @@ TEST_F(MergingSnapshotUpdateTest, AddDeleteFileWithExplicitSequenceWritesSequenc return manifest.content == ManifestContent::kDeletes; }); ASSERT_NE(delete_manifest_it, manifests.end()); - ICEBERG_UNWRAP_OR_FAIL( - auto entries, ReadAllEntries(std::vector{*delete_manifest_it}, - *table_->metadata())); + ICEBERG_UNWRAP_OR_FAIL(auto entries, + ReadAllEntries(std::vector{*delete_manifest_it}, + *table_->metadata())); ASSERT_EQ(entries.size(), 1U); ASSERT_TRUE(entries[0].sequence_number.has_value()); EXPECT_EQ(entries[0].sequence_number.value(), 17); @@ -888,7 +892,8 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesFailsForTruncatedHistory metadata->current_schema_id = 0; metadata->schemas.push_back(schema_); - auto make_snapshot = [](int64_t snapshot_id, std::optional parent_snapshot_id) { + auto make_snapshot = [](int64_t snapshot_id, + std::optional parent_snapshot_id) { return std::make_shared(Snapshot{ .snapshot_id = snapshot_id, .parent_snapshot_id = parent_snapshot_id, @@ -928,7 +933,8 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithPartitionSetDetectsC IsError(ErrorKind::kInvalidArgument)); } -TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesWithFilterDetectsConflict) { +TEST_F(MergingSnapshotUpdateTest, + ValidateNoNewDeletesForDataFilesWithFilterDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); @@ -951,8 +957,8 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesWithFilterDete DataFileSet replaced_files; replaced_files.insert(file_a_); EXPECT_THAT(TestMergeAppend::ValidateNoNewDeletesForDataFilesForTest( - *metadata, first_snapshot->snapshot_id, - Expressions::AlwaysTrue(), replaced_files, second_snapshot, file_io_), + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), + replaced_files, second_snapshot, file_io_), IsError(ErrorKind::kInvalidArgument)); } @@ -977,12 +983,13 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithExpressionDetectsC metadata->last_sequence_number = second_snapshot->sequence_number; EXPECT_THAT(TestMergeAppend::ValidateNoNewDeleteFilesForTest( - *metadata, first_snapshot->snapshot_id, - Expressions::AlwaysTrue(), second_snapshot, file_io_), + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), + second_snapshot, file_io_), IsError(ErrorKind::kInvalidArgument)); } -TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithPartitionSetDetectsConflict) { +TEST_F(MergingSnapshotUpdateTest, + ValidateNoNewDeleteFilesWithPartitionSetDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); @@ -1005,8 +1012,8 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithPartitionSetDetect PartitionSet partition_set; ASSERT_TRUE(partition_set.add(spec_->spec_id(), del_file->partition)); EXPECT_THAT(TestMergeAppend::ValidateNoNewDeleteFilesForTest( - *metadata, first_snapshot->snapshot_id, partition_set, - second_snapshot, file_io_), + *metadata, first_snapshot->snapshot_id, partition_set, second_snapshot, + file_io_), IsError(ErrorKind::kInvalidArgument)); } @@ -1030,12 +1037,13 @@ TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsC metadata->last_sequence_number = second_snapshot->sequence_number; EXPECT_THAT(TestMergeAppend::ValidateDeletedDataFilesForTest( - *metadata, first_snapshot->snapshot_id, - Expressions::AlwaysTrue(), second_snapshot, file_io_), + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), + second_snapshot, file_io_), IsError(ErrorKind::kInvalidArgument)); } -TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithPartitionSetDetectsConflict) { +TEST_F(MergingSnapshotUpdateTest, + ValidateDeletedDataFilesWithPartitionSetDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); @@ -1057,8 +1065,8 @@ TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithPartitionSetDetect PartitionSet partition_set; ASSERT_TRUE(partition_set.add(spec_->spec_id(), file_a_->partition)); EXPECT_THAT(TestMergeAppend::ValidateDeletedDataFilesForTest( - *metadata, first_snapshot->snapshot_id, partition_set, - second_snapshot, file_io_), + *metadata, first_snapshot->snapshot_id, partition_set, second_snapshot, + file_io_), IsError(ErrorKind::kInvalidArgument)); } diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index 738dc2dcd..93b169682 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -50,8 +50,7 @@ namespace { bool MatchesOperation(std::optional operation, std::initializer_list expected) { return operation.has_value() && - std::find(expected.begin(), expected.end(), operation.value()) != - expected.end(); + std::find(expected.begin(), expected.end(), operation.value()) != expected.end(); } struct ValidationHistoryResult { @@ -69,17 +68,19 @@ Result>> ValidationAncestorsBetween( return ancestors; } if (ancestors.empty()) { - return InvalidArgument("Cannot validate history: starting snapshot {} is not an ancestor " - "of snapshot {}", - starting_snapshot_id, latest_snapshot_id); + return InvalidArgument( + "Cannot validate history: starting snapshot {} is not an ancestor " + "of snapshot {}", + starting_snapshot_id, latest_snapshot_id); } const auto& oldest_checked = ancestors.back(); if (oldest_checked == nullptr || !oldest_checked->parent_snapshot_id.has_value() || oldest_checked->parent_snapshot_id.value() != starting_snapshot_id) { - return InvalidArgument("Cannot validate history: starting snapshot {} is not an ancestor " - "of snapshot {}", - starting_snapshot_id, latest_snapshot_id); + return InvalidArgument( + "Cannot validate history: starting snapshot {} is not an ancestor " + "of snapshot {}", + starting_snapshot_id, latest_snapshot_id); } return ancestors; } @@ -87,8 +88,8 @@ Result>> ValidationAncestorsBetween( Result ValidationHistory( const TableMetadata& metadata, int64_t latest_snapshot_id, int64_t starting_snapshot_id, - std::initializer_list matching_operations, - ManifestContent content, const std::shared_ptr& io) { + std::initializer_list matching_operations, ManifestContent content, + const std::shared_ptr& io) { ICEBERG_ASSIGN_OR_RAISE( auto ancestors, ValidationAncestorsBetween(metadata, latest_snapshot_id, starting_snapshot_id)); @@ -101,9 +102,9 @@ Result ValidationHistory( result.snapshot_ids.insert(snapshot->snapshot_id); auto cached = SnapshotCache(snapshot.get()); - ICEBERG_ASSIGN_OR_RAISE( - auto manifests, content == ManifestContent::kData ? cached.DataManifests(io) - : cached.DeleteManifests(io)); + ICEBERG_ASSIGN_OR_RAISE(auto manifests, content == ManifestContent::kData + ? cached.DataManifests(io) + : cached.DeleteManifests(io)); for (const auto& manifest : manifests) { if (manifest.added_snapshot_id == snapshot->snapshot_id) { result.manifests.push_back(manifest); @@ -127,7 +128,8 @@ Result> FindMatchingDataFile( for (const auto& manifest : manifests) { ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata.PartitionSpecById(manifest.partition_spec_id)); - ICEBERG_ASSIGN_OR_RAISE(auto reader, ManifestReader::Make(manifest, io, schema, spec)); + ICEBERG_ASSIGN_OR_RAISE(auto reader, + ManifestReader::Make(manifest, io, schema, spec)); reader->CaseSensitive(case_sensitive); if (filter != nullptr) { reader->FilterRows(filter); @@ -247,9 +249,8 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, base().PartitionSpecById(file->partition_spec_id.value())); ICEBERG_RETURN_UNEXPECTED(added_delete_files_summary_.AddedFile(*spec, *file)); has_new_delete_files_ = true; - new_delete_files_.push_back( - PendingDeleteFile{.file = std::move(file), - .data_sequence_number = std::move(data_sequence_number)}); + new_delete_files_.push_back(PendingDeleteFile{ + .file = std::move(file), .data_sequence_number = std::move(data_sequence_number)}); return {}; } @@ -381,15 +382,17 @@ bool MergingSnapshotUpdate::DeletesDeleteFiles() const { ManifestWriterFactory MergingSnapshotUpdate::MakeTrackedWriterFactory( const std::shared_ptr& schema) { - return [this, schema](int32_t spec_id, - ManifestContent content) -> Result> { - const TableMetadata& meta = base(); - ICEBERG_ASSIGN_OR_RAISE(auto spec, meta.PartitionSpecById(spec_id)); - std::string path = ManifestPath(); - all_written_manifests_.insert(path); - return ManifestWriter::MakeWriter(meta.format_version, SnapshotId(), std::move(path), - ctx_->table->io(), std::move(spec), schema, content); - }; + return + [this, schema](int32_t spec_id, + ManifestContent content) -> Result> { + const TableMetadata& meta = base(); + ICEBERG_ASSIGN_OR_RAISE(auto spec, meta.PartitionSpecById(spec_id)); + std::string path = ManifestPath(); + all_written_manifests_.insert(path); + return ManifestWriter::MakeWriter(meta.format_version, SnapshotId(), + std::move(path), ctx_->table->io(), + std::move(spec), schema, content); + }; } Result> MergingSnapshotUpdate::WriteNewDataManifests() { @@ -454,8 +457,7 @@ Result> MergingSnapshotUpdate::WriteNewDeleteManifests .data_sequence_number = pending_file.data_sequence_number, }); } - ICEBERG_ASSIGN_OR_RAISE(auto written, - WriteDeleteManifests(delete_entries, spec)); + ICEBERG_ASSIGN_OR_RAISE(auto written, WriteDeleteManifests(delete_entries, spec)); for (const auto& m : written) { all_written_manifests_.insert(m.manifest_path); } @@ -487,9 +489,9 @@ Result> MergingSnapshotUpdate::Apply( auto tracked_factory = MakeTrackedWriterFactory(target_schema); // Step 1: Filter data manifests. - ICEBERG_ASSIGN_OR_RAISE(auto filtered_data, - data_filter_manager_.FilterManifests( - target_schema, metadata_to_update, snapshot, tracked_factory)); + ICEBERG_ASSIGN_OR_RAISE(auto filtered_data, data_filter_manager_.FilterManifests( + target_schema, metadata_to_update, + snapshot, tracked_factory)); // Track deleted data files in the summary builder. for (const auto& file : data_filter_manager_.FilesToBeDeleted()) { @@ -520,9 +522,9 @@ Result> MergingSnapshotUpdate::Apply( data_filter_manager_.FilesToBeDeleted()); // Step 3: Filter delete manifests. - ICEBERG_ASSIGN_OR_RAISE(auto filtered_deletes, - delete_filter_manager_.FilterManifests( - target_schema, metadata_to_update, snapshot, tracked_factory)); + ICEBERG_ASSIGN_OR_RAISE(auto filtered_deletes, delete_filter_manager_.FilterManifests( + target_schema, metadata_to_update, + snapshot, tracked_factory)); // Track deleted delete files in the summary builder. for (const auto& file : delete_filter_manager_.FilesToBeDeleted()) { @@ -656,18 +658,17 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( } ICEBERG_ASSIGN_OR_RAISE( - auto history, - ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kAppend, DataOperation::kOverwrite}, - ManifestContent::kData, io)); - ICEBERG_ASSIGN_OR_RAISE(auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, - ManifestStatus::kAdded, filter, nullptr, io, - case_sensitive)); + auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kAppend, DataOperation::kOverwrite}, + ManifestContent::kData, io)); + ICEBERG_ASSIGN_OR_RAISE( + auto conflict_path, + FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kAdded, filter, + nullptr, io, case_sensitive)); if (conflict_path.has_value()) { - return InvalidArgument("Found conflicting files that can contain rows matching {}: {}", - filter != nullptr ? filter->ToString() : "any expression", - conflict_path.value()); + return InvalidArgument( + "Found conflicting files that can contain rows matching {}: {}", + filter != nullptr ? filter->ToString() : "any expression", conflict_path.value()); } return {}; } @@ -682,9 +683,9 @@ Status MergingSnapshotUpdate::ValidateDataFilesExist( } ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); - ICEBERG_ASSIGN_OR_RAISE(auto ancestors, - ValidationAncestorsBetween(metadata, parent->snapshot_id, - starting_snapshot_id)); + ICEBERG_ASSIGN_OR_RAISE( + auto ancestors, + ValidationAncestorsBetween(metadata, parent->snapshot_id, starting_snapshot_id)); // Build the full set of matching snapshot IDs first, then scan their manifests. // The full set must be known before filtering manifests, since a manifest may have @@ -807,10 +808,9 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( } ICEBERG_ASSIGN_OR_RAISE( - auto history, - ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kAppend, DataOperation::kOverwrite}, - ManifestContent::kData, io)); + auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kAppend, DataOperation::kOverwrite}, + ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( auto conflict_path, FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kAdded, nullptr, @@ -831,9 +831,8 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( return {}; } - ICEBERG_ASSIGN_OR_RAISE( - auto deletes, - AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, nullptr, parent, io)); + ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, + nullptr, nullptr, parent, io)); if (deletes->empty()) { return {}; } @@ -847,9 +846,9 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); std::unique_ptr evaluator; if (data_filter != nullptr) { - ICEBERG_ASSIGN_OR_RAISE( - evaluator, InclusiveMetricsEvaluator::Make(data_filter, *schema, - /*case_sensitive=*/true)); + ICEBERG_ASSIGN_OR_RAISE(evaluator, + InclusiveMetricsEvaluator::Make(data_filter, *schema, + /*case_sensitive=*/true)); } for (const auto& data_file : replaced_files) { @@ -871,19 +870,18 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( const TableMetadata& metadata, int64_t starting_snapshot_id, - std::shared_ptr data_filter, - const std::shared_ptr& parent, std::shared_ptr io) { - ICEBERG_ASSIGN_OR_RAISE( - auto deletes, - AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, nullptr, parent, io)); + std::shared_ptr data_filter, const std::shared_ptr& parent, + std::shared_ptr io) { + ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, + nullptr, nullptr, parent, io)); auto referenced_delete_files = deletes->ReferencedDeleteFiles(); ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); std::unique_ptr evaluator; if (data_filter != nullptr) { - ICEBERG_ASSIGN_OR_RAISE( - evaluator, InclusiveMetricsEvaluator::Make(data_filter, *schema, - /*case_sensitive=*/true)); + ICEBERG_ASSIGN_OR_RAISE(evaluator, + InclusiveMetricsEvaluator::Make(data_filter, *schema, + /*case_sensitive=*/true)); } for (const auto& delete_file : referenced_delete_files) { @@ -903,9 +901,8 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( const TableMetadata& metadata, int64_t starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io) { - ICEBERG_ASSIGN_OR_RAISE( - auto deletes, - AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, nullptr, parent, io)); + ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, + nullptr, nullptr, parent, io)); auto referenced_delete_files = deletes->ReferencedDeleteFiles(); for (const auto& delete_file : referenced_delete_files) { if (!delete_file->partition_spec_id.has_value() || @@ -913,30 +910,30 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( delete_file->partition)) { continue; } - return InvalidArgument("Found new conflicting delete files in validated partitions: {}", - delete_file->file_path); + return InvalidArgument( + "Found new conflicting delete files in validated partitions: {}", + delete_file->file_path); } return {}; } Status MergingSnapshotUpdate::ValidateDeletedDataFiles( const TableMetadata& metadata, int64_t starting_snapshot_id, - std::shared_ptr data_filter, - const std::shared_ptr& parent, std::shared_ptr io) { + std::shared_ptr data_filter, const std::shared_ptr& parent, + std::shared_ptr io) { if (parent == nullptr) { return {}; } ICEBERG_ASSIGN_OR_RAISE( - auto history, - ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kOverwrite, DataOperation::kReplace, - DataOperation::kDelete}, - ManifestContent::kData, io)); - ICEBERG_ASSIGN_OR_RAISE(auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, - ManifestStatus::kDeleted, data_filter, - nullptr, io, /*case_sensitive=*/true)); + auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kOverwrite, DataOperation::kReplace, + DataOperation::kDelete}, + ManifestContent::kData, io)); + ICEBERG_ASSIGN_OR_RAISE( + auto conflict_path, + FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kDeleted, + data_filter, nullptr, io, /*case_sensitive=*/true)); if (conflict_path.has_value()) { return InvalidArgument( "Found conflicting deleted files that can contain rows matching {}: {}", @@ -955,19 +952,17 @@ Status MergingSnapshotUpdate::ValidateDeletedDataFiles( } ICEBERG_ASSIGN_OR_RAISE( - auto history, - ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kOverwrite, DataOperation::kReplace, - DataOperation::kDelete}, - ManifestContent::kData, io)); + auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kOverwrite, DataOperation::kReplace, + DataOperation::kDelete}, + ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( auto conflict_path, FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kDeleted, nullptr, &partition_set, io, /*case_sensitive=*/true)); if (conflict_path.has_value()) { - return InvalidArgument( - "Found conflicting deleted files in validated partitions: {}", - conflict_path.value()); + return InvalidArgument("Found conflicting deleted files in validated partitions: {}", + conflict_path.value()); } return {}; } @@ -990,10 +985,9 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles } ICEBERG_ASSIGN_OR_RAISE( - auto history, - ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kOverwrite, DataOperation::kDelete}, - ManifestContent::kDeletes, io)); + auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kOverwrite, DataOperation::kDelete}, + ManifestContent::kDeletes, io)); // Compute the starting sequence number from the starting snapshot. int64_t starting_seq = TableMetadata::kInitialSequenceNumber; From eb175226ddd600129bce3772a610d7ea52e96f4a Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Wed, 27 May 2026 15:25:56 +0800 Subject: [PATCH 04/16] Use ranges algorithms in parity updates Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/iceberg/test/merging_snapshot_update_test.cc | 2 +- src/iceberg/update/merging_snapshot_update.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index a8317ddc3..ec3f09967 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -452,7 +452,7 @@ TEST_F(MergingSnapshotUpdateTest, AddDeleteFileWithExplicitSequenceWritesSequenc EXPECT_THAT(op->AddDelete(del_file, 17), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); auto delete_manifest_it = - std::find_if(manifests.begin(), manifests.end(), [](const ManifestFile& manifest) { + std::ranges::find_if(manifests, [](const ManifestFile& manifest) { return manifest.content == ManifestContent::kDeletes; }); ASSERT_NE(delete_manifest_it, manifests.end()); diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index 93b169682..a25154e58 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -50,7 +50,7 @@ namespace { bool MatchesOperation(std::optional operation, std::initializer_list expected) { return operation.has_value() && - std::find(expected.begin(), expected.end(), operation.value()) != expected.end(); + std::ranges::find(expected, operation.value()) != expected.end(); } struct ValidationHistoryResult { From 56b4df190664720ab56d5e966e4ac9245994325f Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Wed, 27 May 2026 16:01:20 +0800 Subject: [PATCH 05/16] Fix manifest entry null data file crash --- src/iceberg/manifest/manifest_entry.h | 2 +- src/iceberg/test/manifest_writer_versions_test.cc | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/iceberg/manifest/manifest_entry.h b/src/iceberg/manifest/manifest_entry.h index 17c5388bd..e2dd5ea61 100644 --- a/src/iceberg/manifest/manifest_entry.h +++ b/src/iceberg/manifest/manifest_entry.h @@ -374,7 +374,7 @@ struct ICEBERG_EXPORT ManifestEntry { ManifestEntry AsAdded() const { ManifestEntry copy = *this; copy.status = ManifestStatus::kAdded; - if (copy.data_file->first_row_id.has_value()) { + if (copy.data_file != nullptr && copy.data_file->first_row_id.has_value()) { copy.data_file = std::make_unique(*copy.data_file); copy.data_file->first_row_id = std::nullopt; } diff --git a/src/iceberg/test/manifest_writer_versions_test.cc b/src/iceberg/test/manifest_writer_versions_test.cc index 990224528..24669d5b5 100644 --- a/src/iceberg/test/manifest_writer_versions_test.cc +++ b/src/iceberg/test/manifest_writer_versions_test.cc @@ -435,6 +435,20 @@ TEST_F(ManifestWriterVersionsTest, TestV1WriteDelete) { "Cannot write equality_deletes file to data manifest file")); } +TEST_F(ManifestWriterVersionsTest, TestWriteAddedEntryRejectsMissingDataFile) { + const std::string manifest_path = CreateManifestPath(); + ICEBERG_UNWRAP_OR_FAIL( + auto writer, ManifestWriter::MakeWriter(/*format_version=*/2, kSnapshotId, + manifest_path, file_io_, spec_, schema_)); + + ManifestEntry entry; + entry.snapshot_id = kSnapshotId; + + auto status = writer->WriteAddedEntry(entry); + EXPECT_THAT(status, IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(status, HasErrorMessage("Data file cannot be null")); +} + TEST_F(ManifestWriterVersionsTest, TestV1WriteWithInheritance) { auto manifests = WriteAndReadManifests({WriteManifest(/*format_version=*/1, {data_file_})}, 1); From d1aff7110cbc0be7296cffc541a428bea19817dc Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Wed, 27 May 2026 21:04:08 +0800 Subject: [PATCH 06/16] fix ut fail --- src/iceberg/test/data_file_set_test.cc | 19 +++++++++++++++++++ src/iceberg/util/data_file_set.h | 4 ++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/iceberg/test/data_file_set_test.cc b/src/iceberg/test/data_file_set_test.cc index 60539adfa..ca2571340 100644 --- a/src/iceberg/test/data_file_set_test.cc +++ b/src/iceberg/test/data_file_set_test.cc @@ -19,6 +19,8 @@ #include "iceberg/util/data_file_set.h" +#include + #include #include "iceberg/file_format.h" @@ -73,6 +75,23 @@ TEST_F(DataFileSetTest, InsertDuplicateFile) { EXPECT_EQ(set.size(), 1); // Should still be size 1 } +TEST_F(DataFileSetTest, InsertedPathIndexOwnsKey) { + DataFileSet set; + const std::string original_path(256, 'a'); + auto file1 = CreateDataFile(original_path); + + auto [iter1, inserted1] = set.insert(file1); + EXPECT_TRUE(inserted1); + + file1->file_path = std::string(256, 'b'); + + auto file2 = CreateDataFile(original_path); + auto [iter2, inserted2] = set.insert(file2); + EXPECT_FALSE(inserted2); + EXPECT_EQ(iter1, iter2); + EXPECT_EQ(set.size(), 1); +} + TEST_F(DataFileSetTest, InsertDifferentFiles) { DataFileSet set; auto file1 = CreateDataFile("/path/to/file1.parquet"); diff --git a/src/iceberg/util/data_file_set.h b/src/iceberg/util/data_file_set.h index 741b34e56..aee15a46c 100644 --- a/src/iceberg/util/data_file_set.h +++ b/src/iceberg/util/data_file_set.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include @@ -102,7 +102,7 @@ class ICEBERG_EXPORT DataFileSet { // Vector to preserve insertion order std::vector elements_; - std::unordered_map index_by_path_; + std::unordered_map index_by_path_; }; } // namespace iceberg From 122bca4ef82238d5b1b4f5275d1c20146668cd40 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Thu, 28 May 2026 09:47:15 +0800 Subject: [PATCH 07/16] Fix equality delete schema lifetime --- .github/workflows/sanitizer_test.yml | 2 +- src/iceberg/delete_file_index.cc | 6 ++--- src/iceberg/delete_file_index.h | 6 +++-- src/iceberg/test/delete_file_index_test.cc | 31 +++++++++++++++++++++- 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index efe9f49ac..8e2f7e4bb 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -68,4 +68,4 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-output - path: build/test/out.log* + path: build/out.log* diff --git a/src/iceberg/delete_file_index.cc b/src/iceberg/delete_file_index.cc index 7c8c35032..43d822601 100644 --- a/src/iceberg/delete_file_index.cc +++ b/src/iceberg/delete_file_index.cc @@ -215,7 +215,7 @@ Status EqualityDeletes::Add(ManifestEntry&& entry) { ICEBERG_PRECHECK(entry.sequence_number.has_value(), "Missing sequence number from equality delete: {}", entry.data_file->file_path); - files_.emplace_back(&schema_, std::move(entry)); + files_.emplace_back(schema_.get(), std::move(entry)); indexed_ = false; return {}; } @@ -720,7 +720,7 @@ Status DeleteFileIndex::Builder::AddEqualityDelete( if (existing.has_value()) { ICEBERG_RETURN_UNEXPECTED(existing->get()->Add(std::move(entry))); } else { - auto deletes = std::make_unique(*schema_); + auto deletes = std::make_unique(schema_); ICEBERG_RETURN_UNEXPECTED(deletes->Add(std::move(entry))); deletes_by_partition.put(spec_id, partition, std::move(deletes)); } @@ -738,7 +738,7 @@ Result> DeleteFileIndex::Builder::Build() { } // Build index structures - auto global_deletes = std::make_unique(*schema_); + auto global_deletes = std::make_unique(schema_); auto eq_deletes_by_partition = std::make_unique>>(); auto pos_deletes_by_partition = diff --git a/src/iceberg/delete_file_index.h b/src/iceberg/delete_file_index.h index 5444281a0..76a2ce2d8 100644 --- a/src/iceberg/delete_file_index.h +++ b/src/iceberg/delete_file_index.h @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "iceberg/expression/literal.h" @@ -184,7 +185,8 @@ class ICEBERG_EXPORT PositionDeletes { /// data sequence number (i.e., apply_sequence_number = data_sequence_number - 1). class ICEBERG_EXPORT EqualityDeletes { public: - explicit EqualityDeletes(const Schema& schema) : schema_(schema) {} + explicit EqualityDeletes(std::shared_ptr schema) + : schema_(std::move(schema)) {} /// \brief Add an equality delete file to this group. [[nodiscard]] Status Add(ManifestEntry&& entry); @@ -206,7 +208,7 @@ class ICEBERG_EXPORT EqualityDeletes { private: void IndexIfNeeded(); - const Schema& schema_; + std::shared_ptr schema_; std::vector files_; std::vector seqs_; bool indexed_ = false; diff --git a/src/iceberg/test/delete_file_index_test.cc b/src/iceberg/test/delete_file_index_test.cc index 0c8c8821b..fb21b39e1 100644 --- a/src/iceberg/test/delete_file_index_test.cc +++ b/src/iceberg/test/delete_file_index_test.cc @@ -776,7 +776,7 @@ TEST_P(DeleteFileIndexTest, TestPositionDeletesGroup) { } TEST_P(DeleteFileIndexTest, TestEqualityDeletesGroup) { - internal::EqualityDeletes group(*schema_); + internal::EqualityDeletes group(schema_); auto partition_a = PartitionValues({Literal::Int(0)}); auto file1 = MakeEqualityDeleteFile("/path/to/eq-delete-1.parquet", partition_a, @@ -840,6 +840,35 @@ TEST_P(DeleteFileIndexTest, TestEqualityDeletesGroup) { } } +TEST_P(DeleteFileIndexTest, TestIndexOwnsSchemaForEqualityDeletes) { + auto version = GetParam(); + + auto eq_delete = MakeEqualityDeleteFile("/path/to/eq-delete.parquet", + PartitionValues(std::vector{}), + unpartitioned_spec_->spec_id()); + std::vector entries; + entries.push_back( + MakeDeleteEntry(/*snapshot_id=*/1000L, /*sequence_number=*/2, eq_delete)); + + auto manifest = WriteDeleteManifest(version, /*snapshot_id=*/1000L, std::move(entries), + unpartitioned_spec_); + + auto schema = std::make_shared(std::vector{ + SchemaField::MakeRequired(/*field_id=*/1, "id", int32()), + SchemaField::MakeRequired(/*field_id=*/2, "data", string())}); + ICEBERG_UNWRAP_OR_FAIL( + auto builder, + DeleteFileIndex::BuilderFor(file_io_, schema, + {{unpartitioned_spec_->spec_id(), unpartitioned_spec_}}, + {manifest})); + ICEBERG_UNWRAP_OR_FAIL(auto index, builder.Build()); + schema.reset(); + + ICEBERG_UNWRAP_OR_FAIL(auto deletes, index->ForDataFile(1, *unpartitioned_file_)); + ASSERT_EQ(deletes.size(), 1); + EXPECT_EQ(deletes[0]->file_path, "/path/to/eq-delete.parquet"); +} + TEST_P(DeleteFileIndexTest, TestMixDeleteFilesAndDVs) { auto version = GetParam(); if (version < 3) { From 29c76e4d53d62e7fa90080c19dad518d88b78d4a Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Thu, 28 May 2026 10:53:04 +0800 Subject: [PATCH 08/16] Fix metadata cache lifetime in delete validation --- .github/workflows/sanitizer_test.yml | 8 +++++--- src/iceberg/update/merging_snapshot_update.cc | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 8e2f7e4bb..fc636cec7 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -58,9 +58,9 @@ jobs: - name: Run Tests working-directory: build env: - ASAN_OPTIONS: log_path=out.log:detect_leaks=1:symbolize=1:strict_string_checks=1:halt_on_error=1:detect_container_overflow=0 + ASAN_OPTIONS: log_path=${{ github.workspace }}/asan.log:detect_leaks=1:symbolize=1:strict_string_checks=1:halt_on_error=1:detect_container_overflow=0 LSAN_OPTIONS: suppressions=${{ github.workspace }}/.github/lsan-suppressions.txt - UBSAN_OPTIONS: log_path=out.log:halt_on_error=1:print_stacktrace=1:suppressions=${{ github.workspace }}/.github/ubsan-suppressions.txt + UBSAN_OPTIONS: log_path=${{ github.workspace }}/ubsan.log:halt_on_error=1:print_stacktrace=1:suppressions=${{ github.workspace }}/.github/ubsan-suppressions.txt run: | ctest --output-on-failure - name: Save the test output @@ -68,4 +68,6 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-output - path: build/out.log* + path: | + asan.log* + ubsan.log* diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index a25154e58..1ded98cc6 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -975,8 +975,8 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); if (parent == nullptr || metadata.format_version < 2) { - ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, - TableMetadataCache(&metadata).GetPartitionSpecsById()); + TableMetadataCache metadata_cache(&metadata); + ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, metadata_cache.GetPartitionSpecsById()); std::unordered_map> specs_by_id( specs_ref.get().begin(), specs_ref.get().end()); ICEBERG_ASSIGN_OR_RAISE(auto builder, DeleteFileIndex::BuilderFor( @@ -996,8 +996,8 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles starting_seq = snap_result.value()->sequence_number; } - ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, - TableMetadataCache(&metadata).GetPartitionSpecsById()); + TableMetadataCache metadata_cache(&metadata); + ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, metadata_cache.GetPartitionSpecsById()); std::unordered_map> specs_by_id( specs_ref.get().begin(), specs_ref.get().end()); From 4482769973c2439cb87c7255f50aa9cfbbbcaec1 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Thu, 28 May 2026 10:59:59 +0800 Subject: [PATCH 09/16] Minimize delete validation crash fix --- .github/workflows/sanitizer_test.yml | 8 +++--- src/iceberg/delete_file_index.cc | 6 ++--- src/iceberg/delete_file_index.h | 6 ++--- src/iceberg/test/data_file_set_test.cc | 19 ------------- src/iceberg/test/delete_file_index_test.cc | 31 +--------------------- src/iceberg/util/data_file_set.h | 4 +-- 6 files changed, 11 insertions(+), 63 deletions(-) diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index fc636cec7..efe9f49ac 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -58,9 +58,9 @@ jobs: - name: Run Tests working-directory: build env: - ASAN_OPTIONS: log_path=${{ github.workspace }}/asan.log:detect_leaks=1:symbolize=1:strict_string_checks=1:halt_on_error=1:detect_container_overflow=0 + ASAN_OPTIONS: log_path=out.log:detect_leaks=1:symbolize=1:strict_string_checks=1:halt_on_error=1:detect_container_overflow=0 LSAN_OPTIONS: suppressions=${{ github.workspace }}/.github/lsan-suppressions.txt - UBSAN_OPTIONS: log_path=${{ github.workspace }}/ubsan.log:halt_on_error=1:print_stacktrace=1:suppressions=${{ github.workspace }}/.github/ubsan-suppressions.txt + UBSAN_OPTIONS: log_path=out.log:halt_on_error=1:print_stacktrace=1:suppressions=${{ github.workspace }}/.github/ubsan-suppressions.txt run: | ctest --output-on-failure - name: Save the test output @@ -68,6 +68,4 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-output - path: | - asan.log* - ubsan.log* + path: build/test/out.log* diff --git a/src/iceberg/delete_file_index.cc b/src/iceberg/delete_file_index.cc index 43d822601..7c8c35032 100644 --- a/src/iceberg/delete_file_index.cc +++ b/src/iceberg/delete_file_index.cc @@ -215,7 +215,7 @@ Status EqualityDeletes::Add(ManifestEntry&& entry) { ICEBERG_PRECHECK(entry.sequence_number.has_value(), "Missing sequence number from equality delete: {}", entry.data_file->file_path); - files_.emplace_back(schema_.get(), std::move(entry)); + files_.emplace_back(&schema_, std::move(entry)); indexed_ = false; return {}; } @@ -720,7 +720,7 @@ Status DeleteFileIndex::Builder::AddEqualityDelete( if (existing.has_value()) { ICEBERG_RETURN_UNEXPECTED(existing->get()->Add(std::move(entry))); } else { - auto deletes = std::make_unique(schema_); + auto deletes = std::make_unique(*schema_); ICEBERG_RETURN_UNEXPECTED(deletes->Add(std::move(entry))); deletes_by_partition.put(spec_id, partition, std::move(deletes)); } @@ -738,7 +738,7 @@ Result> DeleteFileIndex::Builder::Build() { } // Build index structures - auto global_deletes = std::make_unique(schema_); + auto global_deletes = std::make_unique(*schema_); auto eq_deletes_by_partition = std::make_unique>>(); auto pos_deletes_by_partition = diff --git a/src/iceberg/delete_file_index.h b/src/iceberg/delete_file_index.h index 76a2ce2d8..5444281a0 100644 --- a/src/iceberg/delete_file_index.h +++ b/src/iceberg/delete_file_index.h @@ -27,7 +27,6 @@ #include #include #include -#include #include #include "iceberg/expression/literal.h" @@ -185,8 +184,7 @@ class ICEBERG_EXPORT PositionDeletes { /// data sequence number (i.e., apply_sequence_number = data_sequence_number - 1). class ICEBERG_EXPORT EqualityDeletes { public: - explicit EqualityDeletes(std::shared_ptr schema) - : schema_(std::move(schema)) {} + explicit EqualityDeletes(const Schema& schema) : schema_(schema) {} /// \brief Add an equality delete file to this group. [[nodiscard]] Status Add(ManifestEntry&& entry); @@ -208,7 +206,7 @@ class ICEBERG_EXPORT EqualityDeletes { private: void IndexIfNeeded(); - std::shared_ptr schema_; + const Schema& schema_; std::vector files_; std::vector seqs_; bool indexed_ = false; diff --git a/src/iceberg/test/data_file_set_test.cc b/src/iceberg/test/data_file_set_test.cc index ca2571340..60539adfa 100644 --- a/src/iceberg/test/data_file_set_test.cc +++ b/src/iceberg/test/data_file_set_test.cc @@ -19,8 +19,6 @@ #include "iceberg/util/data_file_set.h" -#include - #include #include "iceberg/file_format.h" @@ -75,23 +73,6 @@ TEST_F(DataFileSetTest, InsertDuplicateFile) { EXPECT_EQ(set.size(), 1); // Should still be size 1 } -TEST_F(DataFileSetTest, InsertedPathIndexOwnsKey) { - DataFileSet set; - const std::string original_path(256, 'a'); - auto file1 = CreateDataFile(original_path); - - auto [iter1, inserted1] = set.insert(file1); - EXPECT_TRUE(inserted1); - - file1->file_path = std::string(256, 'b'); - - auto file2 = CreateDataFile(original_path); - auto [iter2, inserted2] = set.insert(file2); - EXPECT_FALSE(inserted2); - EXPECT_EQ(iter1, iter2); - EXPECT_EQ(set.size(), 1); -} - TEST_F(DataFileSetTest, InsertDifferentFiles) { DataFileSet set; auto file1 = CreateDataFile("/path/to/file1.parquet"); diff --git a/src/iceberg/test/delete_file_index_test.cc b/src/iceberg/test/delete_file_index_test.cc index fb21b39e1..0c8c8821b 100644 --- a/src/iceberg/test/delete_file_index_test.cc +++ b/src/iceberg/test/delete_file_index_test.cc @@ -776,7 +776,7 @@ TEST_P(DeleteFileIndexTest, TestPositionDeletesGroup) { } TEST_P(DeleteFileIndexTest, TestEqualityDeletesGroup) { - internal::EqualityDeletes group(schema_); + internal::EqualityDeletes group(*schema_); auto partition_a = PartitionValues({Literal::Int(0)}); auto file1 = MakeEqualityDeleteFile("/path/to/eq-delete-1.parquet", partition_a, @@ -840,35 +840,6 @@ TEST_P(DeleteFileIndexTest, TestEqualityDeletesGroup) { } } -TEST_P(DeleteFileIndexTest, TestIndexOwnsSchemaForEqualityDeletes) { - auto version = GetParam(); - - auto eq_delete = MakeEqualityDeleteFile("/path/to/eq-delete.parquet", - PartitionValues(std::vector{}), - unpartitioned_spec_->spec_id()); - std::vector entries; - entries.push_back( - MakeDeleteEntry(/*snapshot_id=*/1000L, /*sequence_number=*/2, eq_delete)); - - auto manifest = WriteDeleteManifest(version, /*snapshot_id=*/1000L, std::move(entries), - unpartitioned_spec_); - - auto schema = std::make_shared(std::vector{ - SchemaField::MakeRequired(/*field_id=*/1, "id", int32()), - SchemaField::MakeRequired(/*field_id=*/2, "data", string())}); - ICEBERG_UNWRAP_OR_FAIL( - auto builder, - DeleteFileIndex::BuilderFor(file_io_, schema, - {{unpartitioned_spec_->spec_id(), unpartitioned_spec_}}, - {manifest})); - ICEBERG_UNWRAP_OR_FAIL(auto index, builder.Build()); - schema.reset(); - - ICEBERG_UNWRAP_OR_FAIL(auto deletes, index->ForDataFile(1, *unpartitioned_file_)); - ASSERT_EQ(deletes.size(), 1); - EXPECT_EQ(deletes[0]->file_path, "/path/to/eq-delete.parquet"); -} - TEST_P(DeleteFileIndexTest, TestMixDeleteFilesAndDVs) { auto version = GetParam(); if (version < 3) { diff --git a/src/iceberg/util/data_file_set.h b/src/iceberg/util/data_file_set.h index aee15a46c..741b34e56 100644 --- a/src/iceberg/util/data_file_set.h +++ b/src/iceberg/util/data_file_set.h @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include @@ -102,7 +102,7 @@ class ICEBERG_EXPORT DataFileSet { // Vector to preserve insertion order std::vector elements_; - std::unordered_map index_by_path_; + std::unordered_map index_by_path_; }; } // namespace iceberg From 613ea9955971781a5a4f2f732cfbcb2a3bb826a9 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Thu, 28 May 2026 11:29:49 +0800 Subject: [PATCH 10/16] Capture sanitizer failure logs --- .github/workflows/sanitizer_test.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index efe9f49ac..faac265e8 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -58,9 +58,9 @@ jobs: - name: Run Tests working-directory: build env: - ASAN_OPTIONS: log_path=out.log:detect_leaks=1:symbolize=1:strict_string_checks=1:halt_on_error=1:detect_container_overflow=0 + ASAN_OPTIONS: log_path=${{ github.workspace }}/asan.log:detect_leaks=1:symbolize=1:strict_string_checks=1:halt_on_error=1:detect_container_overflow=0 LSAN_OPTIONS: suppressions=${{ github.workspace }}/.github/lsan-suppressions.txt - UBSAN_OPTIONS: log_path=out.log:halt_on_error=1:print_stacktrace=1:suppressions=${{ github.workspace }}/.github/ubsan-suppressions.txt + UBSAN_OPTIONS: log_path=${{ github.workspace }}/ubsan.log:halt_on_error=1:print_stacktrace=1:suppressions=${{ github.workspace }}/.github/ubsan-suppressions.txt run: | ctest --output-on-failure - name: Save the test output @@ -68,4 +68,8 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-output - path: build/test/out.log* + path: | + asan.log* + ubsan.log* + build/Testing/Temporary/LastTest.log + build/Testing/Temporary/LastTestsFailed.log From 4b3cc536373a1839a011fee8c36276db1209cdb1 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Thu, 28 May 2026 11:56:26 +0800 Subject: [PATCH 11/16] Fix snapshot cache lifetime in update tests --- .../test/merging_snapshot_update_test.cc | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index ec3f09967..12dbada30 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -608,8 +608,8 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestCopiesManifestWithAssignedSnapshotI EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); ASSERT_EQ(data_manifests.size(), 1U); EXPECT_NE(data_manifests[0].manifest_path, path); } @@ -625,8 +625,8 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestCopiesManifestWithFirstRowId) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); ASSERT_EQ(data_manifests.size(), 1U); EXPECT_NE(data_manifests[0].manifest_path, path); } @@ -645,10 +645,10 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestEmptyTable) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + SnapshotCache snapshot_cache(snapshot.get()); // In v2 with snapshot ID inheritance, the manifest path is reused directly. - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); ASSERT_EQ(data_manifests.size(), 1); EXPECT_EQ(snapshot->summary.at("added-data-files"), "2"); @@ -667,8 +667,8 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestWithDataFiles) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); // Written manifest (file_b_) + appended manifest (file_a_, file_b_) EXPECT_EQ(data_manifests.size(), 2); EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); @@ -695,8 +695,8 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestMergeWithMinCountOne) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); // Both manifests merged into one. EXPECT_EQ(data_manifests.size(), 1); EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); @@ -725,8 +725,8 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestDoNotMergeMinCount) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); // Below min-count-to-merge threshold — all 3 pass through unchanged. EXPECT_EQ(data_manifests.size(), 3); EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); @@ -753,8 +753,8 @@ TEST_F(MergingSnapshotUpdateTest, ManifestMergeMergesIntoOne) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); EXPECT_EQ(data_manifests.size(), 1); EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsReplaced), "1"); @@ -770,8 +770,8 @@ TEST_F(MergingSnapshotUpdateTest, ManifestMergeDoesNotMergeWhenBelowMinCount) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); EXPECT_EQ(data_manifests.size(), 2); EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); } @@ -791,8 +791,8 @@ TEST_F(MergingSnapshotUpdateTest, ManifestMergeDoesNotMergeWhenSizeTargetTooSmal EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, - SnapshotCache(snapshot.get()).DataManifests(file_io_)); + SnapshotCache snapshot_cache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); EXPECT_EQ(data_manifests.size(), 2); } From 390ebf0338fecbfc5abb1cc1c35ebe764c29c234 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Thu, 28 May 2026 17:14:49 +0800 Subject: [PATCH 12/16] Address merge update review comments --- .../test/merging_snapshot_update_test.cc | 79 ++++++++++++++++--- src/iceberg/update/merging_snapshot_update.cc | 48 +++-------- 2 files changed, 80 insertions(+), 47 deletions(-) diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 12dbada30..2b386f502 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -543,11 +543,18 @@ TEST_F(MergingSnapshotUpdateV1Test, ValidateNewDeleteFileV1Rejected) { EXPECT_THAT(op->AddDelete(del_file), IsError(ErrorKind::kInvalidArgument)); } -TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV2RejectsDeletionVector) { - // Position delete with referenced_data_file set = deletion vector, not allowed in v2. +TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV2AllowsReferencedPositionDelete) { auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); del_file->referenced_data_file = table_location_ + "/data/file_a.parquet"; + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV2RejectsDeletionVector) { + auto del_file = MakeDeleteFile("/delete/dv_a.puffin", 1L); + del_file->file_format = FileFormatType::kPuffin; + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddDelete(del_file), IsError(ErrorKind::kInvalidArgument)); } @@ -614,21 +621,13 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestCopiesManifestWithAssignedSnapshotI EXPECT_NE(data_manifests[0].manifest_path, path); } -TEST_F(MergingSnapshotUpdateTest, AddManifestCopiesManifestWithFirstRowId) { +TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithFirstRowId) { auto path = table_location_ + "/metadata/rowid.avro"; ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_})); manifest.first_row_id = 0; ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); - EXPECT_THAT(op->AppendManifest(manifest), IsOk()); - EXPECT_THAT(op->Commit(), IsOk()); - - EXPECT_THAT(table_->Refresh(), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - SnapshotCache snapshot_cache(snapshot.get()); - ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); - ASSERT_EQ(data_manifests.size(), 1U); - EXPECT_NE(data_manifests[0].manifest_path, path); + EXPECT_THAT(op->AppendManifest(manifest), IsError(ErrorKind::kInvalidArgument)); } // ------------------------------------------------------------------------- @@ -962,6 +961,35 @@ TEST_F(MergingSnapshotUpdateTest, IsError(ErrorKind::kInvalidArgument)); } +TEST_F(MergingSnapshotUpdateTest, + ValidateNoNewDeletesForDataFilesWithFilterSkipsNonMatchingDeletes) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + DataFileSet replaced_files; + replaced_files.insert(file_a_); + EXPECT_THAT(TestMergeAppend::ValidateNoNewDeletesForDataFilesForTest( + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysFalse(), + replaced_files, second_snapshot, file_io_), + IsOk()); +} + TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithExpressionDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); @@ -988,6 +1016,33 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithExpressionDetectsC IsError(ErrorKind::kInvalidArgument)); } +TEST_F(MergingSnapshotUpdateTest, + ValidateNoNewDeleteFilesWithExpressionSkipsNonMatchingDeletes) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + EXPECT_THAT(TestMergeAppend::ValidateNoNewDeleteFilesForTest( + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysFalse(), + second_snapshot, file_io_), + IsOk()); +} + TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithPartitionSetDetectsConflict) { CommitFileA(); diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index 1ded98cc6..5cabe774e 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -40,6 +40,7 @@ #include "iceberg/table_metadata.h" #include "iceberg/table_properties.h" #include "iceberg/transaction.h" +#include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" #include "iceberg/util/snapshot_util_internal.h" @@ -201,7 +202,7 @@ Status MergingSnapshotUpdate::ValidateNewDeleteFile(const DataFile& file) { file.file_path); } const int8_t format_version = base().format_version; - const bool is_dv = file.referenced_data_file.has_value(); + const bool is_dv = ContentFileUtil::IsDV(file); switch (format_version) { case 1: return InvalidArgument("Deletes are supported in V2 and above"); @@ -336,8 +337,11 @@ Status MergingSnapshotUpdate::AddManifest(ManifestFile manifest) { if (manifest.content != ManifestContent::kData) { return InvalidArgument("Cannot append delete manifest: {}", manifest.manifest_path); } - if (can_inherit_snapshot_id() && manifest.added_snapshot_id == kInvalidSnapshotId && - !manifest.first_row_id.has_value()) { + if (can_inherit_snapshot_id() && manifest.added_snapshot_id == kInvalidSnapshotId) { + if (manifest.first_row_id.has_value()) { + return InvalidArgument("Cannot append manifest with assigned first row ID: {}", + manifest.manifest_path); + } appended_manifests_summary_.AddedManifest(manifest); append_manifests_.push_back(std::move(manifest)); } else { @@ -831,8 +835,9 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( return {}; } - ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, - nullptr, nullptr, parent, io)); + ICEBERG_ASSIGN_OR_RAISE(auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, + std::move(data_filter), nullptr, parent, io)); if (deletes->empty()) { return {}; } @@ -843,24 +848,10 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( starting_seq = snap_result.value()->sequence_number; } - ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); - std::unique_ptr evaluator; - if (data_filter != nullptr) { - ICEBERG_ASSIGN_OR_RAISE(evaluator, - InclusiveMetricsEvaluator::Make(data_filter, *schema, - /*case_sensitive=*/true)); - } - for (const auto& data_file : replaced_files) { ICEBERG_ASSIGN_OR_RAISE(auto delete_files, deletes->ForDataFile(starting_seq, *data_file)); for (const auto& delete_file : delete_files) { - if (evaluator != nullptr) { - ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*delete_file)); - if (!matches) { - continue; - } - } return InvalidArgument("Cannot commit, found new delete for replaced data file: {}", data_file->file_path); } @@ -872,25 +863,12 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( const TableMetadata& metadata, int64_t starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, std::shared_ptr io) { - ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, - nullptr, nullptr, parent, io)); + ICEBERG_ASSIGN_OR_RAISE(auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, + std::move(data_filter), nullptr, parent, io)); auto referenced_delete_files = deletes->ReferencedDeleteFiles(); - ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); - std::unique_ptr evaluator; - if (data_filter != nullptr) { - ICEBERG_ASSIGN_OR_RAISE(evaluator, - InclusiveMetricsEvaluator::Make(data_filter, *schema, - /*case_sensitive=*/true)); - } - for (const auto& delete_file : referenced_delete_files) { - if (evaluator != nullptr) { - ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*delete_file)); - if (!matches) { - continue; - } - } return InvalidArgument("Found new conflicting delete files: {}", delete_file->file_path); } From f5cf69a0e374a432afe4e55d527af24266a85639 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Thu, 28 May 2026 18:37:46 +0800 Subject: [PATCH 13/16] Address delete validation review comments --- .../manifest/manifest_filter_manager.cc | 92 ++++++++++---- .../manifest/manifest_filter_manager.h | 40 +++++- .../test/manifest_filter_manager_test.cc | 80 ++++++++++++ .../test/merging_snapshot_update_test.cc | 81 ++++++++++++ src/iceberg/update/merging_snapshot_update.cc | 116 +++++++++++++++--- src/iceberg/update/merging_snapshot_update.h | 3 +- 6 files changed, 367 insertions(+), 45 deletions(-) diff --git a/src/iceberg/manifest/manifest_filter_manager.cc b/src/iceberg/manifest/manifest_filter_manager.cc index 6630b9fd4..8635abd6f 100644 --- a/src/iceberg/manifest/manifest_filter_manager.cc +++ b/src/iceberg/manifest/manifest_filter_manager.cc @@ -65,6 +65,25 @@ Result FormatPartitionPath(const PartitionSpecsById& specs_by_id, } // namespace +size_t ManifestFilterManager::DeleteFileKeyHash::operator()( + const DeleteFileKey& key) const { + size_t hash = std::hash{}(key.path); + auto combine = [&hash](const auto& value) { + size_t value_hash = value.has_value() ? std::hash{}(*value) : 0; + hash ^= value_hash + 0x9e3779b9 + (hash << 6) + (hash >> 2); + }; + combine(key.content_offset); + combine(key.content_size_in_bytes); + return hash; +} + +ManifestFilterManager::DeleteFileKey ManifestFilterManager::MakeDeleteFileKey( + const DataFile& file) { + return DeleteFileKey{.path = file.file_path, + .content_offset = file.content_offset, + .content_size_in_bytes = file.content_size_in_bytes}; +} + ManifestFilterManager::ManifestFilterManager(ManifestContent content, std::shared_ptr file_io) : manifest_content_(content), @@ -93,7 +112,7 @@ void ManifestFilterManager::DeleteFile(std::string_view path) { Status ManifestFilterManager::DeleteFile(std::shared_ptr file) { ICEBERG_PRECHECK(file != nullptr, "Cannot delete file: null"); - delete_paths_.insert(file->file_path); + delete_file_keys_.insert(MakeDeleteFileKey(*file)); return {}; } @@ -101,6 +120,11 @@ const DataFileSet& ManifestFilterManager::FilesToBeDeleted() const { return delete_files_; } +const std::vector>& ManifestFilterManager::DeletedFiles() + const { + return deleted_files_; +} + void ManifestFilterManager::DropPartition(int32_t spec_id, PartitionValues partition) { drop_partitions_.add(spec_id, std::move(partition)); } @@ -113,7 +137,7 @@ void ManifestFilterManager::FailAnyDelete() { fail_any_delete_ = true; } bool ManifestFilterManager::ContainsDeletes() const { return HasRowFilterExpression(delete_expr_) || !delete_paths_.empty() || - !drop_partitions_.empty(); + !delete_file_keys_.empty() || !drop_partitions_.empty(); } void ManifestFilterManager::DropDeleteFilesOlderThan(int64_t sequence_number) { @@ -129,9 +153,8 @@ void ManifestFilterManager::RemoveDanglingDeletesFor(const DataFileSet& deleted_ Result ManifestFilterManager::CanContainDroppedFiles(const ManifestFile&) const { // TODO(Guotao): Use the manifest descriptor to skip unrelated object-delete // manifests once object-delete partitions are tracked separately. - // Currently, DeleteFile(std::shared_ptr) degrades to a path-based delete, - // which forces scanning all manifests. - return !delete_paths_.empty() || !removed_data_file_paths_.empty(); + return !delete_paths_.empty() || !delete_file_keys_.empty() || + !removed_data_file_paths_.empty(); } Result ManifestFilterManager::CanContainDroppedPartitions( @@ -217,8 +240,9 @@ Result ManifestFilterManager::ShouldDelete(const ManifestEntry& entry, const DataFile& file = *entry.data_file; int32_t spec_id = file.partition_spec_id.value_or(manifest_spec_id); - // Path-based and partition-drop checks + // Path/object-based and partition-drop checks. if (delete_paths_.count(file.file_path) || + delete_file_keys_.count(MakeDeleteFileKey(file)) || drop_partitions_.contains(spec_id, file.partition)) { if (fail_any_delete_) { ICEBERG_ASSIGN_OR_RAISE(auto partition_path, @@ -293,8 +317,7 @@ bool ManifestFilterManager::CanTrustManifestReferences( Result ManifestFilterManager::FilterManifest( const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, const ManifestFile& manifest, bool trust_manifest_references, - const ManifestWriterFactory& writer_factory, - std::unordered_set& found_paths) { + const ManifestWriterFactory& writer_factory, FoundDeletes& found_deletes) { ICEBERG_ASSIGN_OR_RAISE( auto can_contain_deleted_files, CanContainDeletedFiles(manifest, schema, specs_by_id, trust_manifest_references)); @@ -315,7 +338,7 @@ Result ManifestFilterManager::FilterManifest( } return FilterManifestWithDeletedFiles(entries, spec_id, schema, specs_by_id, - writer_factory, found_paths); + writer_factory, found_deletes); } Result ManifestFilterManager::ManifestHasDeletedFiles( @@ -334,21 +357,30 @@ Result ManifestFilterManager::ManifestHasDeletedFiles( Result ManifestFilterManager::FilterManifestWithDeletedFiles( const std::vector& entries, int32_t manifest_spec_id, const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, - const ManifestWriterFactory& writer_factory, - std::unordered_set& found_paths) { + const ManifestWriterFactory& writer_factory, FoundDeletes& found_deletes) { ICEBERG_ASSIGN_OR_RAISE(auto writer, writer_factory(manifest_spec_id, manifest_content_)); for (const auto& entry : entries) { ICEBERG_ASSIGN_OR_RAISE(auto should_delete, ShouldDelete(entry, schema, specs_by_id, manifest_spec_id)); if (should_delete) { - if (entry.data_file && delete_paths_.count(entry.data_file->file_path)) { - found_paths.insert(entry.data_file->file_path); - } if (entry.data_file) { - // TODO(Guotao): Track duplicate deletes and avoid full DataFile copies when - // summary generation can use lighter records. - delete_files_.insert(std::make_shared(*entry.data_file)); + const auto key = MakeDeleteFileKey(*entry.data_file); + if (delete_paths_.count(entry.data_file->file_path)) { + found_deletes.paths.insert(entry.data_file->file_path); + } + if (delete_file_keys_.count(key)) { + found_deletes.files.insert(key); + } + + auto file = std::make_shared(*entry.data_file); + delete_files_.insert(file); + auto [_, inserted] = deleted_file_keys_.insert(key); + if (inserted) { + deleted_files_.push_back(std::move(file)); + } else { + ++duplicate_deletes_count_; + } } ICEBERG_RETURN_UNEXPECTED(writer->WriteDeletedEntry(entry)); } else { @@ -361,18 +393,24 @@ Result ManifestFilterManager::FilterManifestWithDeletedFiles( } Status ManifestFilterManager::ValidateRequiredDeletes( - const std::unordered_set& found_paths) const { + const FoundDeletes& found_deletes) const { if (!fail_missing_delete_paths_) { return {}; } std::string missing; for (const auto& path : delete_paths_) { - if (!found_paths.count(path)) { + if (!found_deletes.paths.count(path)) { if (!missing.empty()) missing += ", "; missing += path; } } + for (const auto& key : delete_file_keys_) { + if (!found_deletes.files.count(key)) { + if (!missing.empty()) missing += ", "; + missing += key.path; + } + } if (!missing.empty()) { return InvalidArgument("Missing delete paths: {}", missing); } @@ -391,9 +429,12 @@ Result> ManifestFilterManager::FilterManifests( const std::shared_ptr& base_snapshot, const ManifestWriterFactory& writer_factory) { delete_files_.clear(); + deleted_files_.clear(); + deleted_file_keys_.clear(); + duplicate_deletes_count_ = 0; replaced_manifests_count_ = 0; if (!base_snapshot) { - ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes({})); + ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(FoundDeletes{})); return std::vector{}; } @@ -431,11 +472,14 @@ Result> ManifestFilterManager::FilterManifests( } } - std::unordered_set found_paths; + FoundDeletes found_deletes; delete_files_.clear(); + deleted_files_.clear(); + deleted_file_keys_.clear(); + duplicate_deletes_count_ = 0; if (manifests.empty()) { replaced_manifests_count_ = 0; - ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(found_paths)); + ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(found_deletes)); return std::vector{}; } @@ -452,14 +496,14 @@ Result> ManifestFilterManager::FilterManifests( ICEBERG_ASSIGN_OR_RAISE( auto filtered_manifest, FilterManifest(schema, specs_by_id, *manifest_ptr, trust_manifest_references, - writer_factory, found_paths)); + writer_factory, found_deletes)); if (filtered_manifest.manifest_path != manifest_ptr->manifest_path) { ++replaced_manifests_count_; } filtered.push_back(std::move(filtered_manifest)); } - ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(found_paths)); + ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(found_deletes)); return filtered; } diff --git a/src/iceberg/manifest/manifest_filter_manager.h b/src/iceberg/manifest/manifest_filter_manager.h index e319623b0..e3a594e2d 100644 --- a/src/iceberg/manifest/manifest_filter_manager.h +++ b/src/iceberg/manifest/manifest_filter_manager.h @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -97,6 +98,14 @@ class ICEBERG_EXPORT ManifestFilterManager { /// operations (e.g. RowDelta) to enumerate deleted data files for follow-up cleanup. const DataFileSet& FilesToBeDeleted() const; + /// \brief Returns content-file objects deleted by the most recent + /// FilterManifests() call, deduplicated by content-file identity. + const std::vector>& DeletedFiles() const; + + /// \brief Returns how many duplicate file deletes were found in the most recent + /// FilterManifests() call. + int32_t DuplicateDeletesCount() const { return duplicate_deletes_count_; } + /// \brief Register a partition for dropping. /// /// Any manifest entry whose (spec_id, partition) pair matches will be marked DELETED. @@ -205,12 +214,31 @@ class ICEBERG_EXPORT ManifestFilterManager { bool CanTrustManifestReferences( const std::vector& manifests) const; + struct DeleteFileKey { + std::string path; + std::optional content_offset; + std::optional content_size_in_bytes; + + bool operator==(const DeleteFileKey& other) const = default; + }; + + struct DeleteFileKeyHash { + size_t operator()(const DeleteFileKey& key) const; + }; + + struct FoundDeletes { + std::unordered_set paths; + std::unordered_set files; + }; + + static DeleteFileKey MakeDeleteFileKey(const DataFile& file); + Result FilterManifest(const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, const ManifestFile& manifest, bool trust_manifest_references, const ManifestWriterFactory& writer_factory, - std::unordered_set& found_paths); + FoundDeletes& found_deletes); Result ManifestHasDeletedFiles(const std::vector& entries, const std::shared_ptr& schema, @@ -220,11 +248,9 @@ class ICEBERG_EXPORT ManifestFilterManager { Result FilterManifestWithDeletedFiles( const std::vector& entries, int32_t manifest_spec_id, const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, - const ManifestWriterFactory& writer_factory, - std::unordered_set& found_paths); + const ManifestWriterFactory& writer_factory, FoundDeletes& found_deletes); - Status ValidateRequiredDeletes( - const std::unordered_set& found_paths) const; + Status ValidateRequiredDeletes(const FoundDeletes& found_deletes) const; /// \brief Get or create a ManifestEvaluator for the given spec. Result GetManifestEvaluator(const std::shared_ptr& schema, @@ -247,11 +273,15 @@ class ICEBERG_EXPORT ManifestFilterManager { std::shared_ptr delete_expr_; std::unordered_set delete_paths_; + std::unordered_set delete_file_keys_; DataFileSet delete_files_; + std::vector> deleted_files_; + std::unordered_set deleted_file_keys_; PartitionSet drop_partitions_; bool fail_missing_delete_paths_{false}; bool fail_any_delete_{false}; bool case_sensitive_{true}; + int32_t duplicate_deletes_count_{0}; int32_t replaced_manifests_count_{0}; diff --git a/src/iceberg/test/manifest_filter_manager_test.cc b/src/iceberg/test/manifest_filter_manager_test.cc index a49445ba3..2f2840636 100644 --- a/src/iceberg/test/manifest_filter_manager_test.cc +++ b/src/iceberg/test/manifest_filter_manager_test.cc @@ -562,4 +562,84 @@ TEST_F(ManifestFilterManagerTest, RemoveDanglingDeletesForFiltersDanglingDV) { EXPECT_EQ(entries[0].status, ManifestStatus::kDeleted); } +TEST_F(ManifestFilterManagerTest, DeleteFileObjectMatchesDeletionVectorByContent) { + auto metadata = *table_->metadata(); + metadata.format_version = 3; + auto factory = MakeWriterFactory(metadata); + + auto make_dv = [&](int64_t offset) { + auto f = std::make_shared(); + f->content = DataFile::Content::kPositionDeletes; + f->file_path = table_location_ + "/delete/dv.puffin"; + f->file_format = FileFormatType::kPuffin; + f->referenced_data_file = + std::format("{}/data/referenced-{}.parquet", table_location_, offset); + f->content_offset = offset; + f->content_size_in_bytes = 10; + f->partition = PartitionValues(std::vector{Literal::Long(1L)}); + f->file_size_in_bytes = 256; + f->record_count = 5; + f->partition_spec_id = spec_->spec_id(); + return f; + }; + auto dv0 = make_dv(0); + auto dv1 = make_dv(10); + + auto manifest_path = std::format("{}/metadata/dv-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto manifest, + WriteDeleteManifest({{dv0, 3L}, {dv1, 3L}}, file_io_, metadata, manifest_path)); + + ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); + EXPECT_THAT(mgr.DeleteFile(dv0), IsOk()); + + std::vector manifests{&manifest}; + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata.Schema()); + ICEBERG_UNWRAP_OR_FAIL( + auto result, mgr.FilterManifests(schema, SpecsById(metadata), manifests, factory)); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, metadata)); + ASSERT_EQ(entries.size(), 2U); + for (const auto& entry : entries) { + ASSERT_NE(entry.data_file, nullptr); + if (entry.data_file->content_offset == 0) { + EXPECT_EQ(entry.status, ManifestStatus::kDeleted); + } else { + EXPECT_EQ(entry.status, ManifestStatus::kExisting); + } + } +} + +TEST_F(ManifestFilterManagerTest, DuplicateDeletesCountRepeatedDeletedFiles) { + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + auto del_file = std::make_shared(); + del_file->content = DataFile::Content::kPositionDeletes; + del_file->file_path = table_location_ + "/delete/duplicate.parquet"; + del_file->file_format = FileFormatType::kParquet; + del_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + del_file->file_size_in_bytes = 512; + del_file->record_count = 10; + del_file->partition_spec_id = spec_->spec_id(); + + auto manifest_path = std::format("{}/metadata/dup-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL(auto manifest, + WriteDeleteManifest({{del_file, 3L}, {del_file, 3L}}, file_io_, + *metadata, manifest_path)); + + ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); + EXPECT_THAT(mgr.DeleteFile(del_file), IsOk()); + + std::vector manifests{&manifest}; + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + ICEBERG_UNWRAP_OR_FAIL( + auto result, mgr.FilterManifests(schema, SpecsById(*metadata), manifests, factory)); + + EXPECT_EQ(mgr.DeletedFiles().size(), 1U); + EXPECT_EQ(mgr.DuplicateDeletesCount(), 1); +} + } // namespace iceberg diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 2b386f502..9ff9a8746 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -139,6 +139,15 @@ class TestMergeAppend : public MergingSnapshotUpdate { return MergingSnapshotUpdate::ValidateDeletedDataFiles( metadata, starting_snapshot_id, partition_set, parent, std::move(io)); } + static Status ValidateAddedDVsForTest(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr conflict_filter, + const std::shared_ptr& parent, + std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateAddedDVs(metadata, starting_snapshot_id, + std::move(conflict_filter), parent, + std::move(io)); + } bool HasDataFiles() const { return AddsDataFiles(); } bool HasDeleteFiles() const { return AddsDeleteFiles(); } @@ -268,6 +277,27 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return writer->ToManifestFile(); } + Result WriteDeleteManifest( + const TableMetadata& metadata, const std::string& path, + const std::vector>& files, int64_t sequence_number) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata.PartitionSpecById(spec_->spec_id())); + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + ManifestWriter::MakeWriter(metadata.format_version, /*snapshot_id=*/1L, path, + file_io_, spec, schema, ManifestContent::kDeletes)); + for (const auto& f : files) { + ManifestEntry entry; + entry.status = ManifestStatus::kAdded; + entry.snapshot_id = 1L; + entry.sequence_number = sequence_number; + entry.data_file = f; + ICEBERG_RETURN_UNEXPECTED(writer->WriteAddedEntry(entry)); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + return writer->ToManifestFile(); + } + Result> MakeSyntheticSnapshot( std::string operation, int64_t snapshot_id, std::optional parent_snapshot_id, int64_t sequence_number, @@ -445,6 +475,20 @@ TEST_F(MergingSnapshotUpdateTest, CommitDeleteFileSummaryHasAddedDeleteFiles) { EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kRemovedDeleteFiles), 0); } +TEST_F(MergingSnapshotUpdateTest, CommitDeduplicatesStagedDeleteFiles) { + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDeleteFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedPosDeleteFiles), "1"); +} + TEST_F(MergingSnapshotUpdateTest, AddDeleteFileWithExplicitSequenceWritesSequenceNumber) { auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); @@ -1072,6 +1116,43 @@ TEST_F(MergingSnapshotUpdateTest, IsError(ErrorKind::kInvalidArgument)); } +TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->format_version = 3; + + auto dv_file = MakeDeleteFile("/delete/dv_a.puffin", 1L); + dv_file->file_format = FileFormatType::kPuffin; + dv_file->referenced_data_file = file_a_->file_path; + dv_file->content_offset = 0; + dv_file->content_size_in_bytes = 10; + + constexpr int64_t kSecondSnapshotId = 123456; + auto manifest_path = table_location_ + "/metadata/dv-conflict.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto manifest, + WriteDeleteManifest(*metadata, manifest_path, {dv_file}, + first_snapshot->sequence_number + 1)); + manifest.added_snapshot_id = kSecondSnapshotId; + manifest.sequence_number = first_snapshot->sequence_number + 1; + manifest.min_sequence_number = first_snapshot->sequence_number + 1; + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, kSecondSnapshotId, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, {manifest})); + + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + EXPECT_THAT(TestMergeAppend::ValidateAddedDVsForTest( + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), + second_snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index 5cabe774e..76ad7a249 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "iceberg/constants.h" @@ -59,6 +60,33 @@ struct ValidationHistoryResult { std::unordered_set snapshot_ids; }; +struct DeleteFileObjectKey { + std::string path; + std::optional content_offset; + std::optional content_size_in_bytes; + + bool operator==(const DeleteFileObjectKey& other) const = default; +}; + +struct DeleteFileObjectKeyHash { + size_t operator()(const DeleteFileObjectKey& key) const { + size_t hash = std::hash{}(key.path); + auto combine = [&hash](const auto& value) { + size_t value_hash = value.has_value() ? std::hash{}(*value) : 0; + hash ^= value_hash + 0x9e3779b9 + (hash << 6) + (hash >> 2); + }; + combine(key.content_offset); + combine(key.content_size_in_bytes); + return hash; + } +}; + +DeleteFileObjectKey MakeDeleteFileObjectKey(const DataFile& file) { + return DeleteFileObjectKey{.path = file.file_path, + .content_offset = file.content_offset, + .content_size_in_bytes = file.content_size_in_bytes}; +} + Result>> ValidationAncestorsBetween( const TableMetadata& metadata, int64_t latest_snapshot_id, int64_t starting_snapshot_id) { @@ -248,7 +276,7 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, } ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(file->partition_spec_id.value())); - ICEBERG_RETURN_UNEXPECTED(added_delete_files_summary_.AddedFile(*spec, *file)); + (void)spec; has_new_delete_files_ = true; new_delete_files_.push_back(PendingDeleteFile{ .file = std::move(file), .data_sequence_number = std::move(data_sequence_number)}); @@ -430,6 +458,41 @@ Result> MergingSnapshotUpdate::WriteNewDataManifests() return result; } +Result> +MergingSnapshotUpdate::NormalizeNewDeleteFiles() const { + std::vector result; + result.reserve(new_delete_files_.size()); + + std::unordered_set seen_delete_files; + std::unordered_map dv_by_referenced_data_file; + + for (const auto& pending_file : new_delete_files_) { + const auto& file = pending_file.file; + ICEBERG_PRECHECK(file != nullptr, "Cannot add a null delete file"); + + auto key = MakeDeleteFileObjectKey(*file); + if (!seen_delete_files.insert(key).second) { + continue; + } + + if (ContentFileUtil::IsDV(*file)) { + ICEBERG_PRECHECK(file->referenced_data_file.has_value(), + "DV must have a referenced data file: {}", file->file_path); + auto [it, inserted] = + dv_by_referenced_data_file.emplace(*file->referenced_data_file, key); + if (!inserted && it->second != key) { + return NotImplemented( + "Cannot merge multiple deletion vectors for referenced data file: {}", + *file->referenced_data_file); + } + } + + result.push_back(pending_file); + } + + return result; +} + Result> MergingSnapshotUpdate::WriteNewDeleteManifests() { // If new files were staged after the cache was populated (commit retry), invalidate. if (has_new_delete_files_ && cached_new_delete_manifests_.has_value()) { @@ -481,6 +544,16 @@ Result> MergingSnapshotUpdate::Apply( for (const auto& pending_file : new_delete_files_) { ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(*pending_file.file)); } + ICEBERG_ASSIGN_OR_RAISE(auto normalized_delete_files, NormalizeNewDeleteFiles()); + new_delete_files_ = std::move(normalized_delete_files); + + added_delete_files_summary_.Clear(); + for (const auto& pending_file : new_delete_files_) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata_to_update.PartitionSpecById( + *pending_file.file->partition_spec_id)); + ICEBERG_RETURN_UNEXPECTED( + added_delete_files_summary_.AddedFile(*spec, *pending_file.file)); + } // Rebuild summary from stable sub-builders so that commit retries don't double-count. summary_builder().Clear(); @@ -498,7 +571,7 @@ Result> MergingSnapshotUpdate::Apply( snapshot, tracked_factory)); // Track deleted data files in the summary builder. - for (const auto& file : data_filter_manager_.FilesToBeDeleted()) { + for (const auto& file : data_filter_manager_.DeletedFiles()) { if (!file->partition_spec_id.has_value()) { continue; } @@ -506,6 +579,8 @@ Result> MergingSnapshotUpdate::Apply( auto spec, metadata_to_update.PartitionSpecById(*file->partition_spec_id)); ICEBERG_RETURN_UNEXPECTED(summary_builder().DeletedFile(*spec, *file)); } + summary_builder().IncrementDuplicateDeletes( + data_filter_manager_.DuplicateDeletesCount()); // Step 2: Compute min data sequence number; set up delete filter cleanup. // Use last_sequence_number as the initial value so that an empty filtered list @@ -531,7 +606,7 @@ Result> MergingSnapshotUpdate::Apply( snapshot, tracked_factory)); // Track deleted delete files in the summary builder. - for (const auto& file : delete_filter_manager_.FilesToBeDeleted()) { + for (const auto& file : delete_filter_manager_.DeletedFiles()) { if (!file->partition_spec_id.has_value()) { continue; } @@ -539,6 +614,8 @@ Result> MergingSnapshotUpdate::Apply( auto spec, metadata_to_update.PartitionSpecById(*file->partition_spec_id)); ICEBERG_RETURN_UNEXPECTED(summary_builder().DeletedFile(*spec, *file)); } + summary_builder().IncrementDuplicateDeletes( + delete_filter_manager_.DuplicateDeletesCount()); // Drop manifests with no live files — they carry no data and should not be merged // into the new snapshot. Manifests written by the current snapshot are always kept @@ -879,15 +956,12 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( const TableMetadata& metadata, int64_t starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io) { - ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, - nullptr, nullptr, parent, io)); + ICEBERG_ASSIGN_OR_RAISE( + auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, + std::make_shared(partition_set), parent, io)); auto referenced_delete_files = deletes->ReferencedDeleteFiles(); for (const auto& delete_file : referenced_delete_files) { - if (!delete_file->partition_spec_id.has_value() || - !partition_set.contains(delete_file->partition_spec_id.value(), - delete_file->partition)) { - continue; - } return InvalidArgument( "Found new conflicting delete files in validated partitions: {}", delete_file->file_path); @@ -994,11 +1068,23 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles } Status MergingSnapshotUpdate::ValidateAddedDVs( - const TableMetadata& /*metadata*/, int64_t /*starting_snapshot_id*/, - std::shared_ptr /*conflict_filter*/, - const std::shared_ptr& /*parent*/, std::shared_ptr /*io*/) { - return NotImplemented( - "ValidateAddedDVs is not yet supported (deletion vectors require format v3)"); + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr conflict_filter, const std::shared_ptr& parent, + std::shared_ptr io) { + if (parent == nullptr || metadata.format_version < 3) { + return {}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, + std::move(conflict_filter), nullptr, parent, io)); + for (const auto& delete_file : deletes->ReferencedDeleteFiles()) { + if (ContentFileUtil::IsDV(*delete_file)) { + return InvalidArgument("Cannot commit, found new deletion vector: {}", + ContentFileUtil::DVDesc(*delete_file)); + } + } + return {}; } } // namespace iceberg diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h index 54e259c23..61821bfd9 100644 --- a/src/iceberg/update/merging_snapshot_update.h +++ b/src/iceberg/update/merging_snapshot_update.h @@ -279,7 +279,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// added a deletion vector that conflicts with DVs being written. /// - /// \note Deletion vectors (format v3) are not yet supported; returns NotImplemented. static Status ValidateAddedDVs(const TableMetadata& metadata, int64_t starting_snapshot_id, std::shared_ptr conflict_filter, @@ -303,6 +302,8 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { Status AddDeleteFile(std::shared_ptr file, std::optional data_sequence_number); + Result> NormalizeNewDeleteFiles() const; + /// \brief Write new data manifests for staged data files; caches the result. Result> WriteNewDataManifests(); From 6ebf1d4811631b4ac4f8935d47ba72f7400077ea Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Fri, 29 May 2026 14:24:58 +0800 Subject: [PATCH 14/16] Address merging snapshot review comments --- .../manifest/manifest_filter_manager.h | 4 +- .../test/merging_snapshot_update_test.cc | 101 +++++++++++++----- src/iceberg/update/merging_snapshot_update.cc | 68 +++++++++--- src/iceberg/update/merging_snapshot_update.h | 16 ++- 4 files changed, 144 insertions(+), 45 deletions(-) diff --git a/src/iceberg/manifest/manifest_filter_manager.h b/src/iceberg/manifest/manifest_filter_manager.h index e3a594e2d..c85cc8717 100644 --- a/src/iceberg/manifest/manifest_filter_manager.h +++ b/src/iceberg/manifest/manifest_filter_manager.h @@ -133,7 +133,7 @@ class ICEBERG_EXPORT ManifestFilterManager { /// \brief Set the minimum data sequence number for delete files to retain. /// /// Only valid for ManifestContent::kDeletes managers. Delete entries whose - /// data_sequence_number is positive and less than \p sequence_number will be + /// data_sequence_number is positive and less than sequence_number will be /// marked DELETED. This continuously removes delete files that cannot match /// any remaining data rows (i.e. all data written before that sequence number /// has itself been deleted). @@ -146,7 +146,7 @@ class ICEBERG_EXPORT ManifestFilterManager { /// can be cleaned up. /// /// Only valid for ManifestContent::kDeletes managers. For each DV whose - /// referenced_data_file path appears in \p deleted_files, the DV entry is + /// referenced_data_file path appears in deleted_files, the DV entry is /// marked DELETED because the data file it targets no longer exists. /// /// \param deleted_files set of data files that have been marked for deletion diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 9ff9a8746..0722b81ff 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -139,14 +139,14 @@ class TestMergeAppend : public MergingSnapshotUpdate { return MergingSnapshotUpdate::ValidateDeletedDataFiles( metadata, starting_snapshot_id, partition_set, parent, std::move(io)); } - static Status ValidateAddedDVsForTest(const TableMetadata& metadata, - int64_t starting_snapshot_id, - std::shared_ptr conflict_filter, - const std::shared_ptr& parent, - std::shared_ptr io) { - return MergingSnapshotUpdate::ValidateAddedDVs(metadata, starting_snapshot_id, - std::move(conflict_filter), parent, - std::move(io)); + static Status ValidateAddedDVsForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr conflict_filter, + const std::unordered_set& referenced_data_files, + const std::shared_ptr& parent, std::shared_ptr io) { + return MergingSnapshotUpdate::ValidateAddedDVs( + metadata, starting_snapshot_id, std::move(conflict_filter), referenced_data_files, + parent, std::move(io)); } bool HasDataFiles() const { return AddsDataFiles(); } @@ -371,8 +371,8 @@ TEST_F(MergingSnapshotUpdateTest, CommitNewDataFile) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_EQ(snapshot->summary.at("added-data-files"), "1"); - EXPECT_EQ(snapshot->summary.at("added-records"), "100"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedRecords), "100"); } TEST_F(MergingSnapshotUpdateTest, CommitMultipleDataFiles) { @@ -383,8 +383,8 @@ TEST_F(MergingSnapshotUpdateTest, CommitMultipleDataFiles) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_EQ(snapshot->summary.at("added-data-files"), "2"); - EXPECT_EQ(snapshot->summary.at("added-records"), "200"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "2"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedRecords), "200"); } TEST_F(MergingSnapshotUpdateTest, CommitDataFileAndDeleteFile) { @@ -398,7 +398,7 @@ TEST_F(MergingSnapshotUpdateTest, CommitDataFileAndDeleteFile) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); // Data file summary - EXPECT_EQ(snapshot->summary.at("added-data-files"), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); } TEST_F(MergingSnapshotUpdateTest, CommitPreservesExistingManifests) { @@ -413,7 +413,7 @@ TEST_F(MergingSnapshotUpdateTest, CommitPreservesExistingManifests) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); // Both data files should be visible — 1 existing + 1 new - EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "2"); } TEST_F(MergingSnapshotUpdateTest, CommitDeletesDataFile) { @@ -426,8 +426,8 @@ TEST_F(MergingSnapshotUpdateTest, CommitDeletesDataFile) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_EQ(snapshot->summary.at("total-data-files"), "0"); - EXPECT_EQ(snapshot->summary.at("deleted-data-files"), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "0"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kDeletedDataFiles), "1"); } TEST_F(MergingSnapshotUpdateTest, SetNewDataFilesDataSequenceNumber) { @@ -438,7 +438,19 @@ TEST_F(MergingSnapshotUpdateTest, SetNewDataFilesDataSequenceNumber) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - EXPECT_EQ(snapshot->summary.at("added-data-files"), "1"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); +} + +TEST_F(MergingSnapshotUpdateTest, CustomSummaryPropertySurvivesApplyRebuild) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + op->Set("custom-prop", "custom-value"); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("custom-prop"), "custom-value"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); } // ------------------------------------------------------------------------- @@ -694,8 +706,8 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestEmptyTable) { ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); ASSERT_EQ(data_manifests.size(), 1); - EXPECT_EQ(snapshot->summary.at("added-data-files"), "2"); - EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "2"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "2"); } TEST_F(MergingSnapshotUpdateTest, AppendManifestWithDataFiles) { @@ -714,7 +726,7 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestWithDataFiles) { ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); // Written manifest (file_b_) + appended manifest (file_a_, file_b_) EXPECT_EQ(data_manifests.size(), 2); - EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "3"); } // ------------------------------------------------------------------------- @@ -742,7 +754,7 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestMergeWithMinCountOne) { ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); // Both manifests merged into one. EXPECT_EQ(data_manifests.size(), 1); - EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "3"); } TEST_F(MergingSnapshotUpdateTest, AppendManifestDoNotMergeMinCount) { @@ -772,7 +784,7 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestDoNotMergeMinCount) { ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); // Below min-count-to-merge threshold — all 3 pass through unchanged. EXPECT_EQ(data_manifests.size(), 3); - EXPECT_EQ(snapshot->summary.at("added-data-files"), "3"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "3"); } // ------------------------------------------------------------------------- @@ -799,7 +811,7 @@ TEST_F(MergingSnapshotUpdateTest, ManifestMergeMergesIntoOne) { SnapshotCache snapshot_cache(snapshot.get()); ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); EXPECT_EQ(data_manifests.size(), 1); - EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "2"); EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kManifestsReplaced), "1"); } @@ -816,7 +828,7 @@ TEST_F(MergingSnapshotUpdateTest, ManifestMergeDoesNotMergeWhenBelowMinCount) { SnapshotCache snapshot_cache(snapshot.get()); ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); EXPECT_EQ(data_manifests.size(), 2); - EXPECT_EQ(snapshot->summary.at("total-data-files"), "2"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "2"); } TEST_F(MergingSnapshotUpdateTest, ManifestMergeDoesNotMergeWhenSizeTargetTooSmall) { @@ -1147,12 +1159,51 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsDetectsConflict) { metadata->current_snapshot_id = second_snapshot->snapshot_id; metadata->last_sequence_number = second_snapshot->sequence_number; + const std::unordered_set referenced_data_files{file_a_->file_path}; EXPECT_THAT(TestMergeAppend::ValidateAddedDVsForTest( *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), - second_snapshot, file_io_), + referenced_data_files, second_snapshot, file_io_), IsError(ErrorKind::kInvalidArgument)); } +TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsIgnoresUnrelatedDVs) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->format_version = 3; + + auto dv_file = MakeDeleteFile("/delete/dv_a.puffin", 1L); + dv_file->file_format = FileFormatType::kPuffin; + dv_file->referenced_data_file = file_a_->file_path; + dv_file->content_offset = 0; + dv_file->content_size_in_bytes = 10; + + constexpr int64_t kSecondSnapshotId = 123456; + auto manifest_path = table_location_ + "/metadata/dv-unrelated.avro"; + ICEBERG_UNWRAP_OR_FAIL(auto manifest, + WriteDeleteManifest(*metadata, manifest_path, {dv_file}, + first_snapshot->sequence_number + 1)); + manifest.added_snapshot_id = kSecondSnapshotId; + manifest.sequence_number = first_snapshot->sequence_number + 1; + manifest.min_sequence_number = first_snapshot->sequence_number + 1; + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, kSecondSnapshotId, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, {manifest})); + + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + const std::unordered_set referenced_data_files{file_b_->file_path}; + EXPECT_THAT(TestMergeAppend::ValidateAddedDVsForTest( + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), + referenced_data_files, second_snapshot, file_io_), + IsOk()); +} + TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index 76ad7a249..c2eb0f3d3 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -274,9 +274,7 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, if (!file->partition_spec_id.has_value()) { return InvalidArgument("Delete file must have a partition spec ID"); } - ICEBERG_ASSIGN_OR_RAISE(auto spec, - base().PartitionSpecById(file->partition_spec_id.value())); - (void)spec; + ICEBERG_RETURN_UNEXPECTED(base().PartitionSpecById(file->partition_spec_id.value())); has_new_delete_files_ = true; new_delete_files_.push_back(PendingDeleteFile{ .file = std::move(file), .data_sequence_number = std::move(data_sequence_number)}); @@ -336,6 +334,7 @@ void MergingSnapshotUpdate::CaseSensitive(bool case_sensitive) { } void MergingSnapshotUpdate::Set(const std::string& property, const std::string& value) { + custom_summary_properties_[property] = value; summary_builder().Set(property, value); } @@ -482,7 +481,8 @@ MergingSnapshotUpdate::NormalizeNewDeleteFiles() const { dv_by_referenced_data_file.emplace(*file->referenced_data_file, key); if (!inserted && it->second != key) { return NotImplemented( - "Cannot merge multiple deletion vectors for referenced data file: {}", + "Merging multiple deletion vectors is not supported yet for referenced " + "data file: {}", *file->referenced_data_file); } } @@ -560,6 +560,9 @@ Result> MergingSnapshotUpdate::Apply( summary_builder().Merge(added_data_files_summary_); summary_builder().Merge(added_delete_files_summary_); summary_builder().Merge(appended_manifests_summary_); + for (const auto& [property, value] : custom_summary_properties_) { + summary_builder().Set(property, value); + } ICEBERG_ASSIGN_OR_RAISE(auto target_schema, SnapshotUtil::SchemaFor(metadata_to_update, target_branch())); @@ -1069,22 +1072,61 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles Status MergingSnapshotUpdate::ValidateAddedDVs( const TableMetadata& metadata, int64_t starting_snapshot_id, - std::shared_ptr conflict_filter, const std::shared_ptr& parent, - std::shared_ptr io) { - if (parent == nullptr || metadata.format_version < 3) { + std::shared_ptr conflict_filter, + const std::unordered_set& referenced_data_files, + const std::shared_ptr& parent, std::shared_ptr io) { + if (parent == nullptr || referenced_data_files.empty() || metadata.format_version < 3) { return {}; } ICEBERG_ASSIGN_OR_RAISE( - auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, - std::move(conflict_filter), nullptr, parent, io)); - for (const auto& delete_file : deletes->ReferencedDeleteFiles()) { - if (ContentFileUtil::IsDV(*delete_file)) { - return InvalidArgument("Cannot commit, found new deletion vector: {}", - ContentFileUtil::DVDesc(*delete_file)); + auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + {DataOperation::kOverwrite, DataOperation::kDelete, + DataOperation::kReplace}, + ManifestContent::kDeletes, io)); + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + + for (const auto& manifest : history.manifests) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, + metadata.PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto reader, + ManifestReader::Make(manifest, io, schema, spec)); + if (conflict_filter != nullptr) { + reader->FilterRows(conflict_filter); + } + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); + + for (const auto& entry : entries) { + if (entry.data_file == nullptr || !ContentFileUtil::IsDV(*entry.data_file) || + !entry.data_file->referenced_data_file.has_value()) { + continue; + } + if (referenced_data_files.contains(*entry.data_file->referenced_data_file)) { + return InvalidArgument("Cannot commit, found new deletion vector: {}", + ContentFileUtil::DVDesc(*entry.data_file)); + } } } return {}; } +Status MergingSnapshotUpdate::ValidateAddedDVs( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr conflict_filter, const std::shared_ptr& parent, + std::shared_ptr io) const { + std::unordered_set referenced_data_files; + for (const auto& pending_file : new_delete_files_) { + if (pending_file.file == nullptr || !ContentFileUtil::IsDV(*pending_file.file) || + !pending_file.file->referenced_data_file.has_value()) { + continue; + } + referenced_data_files.insert(*pending_file.file->referenced_data_file); + } + if (referenced_data_files.empty()) { + return {}; + } + return ValidateAddedDVs(metadata, starting_snapshot_id, std::move(conflict_filter), + referenced_data_files, parent, std::move(io)); +} + } // namespace iceberg diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h index 61821bfd9..bcc7c0e82 100644 --- a/src/iceberg/update/merging_snapshot_update.h +++ b/src/iceberg/update/merging_snapshot_update.h @@ -279,11 +279,11 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] /// added a deletion vector that conflicts with DVs being written. /// - static Status ValidateAddedDVs(const TableMetadata& metadata, - int64_t starting_snapshot_id, - std::shared_ptr conflict_filter, - const std::shared_ptr& parent, - std::shared_ptr io); + static Status ValidateAddedDVs( + const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr conflict_filter, + const std::unordered_set& referenced_data_files, + const std::shared_ptr& parent, std::shared_ptr io); private: struct PendingDeleteFile { @@ -302,6 +302,11 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { Status AddDeleteFile(std::shared_ptr file, std::optional data_sequence_number); + Status ValidateAddedDVs(const TableMetadata& metadata, int64_t starting_snapshot_id, + std::shared_ptr conflict_filter, + const std::shared_ptr& parent, + std::shared_ptr io) const; + Result> NormalizeNewDeleteFiles() const; /// \brief Write new data manifests for staged data files; caches the result. @@ -320,6 +325,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { SnapshotSummaryBuilder added_data_files_summary_; SnapshotSummaryBuilder added_delete_files_summary_; SnapshotSummaryBuilder appended_manifests_summary_; + std::unordered_map custom_summary_properties_; ManifestFilterManager data_filter_manager_; ManifestFilterManager delete_filter_manager_; From 1001097c67d3ffadd123c3678d71851d04c6a8b2 Mon Sep 17 00:00:00 2001 From: Guotao Yu Date: Fri, 29 May 2026 16:34:31 +0800 Subject: [PATCH 15/16] Align merging snapshot validation with Java --- .../test/merging_snapshot_update_test.cc | 184 ++++++++++++++++-- src/iceberg/test/snapshot_util_test.cc | 30 ++- src/iceberg/update/merging_snapshot_update.cc | 119 +++++++---- src/iceberg/update/merging_snapshot_update.h | 99 +++++----- src/iceberg/update/snapshot_update.cc | 2 +- .../update/update_partition_statistics.h | 5 +- src/iceberg/util/snapshot_util.cc | 4 +- src/iceberg/util/snapshot_util_internal.h | 12 +- 8 files changed, 337 insertions(+), 118 deletions(-) diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 0722b81ff..76b9ac32c 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -85,7 +85,7 @@ class TestMergeAppend : public MergingSnapshotUpdate { int64_t GeneratedSnapshotId() { return SnapshotId(); } void SetDataSeqNumber(int64_t seq) { SetNewDataFilesDataSequenceNumber(seq); } static Status ValidateAddedDataFilesForTest(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, const std::shared_ptr& parent, std::shared_ptr io) { return MergingSnapshotUpdate::ValidateAddedDataFiles(metadata, starting_snapshot_id, @@ -234,6 +234,10 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return TestOverwriteUpdate::Make(TableName(), table_); } + void SetTableFormatVersion(int8_t format_version) { + table_->metadata()->format_version = format_version; + } + // Commit file_a_ with FastAppend and refresh the table. void CommitFileA() { ICEBERG_UNWRAP_OR_FAIL(auto fa, table_->NewFastAppend()); @@ -277,19 +281,37 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return writer->ToManifestFile(); } + Result WriteDataManifest( + const TableMetadata& metadata, const std::string& path, + const std::vector>& files, int64_t snapshot_id, + int64_t sequence_number) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata.PartitionSpecById(spec_->spec_id())); + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + ManifestWriter::MakeWriter(metadata.format_version, snapshot_id, path, file_io_, + spec, schema, ManifestContent::kData)); + for (const auto& f : files) { + ICEBERG_RETURN_UNEXPECTED(writer->WriteAddedEntry(f, sequence_number)); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + return writer->ToManifestFile(); + } + Result WriteDeleteManifest( const TableMetadata& metadata, const std::string& path, - const std::vector>& files, int64_t sequence_number) { + const std::vector>& files, int64_t snapshot_id, + int64_t sequence_number) { ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata.PartitionSpecById(spec_->spec_id())); ICEBERG_ASSIGN_OR_RAISE( auto writer, - ManifestWriter::MakeWriter(metadata.format_version, /*snapshot_id=*/1L, path, - file_io_, spec, schema, ManifestContent::kDeletes)); + ManifestWriter::MakeWriter(metadata.format_version, snapshot_id, path, file_io_, + spec, schema, ManifestContent::kDeletes)); for (const auto& f : files) { ManifestEntry entry; entry.status = ManifestStatus::kAdded; - entry.snapshot_id = 1L; + entry.snapshot_id = snapshot_id; entry.sequence_number = sequence_number; entry.data_file = f; ICEBERG_RETURN_UNEXPECTED(writer->WriteAddedEntry(entry)); @@ -623,6 +645,39 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV2AllowsEqualityDelete) { EXPECT_THAT(op->AddDelete(eq_del), IsOk()); } +TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV3RejectsNonDVPositionDelete) { + SetTableFormatVersion(3); + + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV3AllowsDeletionVector) { + SetTableFormatVersion(3); + + auto del_file = MakeDeleteFile("/delete/dv_a.puffin", 1L); + del_file->file_format = FileFormatType::kPuffin; + del_file->referenced_data_file = file_a_->file_path; + del_file->content_offset = 0; + del_file->content_size_in_bytes = 10; + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); +} + +TEST_F(MergingSnapshotUpdateTest, ApplyRejectsV2StagedPositionDeleteAfterV3Upgrade) { + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->format_version = 3; + EXPECT_THAT(op->Apply(*metadata, nullptr), IsError(ErrorKind::kInvalidArgument)); +} + // ------------------------------------------------------------------------- // AddManifest — invalid manifest rejection // ------------------------------------------------------------------------- @@ -970,6 +1025,42 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesFailsForTruncatedHistory IsError(ErrorKind::kInvalidArgument)); } +TEST_F(MergingSnapshotUpdateTest, + ValidateAddedDataFilesWithNoStartingSnapshotFailsForTruncatedHistory) { + auto metadata = std::make_shared(); + metadata->format_version = 2; + metadata->location = table_location_; + metadata->current_schema_id = 0; + metadata->schemas.push_back(schema_); + + auto snapshot = std::make_shared(Snapshot{ + .snapshot_id = 2, + .parent_snapshot_id = 1, + .sequence_number = 2, + .timestamp_ms = TimePointMs{}, + .manifest_list = "", + .summary = {}, + .schema_id = 0, + }); + metadata->snapshots = {snapshot}; + + EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest(*metadata, std::nullopt, + snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithNoStartingSnapshotChecksAll) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + + EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest( + *table_->metadata(), std::nullopt, snapshot, file_io_), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest( + *table_->metadata(), snapshot->snapshot_id, snapshot, file_io_), + IsOk()); +} + TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithPartitionSetDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); @@ -988,6 +1079,36 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithPartitionSetDetectsC IsError(ErrorKind::kInvalidArgument)); } +TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesIgnoresOldEntrySnapshotId) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto metadata = std::make_shared(*table_->metadata()); + + constexpr int64_t kSecondSnapshotId = 123456; + auto manifest_path = table_location_ + "/metadata/old-entry-data.avro"; + ICEBERG_UNWRAP_OR_FAIL( + auto manifest, + WriteDataManifest(*metadata, manifest_path, {file_b_}, first_snapshot->snapshot_id, + first_snapshot->sequence_number)); + manifest.added_snapshot_id = kSecondSnapshotId; + manifest.sequence_number = first_snapshot->sequence_number + 1; + manifest.min_sequence_number = first_snapshot->sequence_number; + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kAppend, kSecondSnapshotId, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, {manifest})); + + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest( + *metadata, first_snapshot->snapshot_id, second_snapshot, file_io_), + IsOk()); +} + TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesWithFilterDetectsConflict) { CommitFileA(); @@ -1143,9 +1264,10 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsDetectsConflict) { constexpr int64_t kSecondSnapshotId = 123456; auto manifest_path = table_location_ + "/metadata/dv-conflict.avro"; - ICEBERG_UNWRAP_OR_FAIL(auto manifest, - WriteDeleteManifest(*metadata, manifest_path, {dv_file}, - first_snapshot->sequence_number + 1)); + ICEBERG_UNWRAP_OR_FAIL( + auto manifest, + WriteDeleteManifest(*metadata, manifest_path, {dv_file}, kSecondSnapshotId, + first_snapshot->sequence_number + 1)); manifest.added_snapshot_id = kSecondSnapshotId; manifest.sequence_number = first_snapshot->sequence_number + 1; manifest.min_sequence_number = first_snapshot->sequence_number + 1; @@ -1181,9 +1303,10 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsIgnoresUnrelatedDVs) { constexpr int64_t kSecondSnapshotId = 123456; auto manifest_path = table_location_ + "/metadata/dv-unrelated.avro"; - ICEBERG_UNWRAP_OR_FAIL(auto manifest, - WriteDeleteManifest(*metadata, manifest_path, {dv_file}, - first_snapshot->sequence_number + 1)); + ICEBERG_UNWRAP_OR_FAIL( + auto manifest, + WriteDeleteManifest(*metadata, manifest_path, {dv_file}, kSecondSnapshotId, + first_snapshot->sequence_number + 1)); manifest.added_snapshot_id = kSecondSnapshotId; manifest.sequence_number = first_snapshot->sequence_number + 1; manifest.min_sequence_number = first_snapshot->sequence_number + 1; @@ -1204,6 +1327,45 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsIgnoresUnrelatedDVs) { IsOk()); } +TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsIgnoresOldEntrySnapshotId) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->format_version = 3; + + auto dv_file = MakeDeleteFile("/delete/dv_a.puffin", 1L); + dv_file->file_format = FileFormatType::kPuffin; + dv_file->referenced_data_file = file_a_->file_path; + dv_file->content_offset = 0; + dv_file->content_size_in_bytes = 10; + + constexpr int64_t kSecondSnapshotId = 123456; + auto manifest_path = table_location_ + "/metadata/old-entry-dv.avro"; + ICEBERG_UNWRAP_OR_FAIL( + auto manifest, + WriteDeleteManifest(*metadata, manifest_path, {dv_file}, + first_snapshot->snapshot_id, first_snapshot->sequence_number)); + manifest.added_snapshot_id = kSecondSnapshotId; + manifest.sequence_number = first_snapshot->sequence_number + 1; + manifest.min_sequence_number = first_snapshot->sequence_number; + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, kSecondSnapshotId, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, {manifest})); + + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + const std::unordered_set referenced_data_files{file_a_->file_path}; + EXPECT_THAT(TestMergeAppend::ValidateAddedDVsForTest( + *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), + referenced_data_files, second_snapshot, file_io_), + IsOk()); +} + TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsConflict) { CommitFileA(); ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); diff --git a/src/iceberg/test/snapshot_util_test.cc b/src/iceberg/test/snapshot_util_test.cc index 83ef09f4b..e94769e31 100644 --- a/src/iceberg/test/snapshot_util_test.cc +++ b/src/iceberg/test/snapshot_util_test.cc @@ -312,14 +312,27 @@ TEST_F(SnapshotUtilTest, SchemaForBranch) { std::string branch = "b1"; ICEBERG_UNWRAP_OR_FAIL(auto schema, SnapshotUtil::SchemaFor(*table_, branch)); - EXPECT_EQ(schema->schema_id(), branch_schema->schema_id()); - EXPECT_EQ(schema->fields().size(), branch_schema->fields().size()); - EXPECT_NE(schema->fields().size(), initial_schema->fields().size()); + EXPECT_EQ(schema->schema_id(), initial_schema->schema_id()); + EXPECT_EQ(schema->fields().size(), initial_schema->fields().size()); + + ICEBERG_UNWRAP_OR_FAIL(auto metadata_schema, + SnapshotUtil::SchemaFor(*table_->metadata(), branch)); + EXPECT_EQ(metadata_schema->schema_id(), initial_schema->schema_id()); + EXPECT_EQ(metadata_schema->fields().size(), initial_schema->fields().size()); } TEST_F(SnapshotUtilTest, SchemaForTag) { // Create a tag pointing to base snapshot auto metadata = table_->metadata(); + auto tag_schema = std::make_shared( + std::vector{SchemaField::MakeRequired(1, "id", int32()), + SchemaField::MakeRequired(2, "data", string()), + SchemaField::MakeOptional(3, "tag_only", string())}, + 1); + metadata->schemas.push_back(tag_schema); + ICEBERG_UNWRAP_OR_FAIL(auto base_snapshot, table_->SnapshotById(base_snapshot_id_)); + base_snapshot->schema_id = tag_schema->schema_id(); + std::string tag = "tag1"; metadata->refs[tag] = std::make_shared( SnapshotRef{.snapshot_id = base_snapshot_id_, .retention = SnapshotRef::Tag{}}); @@ -328,9 +341,14 @@ TEST_F(SnapshotUtilTest, SchemaForTag) { ASSERT_NE(initial_schema, nullptr); ICEBERG_UNWRAP_OR_FAIL(auto schema, SnapshotUtil::SchemaFor(*table_, tag)); - // Tag should return the schema of the snapshot it points to - // Since base snapshot has schema_id = 0, it should return the same schema - EXPECT_EQ(schema->fields().size(), initial_schema->fields().size()); + EXPECT_EQ(schema->schema_id(), tag_schema->schema_id()); + EXPECT_EQ(schema->fields().size(), tag_schema->fields().size()); + EXPECT_NE(schema->fields().size(), initial_schema->fields().size()); + + ICEBERG_UNWRAP_OR_FAIL(auto metadata_schema, + SnapshotUtil::SchemaFor(*table_->metadata(), tag)); + EXPECT_EQ(metadata_schema->schema_id(), tag_schema->schema_id()); + EXPECT_EQ(metadata_schema->fields().size(), tag_schema->fields().size()); } TEST_F(SnapshotUtilTest, SnapshotAfter) { diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index c2eb0f3d3..acf4cd171 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -89,34 +89,46 @@ DeleteFileObjectKey MakeDeleteFileObjectKey(const DataFile& file) { Result>> ValidationAncestorsBetween( const TableMetadata& metadata, int64_t latest_snapshot_id, - int64_t starting_snapshot_id) { + std::optional starting_snapshot_id) { ICEBERG_ASSIGN_OR_RAISE( auto ancestors, SnapshotUtil::AncestorsBetween(metadata, latest_snapshot_id, starting_snapshot_id)); - if (latest_snapshot_id == starting_snapshot_id) { + if (!starting_snapshot_id.has_value()) { + if (!ancestors.empty()) { + const auto& oldest_checked = ancestors.back(); + if (oldest_checked == nullptr || oldest_checked->parent_snapshot_id.has_value()) { + return InvalidArgument( + "Cannot validate history: cannot determine complete history for snapshot {}", + latest_snapshot_id); + } + } + return ancestors; + } + + if (latest_snapshot_id == starting_snapshot_id.value()) { return ancestors; } if (ancestors.empty()) { return InvalidArgument( "Cannot validate history: starting snapshot {} is not an ancestor " "of snapshot {}", - starting_snapshot_id, latest_snapshot_id); + starting_snapshot_id.value(), latest_snapshot_id); } const auto& oldest_checked = ancestors.back(); if (oldest_checked == nullptr || !oldest_checked->parent_snapshot_id.has_value() || - oldest_checked->parent_snapshot_id.value() != starting_snapshot_id) { + oldest_checked->parent_snapshot_id.value() != starting_snapshot_id.value()) { return InvalidArgument( "Cannot validate history: starting snapshot {} is not an ancestor " "of snapshot {}", - starting_snapshot_id, latest_snapshot_id); + starting_snapshot_id.value(), latest_snapshot_id); } return ancestors; } Result ValidationHistory( const TableMetadata& metadata, int64_t latest_snapshot_id, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, std::initializer_list matching_operations, ManifestContent content, const std::shared_ptr& io) { ICEBERG_ASSIGN_OR_RAISE( @@ -146,9 +158,9 @@ Result ValidationHistory( Result> FindMatchingDataFile( const TableMetadata& metadata, const std::vector& manifests, - ManifestStatus status, std::shared_ptr filter, - const PartitionSet* partition_set, const std::shared_ptr& io, - bool case_sensitive) { + const std::unordered_set& snapshot_ids, ManifestStatus status, + std::shared_ptr filter, const PartitionSet* partition_set, + const std::shared_ptr& io, bool case_sensitive) { ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); auto partition_filter = partition_set != nullptr ? std::make_shared(*partition_set) @@ -169,6 +181,10 @@ Result> FindMatchingDataFile( ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); for (const auto& entry : entries) { + if (!entry.snapshot_id.has_value() || + !snapshot_ids.contains(entry.snapshot_id.value())) { + continue; + } if (entry.status == status && entry.data_file != nullptr) { return entry.data_file->file_path; } @@ -224,12 +240,13 @@ Status MergingSnapshotUpdate::AddDataFile(std::shared_ptr file) { return {}; } -Status MergingSnapshotUpdate::ValidateNewDeleteFile(const DataFile& file) { +Status MergingSnapshotUpdate::ValidateNewDeleteFile(const TableMetadata& metadata, + const DataFile& file) { if (file.content == DataFile::Content::kData) { return InvalidArgument("Expected a delete file but got a data file: {}", file.file_path); } - const int8_t format_version = base().format_version; + const int8_t format_version = metadata.format_version; const bool is_dv = ContentFileUtil::IsDV(file); switch (format_version) { case 1: @@ -270,7 +287,7 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, if (!file) { return InvalidArgument("Cannot add a null delete file"); } - ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(*file)); + ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(base(), *file)); if (!file->partition_spec_id.has_value()) { return InvalidArgument("Delete file must have a partition spec ID"); } @@ -542,7 +559,8 @@ Result> MergingSnapshotUpdate::Apply( // Re-validate buffered delete files against the current format version. A format // upgrade between staging and commit could make previously-valid files invalid. for (const auto& pending_file : new_delete_files_) { - ICEBERG_RETURN_UNEXPECTED(ValidateNewDeleteFile(*pending_file.file)); + ICEBERG_RETURN_UNEXPECTED( + ValidateNewDeleteFile(metadata_to_update, *pending_file.file)); } ICEBERG_ASSIGN_OR_RAISE(auto normalized_delete_files, NormalizeNewDeleteFiles()); new_delete_files_ = std::move(normalized_delete_files); @@ -734,7 +752,7 @@ std::unordered_map MergingSnapshotUpdate::Summary() { // ------------------------------------------------------------------------- Status MergingSnapshotUpdate::ValidateAddedDataFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr filter, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive) { if (parent == nullptr) { @@ -747,8 +765,8 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kAdded, filter, - nullptr, io, case_sensitive)); + FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, + ManifestStatus::kAdded, filter, nullptr, io, case_sensitive)); if (conflict_path.has_value()) { return InvalidArgument( "Found conflicting files that can contain rows matching {}: {}", @@ -758,7 +776,7 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( } Status MergingSnapshotUpdate::ValidateDataFilesExist( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, const std::unordered_set& file_paths, bool allow_deletes, std::shared_ptr filter, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive) { @@ -811,6 +829,10 @@ Status MergingSnapshotUpdate::ValidateDataFilesExist( ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); for (const auto& entry : entries) { + if (!entry.snapshot_id.has_value() || + !matching_snapshot_ids.contains(entry.snapshot_id.value())) { + continue; + } if (entry.status != ManifestStatus::kDeleted) { continue; } @@ -835,7 +857,7 @@ Status MergingSnapshotUpdate::ValidateDataFilesExist( } Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, const DataFileSet& replaced_files, const std::shared_ptr& parent, std::shared_ptr io, bool ignore_equality_deletes) { if (parent == nullptr || replaced_files.empty() || metadata.format_version < 2) { @@ -854,9 +876,11 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( // Compute the starting sequence number for the data file check. int64_t starting_seq = TableMetadata::kInitialSequenceNumber; - if (auto snap_result = metadata.SnapshotById(starting_snapshot_id); - snap_result.has_value()) { - starting_seq = snap_result.value()->sequence_number; + if (starting_snapshot_id.has_value()) { + if (auto snap_result = metadata.SnapshotById(starting_snapshot_id.value()); + snap_result.has_value()) { + starting_seq = snap_result.value()->sequence_number; + } } for (const auto& data_file : replaced_files) { @@ -884,7 +908,7 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( } Status MergingSnapshotUpdate::ValidateAddedDataFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io) { if (parent == nullptr) { @@ -897,8 +921,9 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kAdded, nullptr, - &partition_set, io, /*case_sensitive=*/true)); + FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, + ManifestStatus::kAdded, nullptr, &partition_set, io, + /*case_sensitive=*/true)); if (conflict_path.has_value()) { return InvalidArgument( "Found conflicting files that can contain rows in validated partitions: {}", @@ -908,7 +933,7 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( } Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const DataFileSet& replaced_files, const std::shared_ptr& parent, std::shared_ptr io) { if (parent == nullptr || replaced_files.empty() || metadata.format_version < 2) { @@ -923,9 +948,11 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( } int64_t starting_seq = TableMetadata::kInitialSequenceNumber; - if (auto snap_result = metadata.SnapshotById(starting_snapshot_id); - snap_result.has_value()) { - starting_seq = snap_result.value()->sequence_number; + if (starting_snapshot_id.has_value()) { + if (auto snap_result = metadata.SnapshotById(starting_snapshot_id.value()); + snap_result.has_value()) { + starting_seq = snap_result.value()->sequence_number; + } } for (const auto& data_file : replaced_files) { @@ -940,7 +967,7 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( } Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, std::shared_ptr io) { ICEBERG_ASSIGN_OR_RAISE(auto deletes, @@ -956,7 +983,7 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( } Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io) { ICEBERG_ASSIGN_OR_RAISE( @@ -973,7 +1000,7 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( } Status MergingSnapshotUpdate::ValidateDeletedDataFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, std::shared_ptr io) { if (parent == nullptr) { @@ -987,8 +1014,9 @@ Status MergingSnapshotUpdate::ValidateDeletedDataFiles( ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kDeleted, - data_filter, nullptr, io, /*case_sensitive=*/true)); + FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, + ManifestStatus::kDeleted, data_filter, nullptr, io, + /*case_sensitive=*/true)); if (conflict_path.has_value()) { return InvalidArgument( "Found conflicting deleted files that can contain rows matching {}: {}", @@ -999,7 +1027,7 @@ Status MergingSnapshotUpdate::ValidateDeletedDataFiles( } Status MergingSnapshotUpdate::ValidateDeletedDataFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io) { if (parent == nullptr) { @@ -1013,8 +1041,9 @@ Status MergingSnapshotUpdate::ValidateDeletedDataFiles( ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, ManifestStatus::kDeleted, nullptr, - &partition_set, io, /*case_sensitive=*/true)); + FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, + ManifestStatus::kDeleted, nullptr, &partition_set, io, + /*case_sensitive=*/true)); if (conflict_path.has_value()) { return InvalidArgument("Found conflicting deleted files in validated partitions: {}", conflict_path.value()); @@ -1023,7 +1052,7 @@ Status MergingSnapshotUpdate::ValidateDeletedDataFiles( } Result> MergingSnapshotUpdate::AddedDeleteFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, std::shared_ptr partition_set, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive) { @@ -1046,9 +1075,11 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles // Compute the starting sequence number from the starting snapshot. int64_t starting_seq = TableMetadata::kInitialSequenceNumber; - if (auto snap_result = metadata.SnapshotById(starting_snapshot_id); - snap_result.has_value()) { - starting_seq = snap_result.value()->sequence_number; + if (starting_snapshot_id.has_value()) { + if (auto snap_result = metadata.SnapshotById(starting_snapshot_id.value()); + snap_result.has_value()) { + starting_seq = snap_result.value()->sequence_number; + } } TableMetadataCache metadata_cache(&metadata); @@ -1071,7 +1102,7 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles } Status MergingSnapshotUpdate::ValidateAddedDVs( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr conflict_filter, const std::unordered_set& referenced_data_files, const std::shared_ptr& parent, std::shared_ptr io) { @@ -1097,6 +1128,10 @@ Status MergingSnapshotUpdate::ValidateAddedDVs( ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); for (const auto& entry : entries) { + if (!entry.snapshot_id.has_value() || + !history.snapshot_ids.contains(entry.snapshot_id.value())) { + continue; + } if (entry.data_file == nullptr || !ContentFileUtil::IsDV(*entry.data_file) || !entry.data_file->referenced_data_file.has_value()) { continue; @@ -1111,7 +1146,7 @@ Status MergingSnapshotUpdate::ValidateAddedDVs( } Status MergingSnapshotUpdate::ValidateAddedDVs( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr conflict_filter, const std::shared_ptr& parent, std::shared_ptr io) const { std::unordered_set referenced_data_files; diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h index bcc7c0e82..3658269f9 100644 --- a/src/iceberg/update/merging_snapshot_update.h +++ b/src/iceberg/update/merging_snapshot_update.h @@ -88,7 +88,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// - Format v1: deletes are not supported. /// - Format v2: position deletes must NOT be deletion vectors (DVs). /// - Format v3+: position deletes MUST be deletion vectors (DVs). - Status ValidateNewDeleteFile(const DataFile& file); + Status ValidateNewDeleteFile(const TableMetadata& metadata, const DataFile& file); /// \brief Stage a delete file with an explicit data sequence number. /// @@ -163,34 +163,36 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Returns all data files staged for addition. std::vector> AddedDataFiles() const; - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a data file matching the given filter expression. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a data file matching the given filter expression. static Status ValidateAddedDataFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, std::shared_ptr filter, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive = true); - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a data file in any partition of the given partition set. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a data file in any partition of the given partition + /// set. /// static Status ValidateAddedDataFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io); - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// removed a file whose path is in file_paths (and allow_deletes is false). + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, removed a file whose path is in file_paths (and + /// allow_deletes is false). static Status ValidateDataFilesExist( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, const std::unordered_set& file_paths, bool allow_deletes, std::shared_ptr filter, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive = true); - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a delete file that covers a file in replaced_files. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a delete file that covers a file in replaced_files. /// /// Whether equality deletes are checked is derived automatically from whether /// a custom data sequence number was set via SetNewDataFilesDataSequenceNumber(): @@ -199,7 +201,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// /// Subclasses should prefer this overload over the static one. Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, const DataFileSet& replaced_files, const std::shared_ptr& parent, std::shared_ptr io) const { @@ -208,79 +210,79 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { new_data_files_data_seq_number_.has_value()); } - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a delete file that covers a file in replaced_files. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a delete file that covers a file in replaced_files. /// /// \param ignore_equality_deletes If true, only position deletes are checked. /// Set to true when replaced data files have the same sequence number as the /// new files (e.g. RewriteFiles), so equality deletes at higher sequence numbers /// still apply and are not a conflict. - static Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, - const DataFileSet& replaced_files, - const std::shared_ptr& parent, - std::shared_ptr io, - bool ignore_equality_deletes = false); - - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a delete file matching the data filter that covers a file in replaced_files. + static Status ValidateNoNewDeletesForDataFiles( + const TableMetadata& metadata, std::optional starting_snapshot_id, + const DataFileSet& replaced_files, const std::shared_ptr& parent, + std::shared_ptr io, bool ignore_equality_deletes = false); + + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a delete file matching the data filter that covers a + /// file in replaced_files. /// - static Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, - std::shared_ptr data_filter, - const DataFileSet& replaced_files, - const std::shared_ptr& parent, - std::shared_ptr io); - - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a delete file matching the given row filter. + static Status ValidateNoNewDeletesForDataFiles( + const TableMetadata& metadata, std::optional starting_snapshot_id, + std::shared_ptr data_filter, const DataFileSet& replaced_files, + const std::shared_ptr& parent, std::shared_ptr io); + + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a delete file matching the given row filter. /// static Status ValidateNoNewDeleteFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, std::shared_ptr io); - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a delete file matching any partition in the given partition set. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a delete file matching any partition in the given + /// partition set. /// static Status ValidateNoNewDeleteFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io); - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// deleted a data file matching the given row filter. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, deleted a data file matching the given row filter. /// static Status ValidateDeletedDataFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, std::shared_ptr io); - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// deleted a data file in any partition of the given partition set. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, deleted a data file in any partition of the given partition + /// set. /// static Status ValidateDeletedDataFiles(const TableMetadata& metadata, - int64_t starting_snapshot_id, + std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io); /// \brief Build a DeleteFileIndex of delete files added since starting_snapshot_id. static Result> AddedDeleteFiles( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, std::shared_ptr partition_set, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive = true); - /// \brief Return an error if any snapshot in [starting_snapshot_id+1, parent] - /// added a deletion vector that conflicts with DVs being written. + /// \brief Return an error if any snapshot after starting_snapshot_id, or from + /// the beginning if unset, added a deletion vector that conflicts with DVs being + /// written. /// static Status ValidateAddedDVs( - const TableMetadata& metadata, int64_t starting_snapshot_id, + const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr conflict_filter, const std::unordered_set& referenced_data_files, const std::shared_ptr& parent, std::shared_ptr io); @@ -302,7 +304,8 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { Status AddDeleteFile(std::shared_ptr file, std::optional data_sequence_number); - Status ValidateAddedDVs(const TableMetadata& metadata, int64_t starting_snapshot_id, + Status ValidateAddedDVs(const TableMetadata& metadata, + std::optional starting_snapshot_id, std::shared_ptr conflict_filter, const std::shared_ptr& parent, std::shared_ptr io) const; diff --git a/src/iceberg/update/snapshot_update.cc b/src/iceberg/update/snapshot_update.cc index bff12c912..b3ce0ff30 100644 --- a/src/iceberg/update/snapshot_update.cc +++ b/src/iceberg/update/snapshot_update.cc @@ -41,7 +41,7 @@ namespace iceberg { namespace { -// Skips updating total if parsing fails would be lenient; here we choose to be strict. +// Java skips updating totals if parsing fails; C++ treats parse failures as errors. Status UpdateTotal(std::unordered_map& summary, const std::unordered_map& previous_summary, const std::string& total_property, const std::string& added_property, diff --git a/src/iceberg/update/update_partition_statistics.h b/src/iceberg/update/update_partition_statistics.h index 86bf1b081..982b1bd39 100644 --- a/src/iceberg/update/update_partition_statistics.h +++ b/src/iceberg/update/update_partition_statistics.h @@ -65,8 +65,9 @@ class ICEBERG_EXPORT UpdatePartitionStatistics : public PendingUpdate { /// \brief Partition statistics updates are intentionally not retried today. /// - /// Commits directly without a retry loop. Keep this conservative until we add - /// explicit replay coverage for this update type. + /// This matches the current Java `SetPartitionStatistics` behavior, which commits + /// directly without a retry loop. Keep this conservative until we add explicit replay + /// coverage for this update type. bool IsRetryable() const override { return false; } struct ApplyResult { diff --git a/src/iceberg/util/snapshot_util.cc b/src/iceberg/util/snapshot_util.cc index 1bbd6ae8c..49019408b 100644 --- a/src/iceberg/util/snapshot_util.cc +++ b/src/iceberg/util/snapshot_util.cc @@ -375,7 +375,7 @@ Result> SnapshotUtil::SchemaFor(const Table& table, const auto& metadata = table.metadata(); auto it = metadata->refs.find(ref); - if (it == metadata->refs.cend()) { + if (it == metadata->refs.cend() || it->second->type() == SnapshotRefType::kBranch) { return table.schema(); } @@ -389,7 +389,7 @@ Result> SnapshotUtil::SchemaFor(const TableMetadata& met } auto it = metadata.refs.find(ref); - if (it == metadata.refs.end()) { + if (it == metadata.refs.end() || it->second->type() == SnapshotRefType::kBranch) { return metadata.Schema(); } diff --git a/src/iceberg/util/snapshot_util_internal.h b/src/iceberg/util/snapshot_util_internal.h index 66a99a3b4..8a3158185 100644 --- a/src/iceberg/util/snapshot_util_internal.h +++ b/src/iceberg/util/snapshot_util_internal.h @@ -306,9 +306,9 @@ class ICEBERG_EXPORT SnapshotUtil { /// \brief Return the schema of the snapshot at a given ref. /// - /// If the ref does not exist, the current table schema is returned. If the ref exists - /// and points to a snapshot (branch or tag), the schema recorded for that snapshot is - /// returned. + /// If the ref does not exist or the ref is a branch, the table schema is returned + /// because it will be the schema when the new branch is created. If the ref is a tag, + /// then the snapshot schema is returned. /// /// \param table The table /// \param ref Ref name of the table (empty string means main branch) @@ -318,9 +318,9 @@ class ICEBERG_EXPORT SnapshotUtil { /// \brief Return the schema of the snapshot at a given ref. /// - /// If the ref does not exist, the current table schema is returned. If the ref exists - /// and points to a snapshot (branch or tag), the schema recorded for that snapshot is - /// returned. + /// If the ref does not exist or the ref is a branch, the table schema is returned + /// because it will be the schema when the new branch is created. If the ref is a tag, + /// then the snapshot schema is returned. /// /// \param metadata The table metadata /// \param ref Ref name of the table (empty string means main branch) From ffde4e8b26b8c313156cefcfe4da8c3aaea6a18e Mon Sep 17 00:00:00 2001 From: Gang Wu Date: Wed, 3 Jun 2026 00:12:03 +0800 Subject: [PATCH 16/16] polish code to align with Java parity --- .../manifest/manifest_filter_manager.cc | 352 +++-- .../manifest/manifest_filter_manager.h | 86 +- .../manifest/manifest_merge_manager.cc | 214 ++- src/iceberg/manifest/manifest_merge_manager.h | 83 +- src/iceberg/manifest/manifest_reader.cc | 39 +- src/iceberg/manifest/manifest_reader.h | 16 +- .../manifest/manifest_reader_internal.h | 4 +- src/iceberg/snapshot.cc | 24 - src/iceberg/snapshot.h | 13 - src/iceberg/test/data_file_set_test.cc | 79 +- src/iceberg/test/fast_append_test.cc | 13 +- .../test/manifest_filter_manager_test.cc | 704 ++++++++-- .../test/manifest_merge_manager_test.cc | 390 ++++-- src/iceberg/test/manifest_reader_test.cc | 35 + .../test/merging_snapshot_update_test.cc | 330 ++++- src/iceberg/update/fast_append.cc | 3 +- src/iceberg/update/fast_append.h | 2 +- src/iceberg/update/merging_snapshot_update.cc | 1164 ++++++++++------- src/iceberg/update/merging_snapshot_update.h | 93 +- src/iceberg/update/meson.build | 1 + src/iceberg/update/snapshot_update.cc | 73 +- src/iceberg/update/snapshot_update.h | 25 +- src/iceberg/util/data_file_set.h | 164 ++- 23 files changed, 2806 insertions(+), 1101 deletions(-) diff --git a/src/iceberg/manifest/manifest_filter_manager.cc b/src/iceberg/manifest/manifest_filter_manager.cc index 8635abd6f..49fec0461 100644 --- a/src/iceberg/manifest/manifest_filter_manager.cc +++ b/src/iceberg/manifest/manifest_filter_manager.cc @@ -20,6 +20,7 @@ #include "iceberg/manifest/manifest_filter_manager.h" #include +#include #include #include @@ -29,6 +30,7 @@ #include "iceberg/expression/manifest_evaluator.h" #include "iceberg/expression/residual_evaluator.h" #include "iceberg/expression/strict_metrics_evaluator.h" +#include "iceberg/file_io.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_list.h" #include "iceberg/manifest/manifest_reader.h" @@ -63,37 +65,76 @@ Result FormatPartitionPath(const PartitionSpecsById& specs_by_id, return spec->PartitionPath(file.partition); } -} // namespace +void AddDeletedFileToManager(ManifestContent manifest_content, DataFileSet& data_files, + DeleteFileSet& delete_files, + std::vector>& deleted_files, + DataFileSet& deleted_data_file_set, + DeleteFileSet& deleted_file_set, + const std::shared_ptr& file) { + if (file == nullptr) { + return; + } -size_t ManifestFilterManager::DeleteFileKeyHash::operator()( - const DeleteFileKey& key) const { - size_t hash = std::hash{}(key.path); - auto combine = [&hash](const auto& value) { - size_t value_hash = value.has_value() ? std::hash{}(*value) : 0; - hash ^= value_hash + 0x9e3779b9 + (hash << 6) + (hash >> 2); - }; - combine(key.content_offset); - combine(key.content_size_in_bytes); - return hash; + bool inserted; + if (manifest_content == ManifestContent::kData) { + data_files.insert(file); + inserted = deleted_data_file_set.insert(file).second; + } else { + delete_files.insert(file); + inserted = deleted_file_set.insert(file).second; + } + if (inserted) { + deleted_files.push_back(file); + } } -ManifestFilterManager::DeleteFileKey ManifestFilterManager::MakeDeleteFileKey( - const DataFile& file) { - return DeleteFileKey{.path = file.file_path, - .content_offset = file.content_offset, - .content_size_in_bytes = file.content_size_in_bytes}; +bool AddDeletedFileToManifest(ManifestContent manifest_content, + std::vector>& deleted_files, + DataFileSet& deleted_data_file_set, + DeleteFileSet& deleted_file_set, + const std::shared_ptr& file) { + if (file == nullptr) { + return false; + } + bool inserted = manifest_content == ManifestContent::kData + ? deleted_data_file_set.insert(file).second + : deleted_file_set.insert(file).second; + if (inserted) { + deleted_files.push_back(file); + } + return inserted; +} + +} // namespace + +Result> ManifestFilterManager::Make( + ManifestContent content, std::shared_ptr file_io, + std::function delete_file) { + ICEBERG_PRECHECK(file_io != nullptr, "FileIO cannot be null"); + return std::unique_ptr( + new ManifestFilterManager(content, std::move(file_io), std::move(delete_file))); } -ManifestFilterManager::ManifestFilterManager(ManifestContent content, - std::shared_ptr file_io) +ManifestFilterManager::ManifestFilterManager( + ManifestContent content, std::shared_ptr file_io, + std::function delete_file) : manifest_content_(content), file_io_(std::move(file_io)), - delete_expr_(Expressions::AlwaysFalse()) {} + delete_file_(std::move(delete_file)), + delete_expr_(Expressions::AlwaysFalse()) { + ICEBERG_DCHECK(file_io_, "FileIO cannot be null"); + if (delete_file_ == nullptr) { + delete_file_ = [this](const std::string& location) { + return file_io_->DeleteFile(location); + }; + } +} ManifestFilterManager::~ManifestFilterManager() = default; Status ManifestFilterManager::DeleteByRowFilter(std::shared_ptr expr) { ICEBERG_PRECHECK(expr != nullptr, "Cannot delete files using filter: null"); + ICEBERG_RETURN_UNEXPECTED(InvalidateFilteredCache()); ICEBERG_ASSIGN_OR_RAISE(delete_expr_, Or::MakeFolded(delete_expr_, std::move(expr))); manifest_evaluator_cache_.clear(); residual_evaluator_cache_.clear(); @@ -106,27 +147,58 @@ void ManifestFilterManager::CaseSensitive(bool case_sensitive) { residual_evaluator_cache_.clear(); } -void ManifestFilterManager::DeleteFile(std::string_view path) { +Status ManifestFilterManager::DeleteFile(std::string_view path) { + ICEBERG_RETURN_UNEXPECTED(InvalidateFilteredCache()); delete_paths_.insert(std::string(path)); + return {}; } Status ManifestFilterManager::DeleteFile(std::shared_ptr file) { ICEBERG_PRECHECK(file != nullptr, "Cannot delete file: null"); - delete_file_keys_.insert(MakeDeleteFileKey(*file)); + ICEBERG_RETURN_UNEXPECTED(InvalidateFilteredCache()); + if (manifest_content_ == ManifestContent::kData) { + data_files_.insert(file); + data_files_to_delete_.insert(std::move(file)); + } else { + delete_files_.insert(file); + delete_files_to_delete_.insert(std::move(file)); + } return {}; } -const DataFileSet& ManifestFilterManager::FilesToBeDeleted() const { - return delete_files_; -} +const DataFileSet& ManifestFilterManager::FilesToBeDeleted() const { return data_files_; } const std::vector>& ManifestFilterManager::DeletedFiles() const { return deleted_files_; } -void ManifestFilterManager::DropPartition(int32_t spec_id, PartitionValues partition) { +Result ManifestFilterManager::BuildSummary( + const std::vector& manifests, + const PartitionSpecsById& specs_by_id) const { + SnapshotSummaryBuilder summary; + for (const auto& manifest : manifests) { + auto deleted_iter = filtered_manifest_to_deleted_files_.find(manifest); + if (deleted_iter == filtered_manifest_to_deleted_files_.end()) { + continue; + } + + ICEBERG_ASSIGN_OR_RAISE(auto spec, + PartitionSpecById(specs_by_id, manifest.partition_spec_id)); + for (const auto& file : deleted_iter->second.files) { + if (file != nullptr) { + ICEBERG_RETURN_UNEXPECTED(summary.DeletedFile(*spec, *file)); + } + } + } + summary.IncrementDuplicateDeletes(duplicate_deletes_count_); + return summary; +} + +Status ManifestFilterManager::DropPartition(int32_t spec_id, PartitionValues partition) { + ICEBERG_RETURN_UNEXPECTED(InvalidateFilteredCache()); drop_partitions_.add(spec_id, std::move(partition)); + return {}; } void ManifestFilterManager::FailMissingDeletePaths() { @@ -137,24 +209,32 @@ void ManifestFilterManager::FailAnyDelete() { fail_any_delete_ = true; } bool ManifestFilterManager::ContainsDeletes() const { return HasRowFilterExpression(delete_expr_) || !delete_paths_.empty() || - !delete_file_keys_.empty() || !drop_partitions_.empty(); + !data_files_to_delete_.empty() || !delete_files_to_delete_.empty() || + !drop_partitions_.empty(); } -void ManifestFilterManager::DropDeleteFilesOlderThan(int64_t sequence_number) { +Status ManifestFilterManager::DropDeleteFilesOlderThan(int64_t sequence_number) { + ICEBERG_PRECHECK(sequence_number >= 0, "Invalid minimum data sequence number: {}", + sequence_number); min_sequence_number_ = sequence_number; + return {}; } void ManifestFilterManager::RemoveDanglingDeletesFor(const DataFileSet& deleted_files) { + std::unordered_set removed_data_file_paths; for (const auto& file : deleted_files) { - removed_data_file_paths_.insert(file->file_path); + if (file != nullptr) { + removed_data_file_paths.insert(file->file_path); + } } + removed_data_file_paths_ = std::move(removed_data_file_paths); } Result ManifestFilterManager::CanContainDroppedFiles(const ManifestFile&) const { - // TODO(Guotao): Use the manifest descriptor to skip unrelated object-delete - // manifests once object-delete partitions are tracked separately. - return !delete_paths_.empty() || !delete_file_keys_.empty() || - !removed_data_file_paths_.empty(); + // TODO(Guotao): prune object deletes by partition once manifest partition + // summary checks are available. + return !delete_paths_.empty() || !data_files_to_delete_.empty() || + !delete_files_to_delete_.empty() || !removed_data_file_paths_.empty(); } Result ManifestFilterManager::CanContainDroppedPartitions( @@ -240,16 +320,23 @@ Result ManifestFilterManager::ShouldDelete(const ManifestEntry& entry, const DataFile& file = *entry.data_file; int32_t spec_id = file.partition_spec_id.value_or(manifest_spec_id); - // Path/object-based and partition-drop checks. - if (delete_paths_.count(file.file_path) || - delete_file_keys_.count(MakeDeleteFileKey(file)) || - drop_partitions_.contains(spec_id, file.partition)) { + // All delete branches share fail-any-delete handling. + auto marked_for_delete = [&]() -> Result { if (fail_any_delete_) { ICEBERG_ASSIGN_OR_RAISE(auto partition_path, FormatPartitionPath(specs_by_id, file, spec_id)); - return InvalidArgument("Operation would delete existing data: {}", partition_path); + return ValidationFailed("Operation would delete existing data: {}", partition_path); } return true; + }; + + // Path/object-based and partition-drop checks. + bool object_delete = manifest_content_ == ManifestContent::kData + ? data_files_to_delete_.contains(file) + : delete_files_to_delete_.contains(file); + if (delete_paths_.count(file.file_path) || object_delete || + drop_partitions_.contains(spec_id, file.partition)) { + return marked_for_delete(); } // Delete-manifest-specific cleanup (only for ManifestContent::kDeletes). @@ -260,14 +347,14 @@ Result ManifestFilterManager::ShouldDelete(const ManifestEntry& entry, // those entries predate sequence number assignment and must not be pruned. int64_t seq = entry.sequence_number.value_or(0); if (min_sequence_number_ > 0 && seq > 0 && seq < min_sequence_number_) { - return true; + return marked_for_delete(); } // Drop DVs that reference a data file that has been removed (dangling DV). if (!removed_data_file_paths_.empty() && file.IsDeletionVector() && file.referenced_data_file.has_value() && removed_data_file_paths_.count(*file.referenced_data_file)) { - return true; + return marked_for_delete(); } } @@ -282,13 +369,7 @@ Result ManifestFilterManager::ShouldDelete(const ManifestEntry& entry, StrictMetricsEvaluator::Make(residual_expr, schema, case_sensitive_)); ICEBERG_ASSIGN_OR_RAISE(auto strict_match, strict_eval->Evaluate(file)); if (strict_match) { - if (fail_any_delete_) { - ICEBERG_ASSIGN_OR_RAISE(auto partition_path, - FormatPartitionPath(specs_by_id, file, spec_id)); - return InvalidArgument("Operation would delete existing data: {}", - partition_path); - } - return true; + return marked_for_delete(); } ICEBERG_ASSIGN_OR_RAISE(auto incl_eval, InclusiveMetricsEvaluator::Make( @@ -298,7 +379,7 @@ Result ManifestFilterManager::ShouldDelete(const ManifestEntry& entry, if (manifest_content_ == ManifestContent::kDeletes) { return false; } - return InvalidArgument( + return ValidationFailed( "Cannot delete file where some, but not all, rows match filter: {}", file.file_path); } @@ -309,19 +390,34 @@ Result ManifestFilterManager::ShouldDelete(const ManifestEntry& entry, bool ManifestFilterManager::CanTrustManifestReferences( const std::vector&) const { - // TODO(Guotao): Track source manifest locations for object deletes so manifests - // outside the referenced set can be skipped before any other delete checks. + // TODO(Guotao): add DataFile manifest locations and use them to skip unrelated + // manifests. Until then, take the conservative path. return false; } Result ManifestFilterManager::FilterManifest( const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, const ManifestFile& manifest, bool trust_manifest_references, - const ManifestWriterFactory& writer_factory, FoundDeletes& found_deletes) { + const ManifestWriterFactory& writer_factory) { + auto cached = filtered_manifests_.find(manifest); + if (cached != filtered_manifests_.end()) { + auto deleted_iter = filtered_manifest_to_deleted_files_.find(cached->second); + if (deleted_iter != filtered_manifest_to_deleted_files_.end()) { + for (const auto& file : deleted_iter->second.files) { + AddDeletedFileToManager(manifest_content_, data_files_, delete_files_, + deleted_files_, deleted_data_file_set_, + deleted_delete_file_set_, file); + } + duplicate_deletes_count_ += deleted_iter->second.duplicate_deletes_count; + } + return cached->second; + } + ICEBERG_ASSIGN_OR_RAISE( auto can_contain_deleted_files, CanContainDeletedFiles(manifest, schema, specs_by_id, trust_manifest_references)); if (!can_contain_deleted_files) { + filtered_manifests_.emplace(manifest, manifest); return manifest; } @@ -334,11 +430,16 @@ Result ManifestFilterManager::FilterManifest( ICEBERG_ASSIGN_OR_RAISE(auto has_deleted_files, ManifestHasDeletedFiles(entries, schema, specs_by_id, spec_id)); if (!has_deleted_files) { + filtered_manifests_.emplace(manifest, manifest); return manifest; } - return FilterManifestWithDeletedFiles(entries, spec_id, schema, specs_by_id, - writer_factory, found_deletes); + ICEBERG_ASSIGN_OR_RAISE(auto filtered_manifest, + FilterManifestWithDeletedFiles(entries, spec_id, schema, + specs_by_id, writer_factory)); + filtered_manifests_.emplace(manifest, filtered_manifest); + ++replaced_manifests_count_; + return filtered_manifest; } Result ManifestFilterManager::ManifestHasDeletedFiles( @@ -357,29 +458,25 @@ Result ManifestFilterManager::ManifestHasDeletedFiles( Result ManifestFilterManager::FilterManifestWithDeletedFiles( const std::vector& entries, int32_t manifest_spec_id, const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, - const ManifestWriterFactory& writer_factory, FoundDeletes& found_deletes) { + const ManifestWriterFactory& writer_factory) { ICEBERG_ASSIGN_OR_RAISE(auto writer, writer_factory(manifest_spec_id, manifest_content_)); + std::vector> deleted_files; + DataFileSet deleted_data_file_set; + DeleteFileSet deleted_file_set; + int32_t duplicate_deletes_count = 0; for (const auto& entry : entries) { ICEBERG_ASSIGN_OR_RAISE(auto should_delete, ShouldDelete(entry, schema, specs_by_id, manifest_spec_id)); if (should_delete) { if (entry.data_file) { - const auto key = MakeDeleteFileKey(*entry.data_file); - if (delete_paths_.count(entry.data_file->file_path)) { - found_deletes.paths.insert(entry.data_file->file_path); - } - if (delete_file_keys_.count(key)) { - found_deletes.files.insert(key); - } - auto file = std::make_shared(*entry.data_file); - delete_files_.insert(file); - auto [_, inserted] = deleted_file_keys_.insert(key); - if (inserted) { - deleted_files_.push_back(std::move(file)); - } else { - ++duplicate_deletes_count_; + AddDeletedFileToManager(manifest_content_, data_files_, delete_files_, + deleted_files_, deleted_data_file_set_, + deleted_delete_file_set_, file); + if (!AddDeletedFileToManifest(manifest_content_, deleted_files, + deleted_data_file_set, deleted_file_set, file)) { + ++duplicate_deletes_count; } } ICEBERG_RETURN_UNEXPECTED(writer->WriteDeletedEntry(entry)); @@ -389,30 +486,72 @@ Result ManifestFilterManager::FilterManifestWithDeletedFiles( } ICEBERG_RETURN_UNEXPECTED(writer->Close()); - return writer->ToManifestFile(); + ICEBERG_ASSIGN_OR_RAISE(auto filtered_manifest, writer->ToManifestFile()); + duplicate_deletes_count_ += duplicate_deletes_count; + filtered_manifest_to_deleted_files_[filtered_manifest] = FilteredManifestDeletes{ + .files = std::move(deleted_files), + .duplicate_deletes_count = duplicate_deletes_count, + }; + return filtered_manifest; +} + +Status ManifestFilterManager::InvalidateFilteredCache() { + ICEBERG_RETURN_UNEXPECTED(CleanUncommitted({})); + replaced_manifests_count_ = 0; + return {}; +} + +void ManifestFilterManager::ResetDeletedFiles() { + data_files_.clear(); + for (const auto& file : data_files_to_delete_) { + data_files_.insert(file); + } + delete_files_.clear(); + for (const auto& file : delete_files_to_delete_) { + delete_files_.insert(file); + } } -Status ManifestFilterManager::ValidateRequiredDeletes( - const FoundDeletes& found_deletes) const { +Status ManifestFilterManager::ValidateRequiredDeletes() const { if (!fail_missing_delete_paths_) { return {}; } - std::string missing; - for (const auto& path : delete_paths_) { - if (!found_deletes.paths.count(path)) { - if (!missing.empty()) missing += ", "; - missing += path; + std::string missing_files; + const auto append_missing = [&missing_files](const std::string& path) { + if (!missing_files.empty()) missing_files += ","; + missing_files += path; + }; + for (const auto& key : data_files_to_delete_) { + if (!deleted_data_file_set_.contains(key)) { + append_missing(key->file_path); } } - for (const auto& key : delete_file_keys_) { - if (!found_deletes.files.count(key)) { - if (!missing.empty()) missing += ", "; - missing += key.path; + for (const auto& key : delete_files_to_delete_) { + if (!deleted_delete_file_set_.contains(key)) { + append_missing(key->file_path); } } - if (!missing.empty()) { - return InvalidArgument("Missing delete paths: {}", missing); + if (!missing_files.empty()) { + return ValidationFailed("Missing required files to delete: {}", missing_files); + } + + std::string missing_paths; + for (const auto& path : delete_paths_) { + bool found = false; + for (const auto& deleted_file : deleted_files_) { + if (deleted_file != nullptr && deleted_file->file_path == path) { + found = true; + break; + } + } + if (!found) { + if (!missing_paths.empty()) missing_paths += ","; + missing_paths += path; + } + } + if (!missing_paths.empty()) { + return ValidationFailed("Missing required files to delete: {}", missing_paths); } return {}; } @@ -428,13 +567,13 @@ Result> ManifestFilterManager::FilterManifests( const std::shared_ptr& schema, const TableMetadata& metadata, const std::shared_ptr& base_snapshot, const ManifestWriterFactory& writer_factory) { - delete_files_.clear(); + ResetDeletedFiles(); deleted_files_.clear(); - deleted_file_keys_.clear(); + deleted_data_file_set_.clear(); + deleted_delete_file_set_.clear(); duplicate_deletes_count_ = 0; - replaced_manifests_count_ = 0; if (!base_snapshot) { - ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(FoundDeletes{})); + ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes()); return std::vector{}; } @@ -472,39 +611,54 @@ Result> ManifestFilterManager::FilterManifests( } } - FoundDeletes found_deletes; - delete_files_.clear(); + ResetDeletedFiles(); deleted_files_.clear(); - deleted_file_keys_.clear(); + deleted_data_file_set_.clear(); + deleted_delete_file_set_.clear(); duplicate_deletes_count_ = 0; if (manifests.empty()) { - replaced_manifests_count_ = 0; - ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(found_deletes)); + ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes()); return std::vector{}; } bool trust_manifest_references = CanTrustManifestReferences(manifests); manifest_evaluator_cache_.clear(); residual_evaluator_cache_.clear(); - replaced_manifests_count_ = 0; // TODO(Guotao): Parallelize manifest filtering with per-manifest results, then // merge found paths and deleted files after the loop. std::vector filtered; filtered.reserve(manifests.size()); for (const auto* manifest_ptr : manifests) { - ICEBERG_ASSIGN_OR_RAISE( - auto filtered_manifest, - FilterManifest(schema, specs_by_id, *manifest_ptr, trust_manifest_references, - writer_factory, found_deletes)); - if (filtered_manifest.manifest_path != manifest_ptr->manifest_path) { - ++replaced_manifests_count_; - } + ICEBERG_ASSIGN_OR_RAISE(auto filtered_manifest, + FilterManifest(schema, specs_by_id, *manifest_ptr, + trust_manifest_references, writer_factory)); filtered.push_back(std::move(filtered_manifest)); } - ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes(found_deletes)); + ICEBERG_RETURN_UNEXPECTED(ValidateRequiredDeletes()); return filtered; } +Status ManifestFilterManager::CleanUncommitted( + const std::unordered_set& committed) { + auto entries = std::vector>{ + filtered_manifests_.begin(), filtered_manifests_.end()}; + for (const auto& [manifest, filtered] : entries) { + if (committed.contains(filtered.manifest_path)) { + continue; + } + + if (manifest != filtered) { + std::ignore = delete_file_(filtered.manifest_path); + if (replaced_manifests_count_ > 0) { + --replaced_manifests_count_; + } + } + filtered_manifests_.erase(manifest); + filtered_manifest_to_deleted_files_.erase(filtered); + } + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/manifest/manifest_filter_manager.h b/src/iceberg/manifest/manifest_filter_manager.h index c85cc8717..e742812cc 100644 --- a/src/iceberg/manifest/manifest_filter_manager.h +++ b/src/iceberg/manifest/manifest_filter_manager.h @@ -24,8 +24,8 @@ /// or EXISTING based on row-filter expressions, exact path deletes, and partition drops. #include +#include #include -#include #include #include #include @@ -35,6 +35,7 @@ #include "iceberg/manifest/manifest_list.h" #include "iceberg/manifest/manifest_writer.h" #include "iceberg/result.h" +#include "iceberg/snapshot.h" #include "iceberg/type_fwd.h" #include "iceberg/util/data_file_set.h" #include "iceberg/util/partition_value_util.h" @@ -48,9 +49,6 @@ namespace iceberg { /// entries are returned unchanged (no I/O). Manifests that do contain deleted /// entries are rewritten with those entries marked DELETED. /// -/// The manager is content-agnostic: pass ManifestContent::kData to process data -/// manifests, or ManifestContent::kDeletes to process delete manifests. -/// /// TODO(Guotao): For ManifestContent::kDeletes, implement cleanup for orphan delete files /// and dangling deletion vectors. /// @@ -59,7 +57,9 @@ class ICEBERG_EXPORT ManifestFilterManager { public: using PartitionSpecsById = std::unordered_map>; - ManifestFilterManager(ManifestContent content, std::shared_ptr file_io); + static Result> Make( + ManifestContent content, std::shared_ptr file_io, + std::function delete_file = {}); ~ManifestFilterManager(); ManifestFilterManager(const ManifestFilterManager&) = delete; @@ -81,7 +81,7 @@ class ICEBERG_EXPORT ManifestFilterManager { /// Any manifest entry whose file_path matches this path will be marked DELETED. /// /// \param path The exact file path to delete - void DeleteFile(std::string_view path); + Status DeleteFile(std::string_view path); /// \brief Register a file object for deletion. /// @@ -93,9 +93,8 @@ class ICEBERG_EXPORT ManifestFilterManager { /// \brief Returns the set of file objects marked for deletion by this manager. /// - /// This is populated by the most recent FilterManifests() call and contains only - /// files that were actually deleted from filtered manifests. Used by higher-level - /// operations (e.g. RowDelta) to enumerate deleted data files for follow-up cleanup. + /// Includes file objects explicitly registered for deletion plus files deleted while + /// filtering manifests. const DataFileSet& FilesToBeDeleted() const; /// \brief Returns content-file objects deleted by the most recent @@ -106,13 +105,19 @@ class ICEBERG_EXPORT ManifestFilterManager { /// FilterManifests() call. int32_t DuplicateDeletesCount() const { return duplicate_deletes_count_; } + /// \brief Build a snapshot-summary fragment from filtered manifests. + /// + Result BuildSummary( + const std::vector& manifests, + const PartitionSpecsById& specs_by_id) const; + /// \brief Register a partition for dropping. /// /// Any manifest entry whose (spec_id, partition) pair matches will be marked DELETED. /// /// \param spec_id The partition spec ID /// \param partition The partition values to drop - void DropPartition(int32_t spec_id, PartitionValues partition); + Status DropPartition(int32_t spec_id, PartitionValues partition); /// \brief Set a flag that makes FilterManifests() fail if any registered /// delete path was not found in any manifest entry. @@ -140,7 +145,7 @@ class ICEBERG_EXPORT ManifestFilterManager { /// /// \param sequence_number the inclusive lower bound; delete files older than /// this value are dropped - void DropDeleteFilesOlderThan(int64_t sequence_number); + Status DropDeleteFilesOlderThan(int64_t sequence_number); /// \brief Register data files that have been removed so their dangling DVs /// can be cleaned up. @@ -189,7 +194,14 @@ class ICEBERG_EXPORT ManifestFilterManager { const std::vector& manifests, const ManifestWriterFactory& writer_factory); + /// \brief Delete cached filtered manifests that were not committed and roll back + /// replaced-manifest accounting. + Status CleanUncommitted(const std::unordered_set& committed); + private: + ManifestFilterManager(ManifestContent content, std::shared_ptr file_io, + std::function delete_file); + /// \brief Returns true if the manifest might contain files matching any expression. Result CanContainExpressionDeletes(const ManifestFile& manifest, const std::shared_ptr& schema, @@ -214,31 +226,16 @@ class ICEBERG_EXPORT ManifestFilterManager { bool CanTrustManifestReferences( const std::vector& manifests) const; - struct DeleteFileKey { - std::string path; - std::optional content_offset; - std::optional content_size_in_bytes; - - bool operator==(const DeleteFileKey& other) const = default; - }; - - struct DeleteFileKeyHash { - size_t operator()(const DeleteFileKey& key) const; + struct FilteredManifestDeletes { + std::vector> files; + int32_t duplicate_deletes_count = 0; }; - struct FoundDeletes { - std::unordered_set paths; - std::unordered_set files; - }; - - static DeleteFileKey MakeDeleteFileKey(const DataFile& file); - Result FilterManifest(const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, const ManifestFile& manifest, bool trust_manifest_references, - const ManifestWriterFactory& writer_factory, - FoundDeletes& found_deletes); + const ManifestWriterFactory& writer_factory); Result ManifestHasDeletedFiles(const std::vector& entries, const std::shared_ptr& schema, @@ -248,9 +245,12 @@ class ICEBERG_EXPORT ManifestFilterManager { Result FilterManifestWithDeletedFiles( const std::vector& entries, int32_t manifest_spec_id, const std::shared_ptr& schema, const PartitionSpecsById& specs_by_id, - const ManifestWriterFactory& writer_factory, FoundDeletes& found_deletes); + const ManifestWriterFactory& writer_factory); - Status ValidateRequiredDeletes(const FoundDeletes& found_deletes) const; + Status ValidateRequiredDeletes() const; + + Status InvalidateFilteredCache(); + void ResetDeletedFiles(); /// \brief Get or create a ManifestEvaluator for the given spec. Result GetManifestEvaluator(const std::shared_ptr& schema, @@ -270,19 +270,31 @@ class ICEBERG_EXPORT ManifestFilterManager { const ManifestContent manifest_content_; std::shared_ptr file_io_; + std::function delete_file_; std::shared_ptr delete_expr_; std::unordered_set delete_paths_; - std::unordered_set delete_file_keys_; - DataFileSet delete_files_; + // Delete files explicitly registered for deletion by object identity. + DeleteFileSet delete_files_to_delete_; + // Data files explicitly registered for deletion by object identity. + DataFileSet data_files_to_delete_; + // Data files to remove: explicit object deletes plus files found while filtering. + DataFileSet data_files_; + // Delete files to remove: explicit object deletes plus files found while filtering. + DeleteFileSet delete_files_; + std::unordered_map + filtered_manifest_to_deleted_files_; + // Ordered files deleted by the latest filter pass, used for summaries. std::vector> deleted_files_; - std::unordered_set deleted_file_keys_; + // Data-file identity set for latest-pass dedup and required-delete validation. + DataFileSet deleted_data_file_set_; + // Delete-file identity set for latest-pass dedup and required-delete validation. + DeleteFileSet deleted_delete_file_set_; PartitionSet drop_partitions_; bool fail_missing_delete_paths_{false}; bool fail_any_delete_{false}; bool case_sensitive_{true}; int32_t duplicate_deletes_count_{0}; - int32_t replaced_manifests_count_{0}; // minimum data sequence number; delete entries older than this are dropped @@ -290,6 +302,8 @@ class ICEBERG_EXPORT ManifestFilterManager { // paths of data files that were removed; DVs referencing these are dangling std::unordered_set removed_data_file_paths_; + std::unordered_map filtered_manifests_; + std::unordered_map> manifest_evaluator_cache_; std::unordered_map> diff --git a/src/iceberg/manifest/manifest_merge_manager.cc b/src/iceberg/manifest/manifest_merge_manager.cc index b924450cf..b2becf920 100644 --- a/src/iceberg/manifest/manifest_merge_manager.cc +++ b/src/iceberg/manifest/manifest_merge_manager.cc @@ -21,12 +21,15 @@ #include #include +#include #include #include #include +#include #include #include +#include "iceberg/file_io.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_reader.h" #include "iceberg/table_metadata.h" @@ -34,60 +37,87 @@ namespace iceberg { -ManifestMergeManager::ManifestMergeManager(int64_t target_size_bytes, - int32_t min_count_to_merge, bool merge_enabled) - : target_size_bytes_(target_size_bytes), +namespace { + +size_t CombineHash(size_t seed, size_t value) { + return seed ^ (value + 0x9e3779b9 + (seed << 6) + (seed >> 2)); +} + +} // namespace + +ManifestMergeManager::ManifestMergeManager( + ManifestContent content, int64_t target_size_bytes, int32_t min_count_to_merge, + bool merge_enabled, std::shared_ptr file_io, + SnapshotIdSupplier snapshot_id_supplier, + std::function delete_file) + : manifest_content_(content), + target_size_bytes_(target_size_bytes), min_count_to_merge_(min_count_to_merge), - merge_enabled_(merge_enabled) {} + merge_enabled_(merge_enabled), + file_io_(std::move(file_io)), + snapshot_id_supplier_(std::move(snapshot_id_supplier)), + delete_file_(std::move(delete_file)) { + ICEBERG_DCHECK(file_io_, "FileIO cannot be null"); + ICEBERG_DCHECK(snapshot_id_supplier_, "Snapshot ID supplier cannot be null"); + if (delete_file_ == nullptr) { + delete_file_ = [this](const std::string& location) { + return file_io_->DeleteFile(location); + }; + } +} + +Result> ManifestMergeManager::Make( + ManifestContent content, int64_t target_size_bytes, int32_t min_count_to_merge, + bool merge_enabled, std::shared_ptr file_io, + SnapshotIdSupplier snapshot_id_supplier, + std::function delete_file) { + ICEBERG_PRECHECK(file_io != nullptr, "FileIO cannot be null"); + ICEBERG_PRECHECK(snapshot_id_supplier != nullptr, + "Snapshot ID supplier cannot be null"); + return std::unique_ptr(new ManifestMergeManager( + content, target_size_bytes, min_count_to_merge, merge_enabled, std::move(file_io), + std::move(snapshot_id_supplier), std::move(delete_file))); +} Result> ManifestMergeManager::MergeManifests( const std::vector& existing_manifests, - const std::vector& new_manifests, int64_t snapshot_id, - const TableMetadata& metadata, std::shared_ptr file_io, + const std::vector& new_manifests, const TableMetadata& metadata, const ManifestWriterFactory& writer_factory) { - // Combine new then existing (new-first ordering is preserved in output). - auto to_manifest_ptr = [](const ManifestFile& manifest) { return &manifest; }; - auto manifest_ranges = std::array{ - new_manifests | std::views::transform(to_manifest_ptr), - existing_manifests | std::views::transform(to_manifest_ptr), + auto append_manifest = [this](const ManifestFile& manifest, + std::vector& manifests) -> Status { + ICEBERG_PRECHECK(manifest.content == manifest_content_, + "Cannot merge manifest with unexpected content"); + manifests.push_back(&manifest); + return {}; }; std::vector all; all.reserve(new_manifests.size() + existing_manifests.size()); - std::ranges::copy(manifest_ranges | std::views::join, std::back_inserter(all)); + for (const auto& manifest : new_manifests) { + ICEBERG_RETURN_UNEXPECTED(append_manifest(manifest, all)); + } + for (const auto& manifest : existing_manifests) { + ICEBERG_RETURN_UNEXPECTED(append_manifest(manifest, all)); + } if (all.empty() || !merge_enabled_) { - replaced_manifests_count_ = 0; return all | std::views::transform([](const ManifestFile* manifest) { return *manifest; }) | std::ranges::to>(); } - // Track the first (newest) manifest independently per content type. - std::map first_by_content; - std::ranges::for_each(all, [&first_by_content](const ManifestFile* manifest) { - first_by_content.try_emplace(manifest->content, manifest); - }); - - // Group manifests by (partition_spec_id, content), never merging across specs or - // content types. Reverse spec ordering preserves v3 first-row-id assignment order. - using GroupKey = std::pair; - auto group_key = [](const ManifestFile* manifest) { - return GroupKey{manifest->partition_spec_id, manifest->content}; - }; - - std::map, std::greater<>> by_spec; - std::ranges::for_each(all, [&by_spec, &group_key](const ManifestFile* manifest) { - by_spec[group_key(manifest)].push_back(manifest); + const auto* first = all.front(); + std::map, std::greater<>> by_spec; + std::ranges::for_each(all, [&by_spec](const ManifestFile* manifest) { + by_spec[manifest->partition_spec_id].push_back(manifest); }); std::vector result; result.reserve(all.size()); - replaced_manifests_count_ = 0; - for (auto& [key, group] : by_spec) { - const auto* first = first_by_content.at(key.second); - ICEBERG_ASSIGN_OR_RAISE(auto merged, MergeGroup(group, first, snapshot_id, metadata, - file_io, writer_factory)); + for (auto& [spec_id, group] : by_spec) { + std::ignore = spec_id; + ICEBERG_ASSIGN_OR_RAISE(auto merged, + MergeGroup(group, first, metadata, writer_factory)); std::ranges::move(merged, std::back_inserter(result)); } return result; @@ -95,8 +125,7 @@ Result> ManifestMergeManager::MergeManifests( Result> ManifestMergeManager::MergeGroup( const std::vector& group, const ManifestFile* first, - int64_t snapshot_id, const TableMetadata& metadata, std::shared_ptr file_io, - const ManifestWriterFactory& writer_factory) { + const TableMetadata& metadata, const ManifestWriterFactory& writer_factory) { // Match packEnd(group, ManifestFile::length) with lookback 1: // 1. Process manifests in reverse order (oldest-first). // 2. Greedy forward-pack with lookback=1: emit the current bin when the next item @@ -131,24 +160,29 @@ Result> ManifestMergeManager::MergeGroup( // pass its contents through unchanged. std::vector result; result.reserve(group.size()); - // TODO(Guotao): Flush independent bins in parallel and cache successful merged bins - // for commit retries. for (auto& bin : bins) { - bool contains_first = std::ranges::find(bin, first) != bin.end(); - if (contains_first && std::cmp_less(bin.size(), min_count_to_merge_)) { + if (bin.size() == 1) { + result.push_back(*bin[0]); + } else if (bool contains_first = std::ranges::find(bin, first) != bin.end(); + contains_first && std::cmp_less(bin.size(), min_count_to_merge_)) { for (const auto* manifest : bin) { result.push_back(*manifest); } } else { - ICEBERG_ASSIGN_OR_RAISE( - auto merged, FlushBin(bin, snapshot_id, metadata, file_io, writer_factory)); - if (bin.size() > 1) { - replaced_manifests_count_ += static_cast( - std::ranges::count_if(bin, [snapshot_id](const ManifestFile* manifest) { - return manifest->added_snapshot_id != snapshot_id; - })); + const auto* cached = merged_manifests_.Find(bin); + if (cached != nullptr) { + result.push_back(*cached); + } else { + const int64_t snapshot_id = snapshot_id_supplier_(); + ICEBERG_ASSIGN_OR_RAISE(auto merged, FlushBin(bin, metadata, writer_factory)); + merged_manifests_.Add(bin, merged); + for (const auto* manifest : bin) { + if (manifest->added_snapshot_id != snapshot_id) { + ++replaced_manifests_count_; + } + } + result.push_back(std::move(merged)); } - result.push_back(std::move(merged)); } } @@ -156,23 +190,22 @@ Result> ManifestMergeManager::MergeGroup( } Result ManifestMergeManager::FlushBin( - const std::vector& bin, int64_t snapshot_id, - const TableMetadata& metadata, std::shared_ptr file_io, + const std::vector& bin, const TableMetadata& metadata, const ManifestWriterFactory& writer_factory) { - // A single-manifest bin requires no merging. - if (bin.size() == 1) return *bin[0]; - const ManifestFile& first = *bin[0]; int32_t spec_id = first.partition_spec_id; ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata.PartitionSpecById(spec_id)); - ICEBERG_ASSIGN_OR_RAISE(auto writer, writer_factory(spec_id, first.content)); + ICEBERG_ASSIGN_OR_RAISE(auto writer, writer_factory(spec_id, manifest_content_)); + const int64_t snapshot_id = snapshot_id_supplier_(); for (const auto* manifest : bin) { - ICEBERG_ASSIGN_OR_RAISE(auto reader, - ManifestReader::Make(*manifest, file_io, schema, spec)); + bool is_committed = manifest->added_snapshot_id != kInvalidSnapshotId && + manifest->added_snapshot_id != snapshot_id; + ICEBERG_ASSIGN_OR_RAISE(auto reader, ManifestReader::Make(*manifest, file_io_, schema, + spec, is_committed)); ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); for (const auto& entry : entries) { bool is_current = @@ -196,4 +229,73 @@ Result ManifestMergeManager::FlushBin( return writer->ToManifestFile(); } +ManifestMergeManager::MergedManifestCache::Key +ManifestMergeManager::MergedManifestCache::MakeKey( + const std::vector& bin) { + Key key; + key.bin.reserve(bin.size()); + for (const auto* manifest : bin) { + key.bin.push_back(*manifest); + } + return key; +} + +const ManifestFile* ManifestMergeManager::MergedManifestCache::Find( + const std::vector& bin) const { + auto iter = entries_.find(MakeKey(bin)); + if (iter == entries_.end()) { + return nullptr; + } + return &iter->second; +} + +void ManifestMergeManager::MergedManifestCache::Add( + const std::vector& bin, const ManifestFile& manifest) { + entries_.emplace(MakeKey(bin), manifest); +} + +size_t ManifestMergeManager::MergedManifestCache::KeyHash::operator()( + const Key& key) const { + size_t hash = 0; + for (const auto& manifest : key.bin) { + hash = CombineHash(hash, std::hash{}(manifest.manifest_path)); + } + return hash; +} + +Result ManifestMergeManager::MergedManifestCache::CleanUncommitted( + const std::unordered_set& committed, int64_t snapshot_id, + const std::function& delete_file) { + int32_t removed_replaced_manifests_count = 0; + auto cached_entries = + std::vector>{entries_.begin(), entries_.end()}; + for (const auto& [bin, merged] : cached_entries) { + if (committed.contains(merged.manifest_path)) { + continue; + } + + std::ignore = delete_file(merged.manifest_path); + for (const auto& manifest : bin.bin) { + if (manifest.added_snapshot_id != snapshot_id) { + ++removed_replaced_manifests_count; + } + } + entries_.erase(bin); + } + return removed_replaced_manifests_count; +} + +Status ManifestMergeManager::CleanUncommitted( + const std::unordered_set& committed) { + if (merged_manifests_.empty()) { + return {}; + } + const int64_t snapshot_id = snapshot_id_supplier_(); + ICEBERG_ASSIGN_OR_RAISE( + auto removed_replaced_manifests_count, + merged_manifests_.CleanUncommitted(committed, snapshot_id, delete_file_)); + replaced_manifests_count_ -= removed_replaced_manifests_count; + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/manifest/manifest_merge_manager.h b/src/iceberg/manifest/manifest_merge_manager.h index 614ab61c6..031c31420 100644 --- a/src/iceberg/manifest/manifest_merge_manager.h +++ b/src/iceberg/manifest/manifest_merge_manager.h @@ -23,7 +23,11 @@ /// Merges small manifests into fewer larger ones according to table properties. #include +#include #include +#include +#include +#include #include #include "iceberg/iceberg_export.h" @@ -45,58 +49,87 @@ namespace iceberg { /// \note This class is non-copyable and non-movable. class ICEBERG_EXPORT ManifestMergeManager { public: + using SnapshotIdSupplier = std::function; + /// \brief Construct a merge manager with the given configuration. /// + /// \param content Manifest content this manager accepts /// \param target_size_bytes Target output manifest size in bytes /// \param min_count_to_merge Minimum number of manifests before any merging occurs /// \param merge_enabled Whether merging is enabled at all - ManifestMergeManager(int64_t target_size_bytes, int32_t min_count_to_merge, - bool merge_enabled); + /// \param file_io File IO used to open manifests for reading + /// \param snapshot_id_supplier Supplies the snapshot id being committed + /// \param delete_file Callback that deletes uncommitted merged manifest files + static Result> Make( + ManifestContent content, int64_t target_size_bytes, int32_t min_count_to_merge, + bool merge_enabled, std::shared_ptr file_io, + SnapshotIdSupplier snapshot_id_supplier, + std::function delete_file = {}); ManifestMergeManager(const ManifestMergeManager&) = delete; ManifestMergeManager& operator=(const ManifestMergeManager&) = delete; /// \brief Merge existing and new manifests according to configured thresholds. /// - /// Manifests are grouped by (partition_spec_id, content) — data and delete manifests - /// are never merged together. Within each group, a greedy bin-packing algorithm - /// combines manifests up to target_size_bytes. The bin that contains the newest - /// manifest for that content type is protected by min_count_to_merge: if it has fewer + /// Manifests are grouped by partition_spec_id. Within each group, a greedy + /// bin-packing algorithm combines manifests up to target_size_bytes. The bin that + /// contains the newest manifest is protected by min_count_to_merge: if it has fewer /// than that many items it is passed through unchanged. /// - /// \note Retry and rollback cleanup are handled by the caller that owns created - /// manifest paths. - /// TODO(Guotao): Add explicit replaced-manifest tracking here if callers need direct - /// access. - /// /// \param existing_manifests Manifests already in the base snapshot /// \param new_manifests Newly written manifests to incorporate - /// \param snapshot_id The ID of the snapshot being committed. Used to preserve - /// ADDED/DELETED status for entries written by this snapshot and to suppress - /// stale DELETED tombstones from prior snapshots. /// \param metadata Table metadata (provides specs and schema for readers) - /// \param file_io File IO used to open existing manifests for reading /// \param writer_factory Factory to create new ManifestWriter instances /// \return The merged manifest list, or an error Result> MergeManifests( const std::vector& existing_manifests, - const std::vector& new_manifests, int64_t snapshot_id, - const TableMetadata& metadata, std::shared_ptr file_io, + const std::vector& new_manifests, const TableMetadata& metadata, const ManifestWriterFactory& writer_factory); - /// \brief Returns the number of manifests replaced (consumed into merged outputs) - /// by the last MergeManifests() call. + /// \brief Returns the number of manifests replaced by cached merged outputs. int32_t ReplacedManifestsCount() const { return replaced_manifests_count_; } + /// \brief Delete cached merged manifests whose paths were not committed and roll + /// back replaced-manifest accounting. + Status CleanUncommitted(const std::unordered_set& committed); + private: + ManifestMergeManager(ManifestContent content, int64_t target_size_bytes, + int32_t min_count_to_merge, bool merge_enabled, + std::shared_ptr file_io, + SnapshotIdSupplier snapshot_id_supplier, + std::function delete_file); + + struct MergedManifestCache { + struct Key { + std::vector bin; + bool operator==(const Key& other) const = default; + }; + + struct KeyHash { + size_t operator()(const Key& key) const; + }; + + const ManifestFile* Find(const std::vector& bin) const; + void Add(const std::vector& bin, const ManifestFile& manifest); + bool empty() const { return entries_.empty(); } + Result CleanUncommitted( + const std::unordered_set& committed, int64_t snapshot_id, + const std::function& delete_file); + + private: + static Key MakeKey(const std::vector& bin); + + std::unordered_map entries_; + }; + /// \brief Merge a group of manifests sharing the same spec_id. /// /// \param first The overall first (newest) manifest across all groups, used to /// apply the min_count_to_merge threshold on the bin that contains it. Result> MergeGroup( const std::vector& group, const ManifestFile* first, - int64_t snapshot_id, const TableMetadata& metadata, std::shared_ptr file_io, - const ManifestWriterFactory& writer_factory); + const TableMetadata& metadata, const ManifestWriterFactory& writer_factory); /// \brief Write a merged manifest from all manifests in a bin. /// @@ -106,14 +139,18 @@ class ICEBERG_EXPORT ManifestMergeManager { /// - DELETED from older snapshots → dropped (stale tombstones are not carried forward) /// - All other entries → WriteExistingEntry Result FlushBin(const std::vector& bin, - int64_t snapshot_id, const TableMetadata& metadata, - std::shared_ptr file_io, + const TableMetadata& metadata, const ManifestWriterFactory& writer_factory); + const ManifestContent manifest_content_; const int64_t target_size_bytes_; const int32_t min_count_to_merge_; const bool merge_enabled_; + std::shared_ptr file_io_; + SnapshotIdSupplier snapshot_id_supplier_; + std::function delete_file_; int32_t replaced_manifests_count_{0}; + MergedManifestCache merged_manifests_; }; } // namespace iceberg diff --git a/src/iceberg/manifest/manifest_reader.cc b/src/iceberg/manifest/manifest_reader.cc index 7747e2be3..6c166df53 100644 --- a/src/iceberg/manifest/manifest_reader.cc +++ b/src/iceberg/manifest/manifest_reader.cc @@ -432,7 +432,7 @@ Status ParsePartitionValues(ArrowArrayView* view, int64_t row_idx, Status ParseDataFile(const std::shared_ptr& data_file_schema, ArrowArrayView* view, std::optional& first_row_id, - std::vector& manifest_entries) { + bool is_committed, std::vector& manifest_entries) { ICEBERG_RETURN_UNEXPECTED( AssertViewTypeAndChildren(view, ArrowType::NANOARROW_TYPE_STRUCT, data_file_schema->fields().size(), "data_file")); @@ -556,12 +556,14 @@ Status ParseDataFile(const std::shared_ptr& data_file_schema, first_row_id = first_row_id.value() + entry.data_file->record_count; } }); - } else { + } else if (is_committed) { // data file's first_row_id is null when the manifest's first_row_id is null std::ranges::for_each( - manifest_entries, [](auto& first_row_id) { first_row_id = std::nullopt; }, + manifest_entries, [](auto& row_id) { row_id = std::nullopt; }, proj_data_file(&DataFile::first_row_id)); } + // Preserve firstRowId for entries in uncommitted manifests, including EXISTING + // entries that may be merged later break; } case DataFile::kReferencedDataFileFieldId: @@ -589,7 +591,7 @@ Status ParseDataFile(const std::shared_ptr& data_file_schema, Result> ParseManifestEntry( ArrowSchema* arrow_schema, ArrowArray* array, const Schema& schema, - std::optional& first_row_id) { + std::optional& first_row_id, bool is_committed) { ArrowError error; ArrowArrayView view; ICEBERG_NANOARROW_RETURN_UNEXPECTED_WITH_ERROR( @@ -642,8 +644,8 @@ Result> ParseManifestEntry( case ManifestEntry::kDataFileFieldId: { auto data_file_schema = internal::checked_pointer_cast(field->get().type()); - ICEBERG_RETURN_UNEXPECTED( - ParseDataFile(data_file_schema, field_view, first_row_id, manifest_entries)); + ICEBERG_RETURN_UNEXPECTED(ParseDataFile( + data_file_schema, field_view, first_row_id, is_committed, manifest_entries)); break; } default: @@ -727,14 +729,15 @@ ManifestReaderImpl::ManifestReaderImpl( std::shared_ptr file_io, std::shared_ptr schema, std::shared_ptr spec, std::unique_ptr inheritable_metadata, - std::optional first_row_id) + std::optional first_row_id, bool is_committed) : manifest_path_(std::move(manifest_path)), manifest_length_(manifest_length), file_io_(std::move(file_io)), schema_(std::move(schema)), spec_(std::move(spec)), inheritable_metadata_(std::move(inheritable_metadata)), - first_row_id_(first_row_id) {} + first_row_id_(first_row_id), + is_committed_(is_committed) {} ManifestReader& ManifestReaderImpl::Select(const std::vector& columns) { columns_ = columns; @@ -907,8 +910,8 @@ Result> ManifestReaderImpl::ReadEntries(bool only_liv internal::ArrowArrayGuard array_guard(&result.value()); ICEBERG_ASSIGN_OR_RAISE( - auto entries, - ParseManifestEntry(&arrow_schema, &result.value(), *file_schema_, first_row_id_)); + auto entries, ParseManifestEntry(&arrow_schema, &result.value(), *file_schema_, + first_row_id_, is_committed_)); for (auto& entry : entries) { ICEBERG_RETURN_UNEXPECTED(inheritable_metadata_->Apply(entry)); @@ -984,7 +987,8 @@ Result ManifestFileFieldFromIndex(int32_t index) { Result> ManifestReader::Make( const ManifestFile& manifest, std::shared_ptr file_io, - std::shared_ptr schema, std::shared_ptr spec) { + std::shared_ptr schema, std::shared_ptr spec, + bool is_committed) { if (file_io == nullptr || schema == nullptr || spec == nullptr) { return InvalidArgument( "FileIO, Schema, and PartitionSpec cannot be null to create ManifestReader"); @@ -996,20 +1000,22 @@ Result> ManifestReader::Make( return std::make_unique( manifest.manifest_path, manifest.manifest_length, std::move(file_io), std::move(schema), std::move(spec), std::move(inheritable_metadata), - manifest.first_row_id); + manifest.first_row_id, is_committed); } Result> ManifestReader::Make( const ManifestFile& manifest, std::shared_ptr file_io, std::shared_ptr schema, - const std::unordered_map>& specs_by_id) { + const std::unordered_map>& specs_by_id, + bool is_committed) { auto spec_it = specs_by_id.find(manifest.partition_spec_id); if (spec_it == specs_by_id.end() || spec_it->second == nullptr) { return InvalidArgument("Partition spec {} not found for manifest {}", manifest.partition_spec_id, manifest.manifest_path); } auto spec = spec_it->second; - return Make(manifest, std::move(file_io), std::move(schema), std::move(spec)); + return Make(manifest, std::move(file_io), std::move(schema), std::move(spec), + is_committed); } Result> ManifestReader::Make( @@ -1017,7 +1023,7 @@ Result> ManifestReader::Make( std::shared_ptr file_io, std::shared_ptr schema, std::shared_ptr spec, std::unique_ptr inheritable_metadata, - std::optional first_row_id) { + std::optional first_row_id, bool is_committed) { ICEBERG_PRECHECK(file_io != nullptr, "FileIO cannot be null to read manifest"); ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null to read manifest"); ICEBERG_PRECHECK(spec != nullptr, "PartitionSpec cannot be null to read manifest"); @@ -1028,7 +1034,8 @@ Result> ManifestReader::Make( return std::make_unique( std::string(manifest_location), manifest_length, std::move(file_io), - std::move(schema), std::move(spec), std::move(inheritable_metadata), first_row_id); + std::move(schema), std::move(spec), std::move(inheritable_metadata), first_row_id, + is_committed); } Result> ManifestListReader::Make( diff --git a/src/iceberg/manifest/manifest_reader.h b/src/iceberg/manifest/manifest_reader.h index 42c56e1c2..b2d1c6505 100644 --- a/src/iceberg/manifest/manifest_reader.h +++ b/src/iceberg/manifest/manifest_reader.h @@ -87,21 +87,26 @@ class ICEBERG_EXPORT ManifestReader { /// \param file_io File IO implementation to use. /// \param schema Schema used to bind the partition type. /// \param spec Partition spec used for this manifest file. + /// \param is_committed Whether the manifest was committed by an older snapshot. /// \return A Result containing the reader or an error. - static Result> Make( - const ManifestFile& manifest, std::shared_ptr file_io, - std::shared_ptr schema, std::shared_ptr spec); + static Result> Make(const ManifestFile& manifest, + std::shared_ptr file_io, + std::shared_ptr schema, + std::shared_ptr spec, + bool is_committed = true); /// \brief Creates a reader for a manifest file using specs keyed by ID. /// \param manifest A ManifestFile object containing metadata about the manifest. /// \param file_io File IO implementation to use. /// \param schema Schema used to bind the partition type. /// \param specs_by_id Mapping of partition spec ID to PartitionSpec. + /// \param is_committed Whether the manifest was committed by an older snapshot. /// \return A Result containing the reader or an error. static Result> Make( const ManifestFile& manifest, std::shared_ptr file_io, std::shared_ptr schema, - const std::unordered_map>& specs_by_id); + const std::unordered_map>& specs_by_id, + bool is_committed = true); /// \brief Creates a reader for a manifest file. /// \param manifest_location Path to the manifest file. @@ -111,13 +116,14 @@ class ICEBERG_EXPORT ManifestReader { /// \param spec Partition spec used for this manifest file. /// \param inheritable_metadata Inheritable metadata. /// \param first_row_id First row ID to use for the manifest entries. + /// \param is_committed Whether the manifest was committed by an older snapshot. /// \return A Result containing the reader or an error. static Result> Make( std::string_view manifest_location, std::optional manifest_length, std::shared_ptr file_io, std::shared_ptr schema, std::shared_ptr spec, std::unique_ptr inheritable_metadata, - std::optional first_row_id = std::nullopt); + std::optional first_row_id = std::nullopt, bool is_committed = true); /// \brief Add stats columns to the column list if needed. static std::vector WithStatsColumns( diff --git a/src/iceberg/manifest/manifest_reader_internal.h b/src/iceberg/manifest/manifest_reader_internal.h index 2b4b1e0ba..53ce2fcb5 100644 --- a/src/iceberg/manifest/manifest_reader_internal.h +++ b/src/iceberg/manifest/manifest_reader_internal.h @@ -53,12 +53,13 @@ class ManifestReaderImpl : public ManifestReader { /// \param spec Partition spec. /// \param inheritable_metadata Metadata inherited from manifest. /// \param first_row_id First row ID for V3 manifests. + /// \param is_committed Whether the manifest was committed by an older snapshot. /// \note ManifestReader::Make() functions should guarantee non-null parameters. ManifestReaderImpl(std::string manifest_path, std::optional manifest_length, std::shared_ptr file_io, std::shared_ptr schema, std::shared_ptr spec, std::unique_ptr inheritable_metadata, - std::optional first_row_id); + std::optional first_row_id, bool is_committed); Result> Entries() override; @@ -106,6 +107,7 @@ class ManifestReaderImpl : public ManifestReader { const std::shared_ptr spec_; const std::unique_ptr inheritable_metadata_; std::optional first_row_id_; + bool is_committed_; // Configuration fields std::vector columns_; diff --git a/src/iceberg/snapshot.cc b/src/iceberg/snapshot.cc index d513e2be1..1b3182fd9 100644 --- a/src/iceberg/snapshot.cc +++ b/src/iceberg/snapshot.cc @@ -441,10 +441,6 @@ void SnapshotSummaryBuilder::Clear() { metrics_.Clear(); deleted_duplicate_files_ = 0; trust_partition_metrics_ = true; - manifests_counts_set_ = false; - manifests_created_ = 0; - manifests_kept_ = 0; - manifests_replaced_ = 0; } void SnapshotSummaryBuilder::SetPartitionSummaryLimit(int32_t max) { @@ -479,14 +475,6 @@ void SnapshotSummaryBuilder::Set(const std::string& property, const std::string& properties_[property] = value; } -void SnapshotSummaryBuilder::SetManifestCounts(int32_t created, int32_t kept, - int32_t replaced) { - manifests_counts_set_ = true; - manifests_created_ = created; - manifests_kept_ = kept; - manifests_replaced_ = replaced; -} - void SnapshotSummaryBuilder::Merge(const SnapshotSummaryBuilder& other) { for (const auto& [key, value] : other.properties_) { properties_[key] = value; @@ -503,10 +491,6 @@ void SnapshotSummaryBuilder::Merge(const SnapshotSummaryBuilder& other) { } deleted_duplicate_files_ += other.deleted_duplicate_files_; - // Manifest counts (manifests_counts_set_ / manifests_created_ / manifests_kept_ / - // manifests_replaced_) are intentionally not merged here. They are set directly - // on the root summary builder by Apply() after all manifests are finalized, and - // are never populated on sub-builders that get Merge()d in. } std::unordered_map SnapshotSummaryBuilder::Build() const { @@ -520,14 +504,6 @@ std::unordered_map SnapshotSummaryBuilder::Build() con SetIf(deleted_duplicate_files_ > 0, builder, SnapshotSummaryFields::kDeletedDuplicatedFiles, deleted_duplicate_files_); - // Always emit all three manifest count fields together when they have been set. - SetIf(manifests_counts_set_, builder, SnapshotSummaryFields::kManifestsCreated, - manifests_created_); - SetIf(manifests_counts_set_, builder, SnapshotSummaryFields::kManifestsKept, - manifests_kept_); - SetIf(manifests_counts_set_, builder, SnapshotSummaryFields::kManifestsReplaced, - manifests_replaced_); - SetIf(trust_partition_metrics_, builder, SnapshotSummaryFields::kChangedPartitionCountProp, partition_metrics_.size()); diff --git a/src/iceberg/snapshot.h b/src/iceberg/snapshot.h index 178c21dd7..f3e7ffb85 100644 --- a/src/iceberg/snapshot.h +++ b/src/iceberg/snapshot.h @@ -338,15 +338,6 @@ class ICEBERG_EXPORT SnapshotSummaryBuilder { /// \param value Property value void Set(const std::string& property, const std::string& value); - /// \brief Set manifest count summary fields. - /// - /// Records how many manifests were created, kept, and replaced in this snapshot. - /// - /// \param created Manifests written by this snapshot - /// \param kept Manifests carried over unchanged from the previous snapshot - /// \param replaced Manifests rewritten or merged away - void SetManifestCounts(int32_t created, int32_t kept, int32_t replaced); - /// \brief Merge another builder's metrics into this one /// /// \param other The builder to merge from @@ -368,10 +359,6 @@ class ICEBERG_EXPORT SnapshotSummaryBuilder { int32_t max_changed_partitions_for_summaries_{0}; int64_t deleted_duplicate_files_{0}; bool trust_partition_metrics_{true}; - bool manifests_counts_set_{false}; - int32_t manifests_created_{0}; - int32_t manifests_kept_{0}; - int32_t manifests_replaced_{0}; }; /// \brief Data operation that produce snapshots. diff --git a/src/iceberg/test/data_file_set_test.cc b/src/iceberg/test/data_file_set_test.cc index 60539adfa..e677b0e2e 100644 --- a/src/iceberg/test/data_file_set_test.cc +++ b/src/iceberg/test/data_file_set_test.cc @@ -38,6 +38,21 @@ class DataFileSetTest : public ::testing::Test { file->content = DataFile::Content::kData; return file; } + + std::shared_ptr CreateDeleteFile(const std::string& path) { + auto file = CreateDataFile(path); + file->content = DataFile::Content::kPositionDeletes; + return file; + } + + std::shared_ptr CreateDV(const std::string& path, int64_t offset, + int64_t size) { + auto file = CreateDeleteFile(path); + file->file_format = FileFormatType::kPuffin; + file->content_offset = offset; + file->content_size_in_bytes = size; + return file; + } }; TEST_F(DataFileSetTest, EmptySet) { @@ -260,12 +275,11 @@ TEST_F(DataFileSetTest, RangeBasedForLoop) { TEST_F(DataFileSetTest, CaseSensitivePaths) { DataFileSet set; auto file1 = CreateDataFile("/path/to/file.parquet"); - auto file2 = CreateDataFile("/path/to/FILE.parquet"); // Different case + auto file2 = CreateDataFile("/path/to/FILE.parquet"); set.insert(file1); set.insert(file2); - // Should be treated as different files EXPECT_EQ(set.size(), 2); } @@ -273,7 +287,6 @@ TEST_F(DataFileSetTest, MultipleInsertsSameFile) { DataFileSet set; auto file = CreateDataFile("/path/to/file.parquet"); - // Insert the same file multiple times set.insert(file); set.insert(file); set.insert(file); @@ -281,4 +294,64 @@ TEST_F(DataFileSetTest, MultipleInsertsSameFile) { EXPECT_EQ(set.size(), 1); } +TEST_F(DataFileSetTest, CopyRebuildsDataFileIndex) { + DataFileSet original; + original.insert(CreateDataFile("/path/to/file1.parquet")); + original.insert(CreateDataFile("/path/to/file2.parquet")); + + DataFileSet copy = original; + auto duplicate = CreateDataFile("/path/to/file1.parquet"); + + auto [iter, inserted] = copy.insert(duplicate); + EXPECT_FALSE(inserted); + ASSERT_NE(iter, copy.end()); + EXPECT_EQ((*iter)->file_path, "/path/to/file1.parquet"); + EXPECT_EQ(copy.size(), 2U); +} + +TEST_F(DataFileSetTest, DeleteFileSetDeduplicatesByPathForRegularDeletes) { + DeleteFileSet set; + auto first = CreateDeleteFile("/path/to/delete.parquet"); + auto duplicate = CreateDeleteFile("/path/to/delete.parquet"); + + auto [first_iter, first_inserted] = set.insert(first); + EXPECT_TRUE(first_inserted); + EXPECT_EQ(*first_iter, first); + + auto [duplicate_iter, duplicate_inserted] = set.insert(duplicate); + EXPECT_FALSE(duplicate_inserted); + EXPECT_EQ(*duplicate_iter, first); + EXPECT_EQ(set.size(), 1U); + EXPECT_TRUE(set.contains(*duplicate)); +} + +TEST_F(DataFileSetTest, DeleteFileSetDistinguishesDeletionVectorContentRanges) { + DeleteFileSet set; + auto first = CreateDV("/path/to/dv.puffin", /*offset=*/0, /*size=*/10); + auto same_range = CreateDV("/path/to/dv.puffin", /*offset=*/0, /*size=*/10); + auto different_offset = CreateDV("/path/to/dv.puffin", /*offset=*/10, /*size=*/10); + auto different_size = CreateDV("/path/to/dv.puffin", /*offset=*/0, /*size=*/20); + + EXPECT_TRUE(set.insert(first).second); + EXPECT_FALSE(set.insert(same_range).second); + EXPECT_TRUE(set.insert(different_offset).second); + EXPECT_TRUE(set.insert(different_size).second); + EXPECT_EQ(set.size(), 3U); +} + +TEST_F(DataFileSetTest, DeleteFileSetCopyRebuildsIndex) { + DeleteFileSet original; + original.insert(CreateDV("/path/to/dv.puffin", /*offset=*/0, /*size=*/10)); + original.insert(CreateDV("/path/to/dv.puffin", /*offset=*/10, /*size=*/10)); + + DeleteFileSet copy = original; + auto duplicate = CreateDV("/path/to/dv.puffin", /*offset=*/0, /*size=*/10); + + auto [iter, inserted] = copy.insert(duplicate); + EXPECT_FALSE(inserted); + ASSERT_NE(iter, copy.end()); + EXPECT_EQ((*iter)->content_offset, 0); + EXPECT_EQ(copy.size(), 2U); +} + } // namespace iceberg diff --git a/src/iceberg/test/fast_append_test.cc b/src/iceberg/test/fast_append_test.cc index 6c77fad16..98956ba7c 100644 --- a/src/iceberg/test/fast_append_test.cc +++ b/src/iceberg/test/fast_append_test.cc @@ -160,6 +160,18 @@ TEST_F(FastAppendTest, AppendNullFile) { EXPECT_THAT(table_->current_snapshot(), HasErrorMessage("No current snapshot")); } +TEST_F(FastAppendTest, FinalizeIgnoresCleanupDeleteFailure) { + std::shared_ptr fast_append; + ICEBERG_UNWRAP_OR_FAIL(fast_append, table_->NewFastAppend()); + fast_append->AppendFile(file_a_); + fast_append->DeleteWith([](const std::string&) { return IOError("delete failed"); }); + + EXPECT_THAT(static_cast(*fast_append).Apply(), IsOk()); + EXPECT_THAT(fast_append->Finalize(Result( + std::unexpected(CommitFailed("commit failed").error()))), + IsOk()); +} + TEST_F(FastAppendTest, AppendDuplicateFile) { std::shared_ptr fast_append; ICEBERG_UNWRAP_OR_FAIL(fast_append, table_->NewFastAppend()); @@ -170,7 +182,6 @@ TEST_F(FastAppendTest, AppendDuplicateFile) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - // Should only count the file once EXPECT_EQ(snapshot->summary.at("added-data-files"), "1"); EXPECT_EQ(snapshot->summary.at("added-records"), "100"); } diff --git a/src/iceberg/test/manifest_filter_manager_test.cc b/src/iceberg/test/manifest_filter_manager_test.cc index 2f2840636..442bbe063 100644 --- a/src/iceberg/test/manifest_filter_manager_test.cc +++ b/src/iceberg/test/manifest_filter_manager_test.cc @@ -128,29 +128,99 @@ class ManifestFilterManagerTest : public MinimalUpdateTestBase { }; TEST_F(ManifestFilterManagerTest, NullSnapshotReturnsEmpty) { - ManifestFilterManager mgr(ManifestContent::kData, file_io_); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, nullptr, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, nullptr, factory)); EXPECT_TRUE(result.empty()); } +TEST_F(ManifestFilterManagerTest, MakeRejectsNullFileIO) { + EXPECT_THAT(ManifestFilterManager::Make(ManifestContent::kData, nullptr), + IsError(ErrorKind::kInvalidArgument)); +} + TEST_F(ManifestFilterManagerTest, ContainsDeletesReturnsCorrectState) { - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - EXPECT_FALSE(mgr.ContainsDeletes()); - mgr.DeleteFile("/some/path.parquet"); - EXPECT_TRUE(mgr.ContainsDeletes()); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + EXPECT_FALSE(mgr->ContainsDeletes()); + ASSERT_THAT(mgr->DeleteFile("/some/path.parquet"), IsOk()); + EXPECT_TRUE(mgr->ContainsDeletes()); +} + +TEST_F(ManifestFilterManagerTest, ContainsDeletesTrueAfterRowFilter) { + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + EXPECT_FALSE(mgr->ContainsDeletes()); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::Equal("x", Literal::Long(1L))), IsOk()); + EXPECT_TRUE(mgr->ContainsDeletes()); +} + +TEST_F(ManifestFilterManagerTest, ContainsDeletesFalseForAlwaysFalseRowFilter) { + // An always-false row filter does not count as a delete condition. + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::AlwaysFalse()), IsOk()); + EXPECT_FALSE(mgr->ContainsDeletes()); +} + +TEST_F(ManifestFilterManagerTest, ContainsDeletesTrueAfterDropPartition) { + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + EXPECT_FALSE(mgr->ContainsDeletes()); + ASSERT_THAT( + mgr->DropPartition(spec_->spec_id(), + PartitionValues(std::vector{Literal::Long(1L)})), + IsOk()); + EXPECT_TRUE(mgr->ContainsDeletes()); +} + +TEST_F(ManifestFilterManagerTest, ContainsDeletesTrueAfterDeleteFileObject) { + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + EXPECT_FALSE(mgr->ContainsDeletes()); + EXPECT_THAT(mgr->DeleteFile(file_a_), IsOk()); + EXPECT_TRUE(mgr->ContainsDeletes()); +} + +TEST_F(ManifestFilterManagerTest, FilesToBeDeletedIncludesRegisteredFileObject) { + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + + EXPECT_THAT(mgr->DeleteFile(file_a_), IsOk()); + + ASSERT_EQ(mgr->FilesToBeDeleted().size(), 1U); + EXPECT_EQ(mgr->FilesToBeDeleted().begin()->get()->file_path, file_a_->file_path); +} + +TEST_F(ManifestFilterManagerTest, FilesToBeDeletedKeepsRegisteredFileObjectAfterFilter) { + ICEBERG_UNWRAP_OR_FAIL(auto snap, CommitFiles({file_b_})); + auto factory = MakeWriterFactory(*table_->metadata()); + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + EXPECT_THAT(mgr->DeleteFile(file_a_), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr->FilterManifests(*table_->metadata(), snap, factory)); + EXPECT_THAT(result, ::testing::Not(::testing::IsEmpty())); + + ASSERT_EQ(mgr->FilesToBeDeleted().size(), 1U); + EXPECT_EQ(mgr->FilesToBeDeleted().begin()->get()->file_path, file_a_->file_path); } TEST_F(ManifestFilterManagerTest, DeleteByRowFilterRejectsNull) { - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - EXPECT_THAT(mgr.DeleteByRowFilter(nullptr), IsError(ErrorKind::kInvalidArgument)); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + EXPECT_THAT(mgr->DeleteByRowFilter(nullptr), IsError(ErrorKind::kInvalidArgument)); } TEST_F(ManifestFilterManagerTest, DeleteFileObjectRejectsNull) { - ManifestFilterManager mgr(ManifestContent::kData, file_io_); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); std::shared_ptr null_file; - EXPECT_THAT(mgr.DeleteFile(null_file), IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(mgr->DeleteFile(null_file), IsError(ErrorKind::kInvalidArgument)); } TEST_F(ManifestFilterManagerTest, NoConditionsReturnsManifestsUnchanged) { @@ -163,12 +233,12 @@ TEST_F(ManifestFilterManagerTest, NoConditionsReturnsManifestsUnchanged) { ManifestListReader::Make(snap->manifest_list, file_io_)); ICEBERG_UNWRAP_OR_FAIL(auto orig_manifests, list_reader->Files()); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ASSERT_EQ(result.size(), orig_manifests.size()); for (size_t i = 0; i < result.size(); ++i) { - // No rewrite → same manifest path EXPECT_EQ(result[i].manifest_path, orig_manifests[i].manifest_path); } } @@ -178,10 +248,11 @@ TEST_F(ManifestFilterManagerTest, DeleteFileByPath) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - mgr.DeleteFile(file_a_->file_path); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); int deleted_count = 0; @@ -199,6 +270,146 @@ TEST_F(ManifestFilterManagerTest, DeleteFileByPath) { EXPECT_EQ(live_count, 1); } +TEST_F(ManifestFilterManagerTest, FilterManifestsCachesFilteredManifestAcrossRetries) { + ICEBERG_UNWRAP_OR_FAIL(auto snap, CommitFiles({file_a_, file_b_})); + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto list_reader, + ManifestListReader::Make(snap->manifest_list, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto manifest_files, list_reader->Files()); + std::vector manifests; + manifests.reserve(manifest_files.size()); + for (const auto& manifest : manifest_files) { + manifests.push_back(&manifest); + } + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + auto specs = SpecsById(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto first, + mgr->FilterManifests(schema, specs, manifests, factory)); + ASSERT_EQ(first.size(), 1U); + EXPECT_NE(first[0].manifest_path, manifest_files[0].manifest_path); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 1); + int after_first = manifest_counter_; + + ICEBERG_UNWRAP_OR_FAIL(auto second, + mgr->FilterManifests(schema, specs, manifests, factory)); + ASSERT_EQ(second.size(), 1U); + EXPECT_EQ(second[0].manifest_path, first[0].manifest_path); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 1); + EXPECT_EQ(manifest_counter_, after_first); +} + +TEST_F(ManifestFilterManagerTest, DeleteFileInvalidatesFilteredCache) { + ICEBERG_UNWRAP_OR_FAIL(auto snap, CommitFiles({file_a_, file_b_})); + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto list_reader, + ManifestListReader::Make(snap->manifest_list, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto manifest_files, list_reader->Files()); + std::vector manifests; + manifests.reserve(manifest_files.size()); + for (const auto& manifest : manifest_files) { + manifests.push_back(&manifest); + } + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + auto specs = SpecsById(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto first, + mgr->FilterManifests(schema, specs, manifests, factory)); + ASSERT_EQ(first.size(), 1U); + + ASSERT_THAT(mgr->DeleteFile(file_b_->file_path), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto second, + mgr->FilterManifests(schema, specs, manifests, factory)); + EXPECT_NE(second[0].manifest_path, first[0].manifest_path); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(second, *metadata)); + auto deleted_count = + std::count_if(entries.begin(), entries.end(), [](const ManifestEntry& entry) { + return entry.status == ManifestStatus::kDeleted; + }); + EXPECT_EQ(deleted_count, 2); +} + +TEST_F(ManifestFilterManagerTest, CleanUncommittedDropsFilteredCacheAndRollsBackCount) { + ICEBERG_UNWRAP_OR_FAIL(auto snap, CommitFiles({file_a_, file_b_})); + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto list_reader, + ManifestListReader::Make(snap->manifest_list, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto manifest_files, list_reader->Files()); + std::vector manifests; + manifests.reserve(manifest_files.size()); + for (const auto& manifest : manifest_files) { + manifests.push_back(&manifest); + } + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + auto specs = SpecsById(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto first, + mgr->FilterManifests(schema, specs, manifests, factory)); + ASSERT_EQ(first.size(), 1U); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 1); + EXPECT_THAT(mgr->CleanUncommitted({}), IsOk()); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 0); + + int after_cleanup = manifest_counter_; + ICEBERG_UNWRAP_OR_FAIL(auto second, + mgr->FilterManifests(schema, specs, manifests, factory)); + ASSERT_EQ(second.size(), 1U); + EXPECT_NE(second[0].manifest_path, first[0].manifest_path); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 1); + EXPECT_GT(manifest_counter_, after_cleanup); +} + +TEST_F(ManifestFilterManagerTest, CleanUncommittedIgnoresDeleteCallbackError) { + ICEBERG_UNWRAP_OR_FAIL(auto snap, CommitFiles({file_a_, file_b_})); + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto list_reader, + ManifestListReader::Make(snap->manifest_list, file_io_)); + ICEBERG_UNWRAP_OR_FAIL(auto manifest_files, list_reader->Files()); + std::vector manifests; + manifests.reserve(manifest_files.size()); + for (const auto& manifest : manifest_files) { + manifests.push_back(&manifest); + } + + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make( + ManifestContent::kData, file_io_, + [](const std::string&) { return IOError("delete failed"); })); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + auto specs = SpecsById(*metadata); + + ICEBERG_UNWRAP_OR_FAIL(auto first, + mgr->FilterManifests(schema, specs, manifests, factory)); + ASSERT_EQ(first.size(), 1U); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 1); + + EXPECT_THAT(mgr->CleanUncommitted({}), IsOk()); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 0); +} + TEST_F(ManifestFilterManagerTest, ExplicitContextFilterManifestsDeletesByPath) { ICEBERG_UNWRAP_OR_FAIL(auto snap, CommitFiles({file_a_, file_b_})); auto* metadata = table_->metadata().get(); @@ -213,11 +424,12 @@ TEST_F(ManifestFilterManagerTest, ExplicitContextFilterManifestsDeletesByPath) { manifests.push_back(&manifest); } - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - mgr.DeleteFile(file_a_->file_path); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(schema_, SpecsById(*metadata), - manifests, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(schema_, SpecsById(*metadata), + manifests, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); int deleted_count = 0; @@ -236,10 +448,11 @@ TEST_F(ManifestFilterManagerTest, RowFilterAlwaysTrueDeletesAll) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - ASSERT_THAT(mgr.DeleteByRowFilter(Expressions::AlwaysTrue()), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::AlwaysTrue()), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); for (const auto& e : entries) { @@ -252,14 +465,14 @@ TEST_F(ManifestFilterManagerTest, RowFilterAlwaysFalseDeletesNone) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - ASSERT_THAT(mgr.DeleteByRowFilter(Expressions::AlwaysFalse()), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::AlwaysFalse()), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); for (const auto& e : entries) { - // AlwaysFalse means nothing can match → entries remain ADDED or EXISTING EXPECT_NE(e.status, ManifestStatus::kDeleted) << "Expected no entries to be DELETED"; } } @@ -269,11 +482,12 @@ TEST_F(ManifestFilterManagerTest, RowFilterUsesPartitionResiduals) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - mgr.CaseSensitive(false); - ASSERT_THAT(mgr.DeleteByRowFilter(Expressions::Equal("X", Literal::Long(1L))), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + mgr->CaseSensitive(false); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::Equal("X", Literal::Long(1L))), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); int deleted_count = 0; @@ -291,8 +505,8 @@ TEST_F(ManifestFilterManagerTest, RowFilterUsesPartitionResiduals) { EXPECT_EQ(deleted_count, 1); EXPECT_EQ(live_count, 1); - ASSERT_EQ(mgr.FilesToBeDeleted().size(), 1U); - EXPECT_EQ(mgr.FilesToBeDeleted().begin()->get()->file_path, file_a_->file_path); + ASSERT_EQ(mgr->FilesToBeDeleted().size(), 1U); + EXPECT_EQ(mgr->FilesToBeDeleted().begin()->get()->file_path, file_a_->file_path); } TEST_F(ManifestFilterManagerTest, DropPartition) { @@ -301,11 +515,14 @@ TEST_F(ManifestFilterManagerTest, DropPartition) { auto factory = MakeWriterFactory(*metadata); // Drop partition of file_a (partition_x = 1) - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - mgr.DropPartition(spec_->spec_id(), - PartitionValues(std::vector{Literal::Long(1L)})); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT( + mgr->DropPartition(spec_->spec_id(), + PartitionValues(std::vector{Literal::Long(1L)})), + IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); int deleted_count = 0; @@ -324,12 +541,14 @@ TEST_F(ManifestFilterManagerTest, FailMissingDeletePathsReturnsError) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - mgr.DeleteFile("/does/not/exist.parquet"); - mgr.FailMissingDeletePaths(); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile("/does/not/exist.parquet"), IsOk()); + mgr->FailMissingDeletePaths(); - auto result = mgr.FilterManifests(*metadata, snap, factory); - EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); + auto result = mgr->FilterManifests(*metadata, snap, factory); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Missing required files to delete")); } TEST_F(ManifestFilterManagerTest, FailAnyDeleteReportsPartitionPath) { @@ -337,12 +556,13 @@ TEST_F(ManifestFilterManagerTest, FailAnyDeleteReportsPartitionPath) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - mgr.DeleteFile(file_a_->file_path); - mgr.FailAnyDelete(); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); + mgr->FailAnyDelete(); - auto result = mgr.FilterManifests(*metadata, snap, factory); - EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument)); + auto result = mgr->FilterManifests(*metadata, snap, factory); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(result, HasErrorMessage("x=1")); } @@ -352,11 +572,12 @@ TEST_F(ManifestFilterManagerTest, MultipleConditionsOrCombined) { auto factory = MakeWriterFactory(*metadata); // Both files should be deleted: file_a by path, file_b by AlwaysTrue expression - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - mgr.DeleteFile(file_a_->file_path); - ASSERT_THAT(mgr.DeleteByRowFilter(Expressions::AlwaysTrue()), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteFile(file_a_->file_path), IsOk()); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::AlwaysTrue()), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); for (const auto& e : entries) { @@ -369,11 +590,12 @@ TEST_F(ManifestFilterManagerTest, MultipleRowFiltersUseCombinedExpression) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - ManifestFilterManager mgr(ManifestContent::kData, file_io_); - ASSERT_THAT(mgr.DeleteByRowFilter(Expressions::Equal("y", Literal::Long(7L))), IsOk()); - ASSERT_THAT(mgr.DeleteByRowFilter(Expressions::Equal("x", Literal::Long(1L))), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestFilterManager::Make(ManifestContent::kData, file_io_)); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::Equal("y", Literal::Long(7L))), IsOk()); + ASSERT_THAT(mgr->DeleteByRowFilter(Expressions::Equal("x", Literal::Long(1L))), IsOk()); - ICEBERG_UNWRAP_OR_FAIL(auto result, mgr.FilterManifests(*metadata, snap, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(*metadata, snap, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); ASSERT_EQ(entries.size(), 1U); @@ -382,10 +604,11 @@ TEST_F(ManifestFilterManagerTest, MultipleRowFiltersUseCombinedExpression) { // Helper: write one or more delete-file entries to a new manifest. // Each entry is (DataFile, data_sequence_number). -using DeleteManifestEntry = std::pair, int64_t>; +using DeleteFileWithSequenceNumber = std::pair, int64_t>; static Result WriteDeleteManifest( - const std::vector& files, std::shared_ptr file_io, - const TableMetadata& metadata, const std::string& path) { + const std::vector& files, + std::shared_ptr file_io, const TableMetadata& metadata, + const std::string& path) { if (files.empty()) { return InvalidArgument("WriteDeleteManifest requires at least one entry"); } @@ -438,26 +661,31 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanDoesNotRewriteOnItsOwn auto del_manifest, WriteDeleteManifest(del_file, /*data_seq=*/2L, file_io_, *metadata, manifest_path)); - ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); - mgr.DropDeleteFilesOlderThan(5); + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + EXPECT_THAT(mgr->DropDeleteFilesOlderThan(5), IsOk()); std::vector manifests{&del_manifest}; auto specs = SpecsById(*metadata); ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.FilterManifests(schema, specs, manifests, factory)); + mgr->FilterManifests(schema, specs, manifests, factory)); ASSERT_EQ(result.size(), 1U); EXPECT_EQ(result[0].manifest_path, del_manifest.manifest_path); } +TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanRejectsNegativeSequenceNumber) { + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + EXPECT_THAT(mgr->DropDeleteFilesOlderThan(-1), IsError(ErrorKind::kInvalidArgument)); +} + TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanDuringDeleteManifestRewrite) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); - // Three entries in the same manifest: old (seq=2, below threshold), targeted - // (explicit path delete), and keep (survives the rewrite). auto make_del_file = [&](const std::string& path) { auto f = std::make_shared(); f->content = DataFile::Content::kPositionDeletes; @@ -480,16 +708,17 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanDuringDeleteManifestRe WriteDeleteManifest({{old_file, 2L}, {targeted_file, 10L}, {keep_file, 10L}}, file_io_, *metadata, manifest_path)); - ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); - mgr.DeleteFile(targeted_file->file_path); - mgr.DropDeleteFilesOlderThan(5); + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + ASSERT_THAT(mgr->DeleteFile(targeted_file->file_path), IsOk()); + EXPECT_THAT(mgr->DropDeleteFilesOlderThan(5), IsOk()); std::vector manifests{&del_manifest}; auto specs = SpecsById(*metadata); ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.FilterManifests(schema, specs, manifests, factory)); + mgr->FilterManifests(schema, specs, manifests, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); ASSERT_EQ(entries.size(), 3U); @@ -512,6 +741,100 @@ TEST_F(ManifestFilterManagerTest, DropDeleteFilesOlderThanDuringDeleteManifestRe } } +TEST_F(ManifestFilterManagerTest, FailAnyDeleteFailsOnSequenceNumberPruning) { + // An unrelated object delete opens the manifest; sequence-number pruning is the + // branch that marks this entry. + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + auto old_file = std::make_shared(); + old_file->content = DataFile::Content::kPositionDeletes; + old_file->file_path = table_location_ + "/delete/del_old.parquet"; + old_file->file_format = FileFormatType::kParquet; + old_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + old_file->file_size_in_bytes = 512; + old_file->record_count = 10; + old_file->partition_spec_id = spec_->spec_id(); + + auto manifest_path = std::format("{}/metadata/del-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto del_manifest, + WriteDeleteManifest(old_file, /*data_seq=*/2L, file_io_, *metadata, manifest_path)); + + // Unrelated registered delete file: opens the manifest without matching its entry. + auto unrelated = std::make_shared(); + unrelated->content = DataFile::Content::kPositionDeletes; + unrelated->file_path = table_location_ + "/delete/unrelated.parquet"; + unrelated->file_format = FileFormatType::kParquet; + unrelated->partition = PartitionValues(std::vector{Literal::Long(9L)}); + unrelated->file_size_in_bytes = 512; + unrelated->record_count = 10; + unrelated->partition_spec_id = spec_->spec_id(); + + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + EXPECT_THAT(mgr->DeleteFile(unrelated), IsOk()); + EXPECT_THAT(mgr->DropDeleteFilesOlderThan(5), IsOk()); + mgr->FailAnyDelete(); + + std::vector manifests{&del_manifest}; + auto specs = SpecsById(*metadata); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + + auto result = mgr->FilterManifests(schema, specs, manifests, factory); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Operation would delete existing data")); +} + +TEST_F(ManifestFilterManagerTest, FailAnyDeleteFailsOnDanglingDeletionVector) { + // Dangling DVs honor fail-any-delete just like other delete branches. + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + const std::string data_file_path = table_location_ + "/data/referenced.parquet"; + + auto dv_file = std::make_shared(); + dv_file->content = DataFile::Content::kPositionDeletes; + dv_file->file_path = table_location_ + "/delete/dv.puffin"; + dv_file->file_format = FileFormatType::kPuffin; + dv_file->referenced_data_file = data_file_path; + dv_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + dv_file->file_size_in_bytes = 256; + dv_file->record_count = 5; + dv_file->partition_spec_id = spec_->spec_id(); + + auto manifest_path = std::format("{}/metadata/dv-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto dv_manifest, + WriteDeleteManifest(dv_file, /*data_seq=*/3L, file_io_, *metadata, manifest_path)); + + auto deleted_data_file = std::make_shared(); + deleted_data_file->content = DataFile::Content::kData; + deleted_data_file->file_path = data_file_path; + deleted_data_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + deleted_data_file->file_size_in_bytes = 1024; + deleted_data_file->record_count = 50; + deleted_data_file->partition_spec_id = spec_->spec_id(); + + DataFileSet deleted_files; + deleted_files.insert(deleted_data_file); + + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + mgr->RemoveDanglingDeletesFor(deleted_files); + mgr->FailAnyDelete(); + + std::vector manifests{&dv_manifest}; + auto specs = SpecsById(*metadata); + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + + auto result = mgr->FilterManifests(schema, specs, manifests, factory); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Operation would delete existing data")); +} + TEST_F(ManifestFilterManagerTest, RemoveDanglingDeletesForFiltersDanglingDV) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); @@ -547,21 +870,163 @@ TEST_F(ManifestFilterManagerTest, RemoveDanglingDeletesForFiltersDanglingDV) { DataFileSet deleted_files; deleted_files.insert(deleted_data_file); - ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); - mgr.RemoveDanglingDeletesFor(deleted_files); + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + mgr->RemoveDanglingDeletesFor(deleted_files); std::vector manifests{&dv_manifest}; auto specs = SpecsById(*metadata); ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.FilterManifests(schema, specs, manifests, factory)); + mgr->FilterManifests(schema, specs, manifests, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, *metadata)); ASSERT_EQ(entries.size(), 1U); EXPECT_EQ(entries[0].status, ManifestStatus::kDeleted); } +TEST_F(ManifestFilterManagerTest, RemoveDanglingDeletesForKeepsNonDanglingDeletes) { + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + const std::string deleted_data_file_path = table_location_ + "/data/deleted.parquet"; + + auto make_delete = [&](std::string path, FileFormatType format, + std::optional referenced_data_file) { + auto f = std::make_shared(); + f->content = DataFile::Content::kPositionDeletes; + f->file_path = std::move(path); + f->file_format = format; + f->referenced_data_file = std::move(referenced_data_file); + f->partition = PartitionValues(std::vector{Literal::Long(1L)}); + f->file_size_in_bytes = 256; + f->record_count = 5; + f->partition_spec_id = spec_->spec_id(); + return f; + }; + + auto ordinary_position_delete = + make_delete(table_location_ + "/delete/ordinary.parquet", FileFormatType::kParquet, + deleted_data_file_path); + auto dv_without_reference = make_delete(table_location_ + "/delete/no-ref.puffin", + FileFormatType::kPuffin, std::nullopt); + auto unrelated_dv = + make_delete(table_location_ + "/delete/unrelated.puffin", FileFormatType::kPuffin, + table_location_ + "/data/other.parquet"); + + auto manifest_path = std::format("{}/metadata/dv-keep-manifest-{}.avro", + table_location_, manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL(auto manifest, + WriteDeleteManifest({{ordinary_position_delete, 3L}, + {dv_without_reference, 3L}, + {unrelated_dv, 3L}}, + file_io_, *metadata, manifest_path)); + + auto deleted_data_file = std::make_shared(); + deleted_data_file->content = DataFile::Content::kData; + deleted_data_file->file_path = deleted_data_file_path; + deleted_data_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + deleted_data_file->file_size_in_bytes = 1024; + deleted_data_file->record_count = 50; + deleted_data_file->partition_spec_id = spec_->spec_id(); + + DataFileSet deleted_files; + deleted_files.insert(deleted_data_file); + + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + mgr->RemoveDanglingDeletesFor(deleted_files); + + std::vector manifests{&manifest}; + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(schema, SpecsById(*metadata), + manifests, factory)); + + ASSERT_EQ(result.size(), 1U); + EXPECT_EQ(result[0].manifest_path, manifest.manifest_path); +} + +TEST_F(ManifestFilterManagerTest, RemoveDanglingDeletesForReplacesRemovedFileSet) { + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + auto make_dv = [&](std::string path, std::string referenced_data_file) { + auto f = std::make_shared(); + f->content = DataFile::Content::kPositionDeletes; + f->file_path = std::move(path); + f->file_format = FileFormatType::kPuffin; + f->referenced_data_file = std::move(referenced_data_file); + f->partition = PartitionValues(std::vector{Literal::Long(1L)}); + f->file_size_in_bytes = 256; + f->record_count = 5; + f->partition_spec_id = spec_->spec_id(); + return f; + }; + + const std::string first_data = table_location_ + "/data/first.parquet"; + const std::string second_data = table_location_ + "/data/second.parquet"; + auto first_dv = make_dv(table_location_ + "/delete/first.puffin", first_data); + auto second_dv = make_dv(table_location_ + "/delete/second.puffin", second_data); + + auto first_manifest_path = + std::format("{}/metadata/dv-first-{}.avro", table_location_, manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto first_manifest, + WriteDeleteManifest(first_dv, 3L, file_io_, *metadata, first_manifest_path)); + auto second_manifest_path = + std::format("{}/metadata/dv-second-{}.avro", table_location_, manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto second_manifest, + WriteDeleteManifest(second_dv, 3L, file_io_, *metadata, second_manifest_path)); + + auto make_deleted_data = [&](std::string path) { + auto file = std::make_shared(); + file->content = DataFile::Content::kData; + file->file_path = std::move(path); + file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + file->file_size_in_bytes = 1024; + file->record_count = 50; + file->partition_spec_id = spec_->spec_id(); + return file; + }; + + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + DataFileSet first_deleted; + first_deleted.insert(make_deleted_data(first_data)); + mgr->RemoveDanglingDeletesFor(first_deleted); + + DataFileSet second_deleted; + second_deleted.insert(make_deleted_data(second_data)); + mgr->RemoveDanglingDeletesFor(second_deleted); + + std::vector manifests{&first_manifest, &second_manifest}; + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + ICEBERG_UNWRAP_OR_FAIL( + auto second_result, + mgr->FilterManifests(schema, SpecsById(*metadata), manifests, factory)); + ASSERT_EQ(second_result.size(), 2U); + EXPECT_EQ(second_result[0].manifest_path, first_manifest.manifest_path); + EXPECT_NE(second_result[1].manifest_path, second_manifest.manifest_path); + ICEBERG_UNWRAP_OR_FAIL(auto second_entries, ReadAllEntries(second_result, *metadata)); + + bool saw_first_live = false; + bool saw_second_deleted = false; + for (const auto& entry : second_entries) { + ASSERT_NE(entry.data_file, nullptr); + if (entry.data_file->file_path == first_dv->file_path) { + saw_first_live = true; + EXPECT_NE(entry.status, ManifestStatus::kDeleted); + } else if (entry.data_file->file_path == second_dv->file_path) { + saw_second_deleted = true; + EXPECT_EQ(entry.status, ManifestStatus::kDeleted); + } + } + EXPECT_TRUE(saw_first_live); + EXPECT_TRUE(saw_second_deleted); +} + TEST_F(ManifestFilterManagerTest, DeleteFileObjectMatchesDeletionVectorByContent) { auto metadata = *table_->metadata(); metadata.format_version = 3; @@ -591,13 +1056,14 @@ TEST_F(ManifestFilterManagerTest, DeleteFileObjectMatchesDeletionVectorByContent auto manifest, WriteDeleteManifest({{dv0, 3L}, {dv1, 3L}}, file_io_, metadata, manifest_path)); - ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); - EXPECT_THAT(mgr.DeleteFile(dv0), IsOk()); + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + EXPECT_THAT(mgr->DeleteFile(dv0), IsOk()); std::vector manifests{&manifest}; ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata.Schema()); ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.FilterManifests(schema, SpecsById(metadata), manifests, factory)); + auto result, mgr->FilterManifests(schema, SpecsById(metadata), manifests, factory)); ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(result, metadata)); ASSERT_EQ(entries.size(), 2U); @@ -611,6 +1077,45 @@ TEST_F(ManifestFilterManagerTest, DeleteFileObjectMatchesDeletionVectorByContent } } +TEST_F(ManifestFilterManagerTest, FailMissingDeleteFileObjectUsesDeleteFileIdentity) { + auto metadata = *table_->metadata(); + metadata.format_version = 3; + auto factory = MakeWriterFactory(metadata); + + auto make_dv = [&](int64_t offset) { + auto f = std::make_shared(); + f->content = DataFile::Content::kPositionDeletes; + f->file_path = table_location_ + "/delete/dv.puffin"; + f->file_format = FileFormatType::kPuffin; + f->referenced_data_file = + std::format("{}/data/referenced-{}.parquet", table_location_, offset); + f->content_offset = offset; + f->content_size_in_bytes = 10; + f->partition = PartitionValues(std::vector{Literal::Long(1L)}); + f->file_size_in_bytes = 256; + f->record_count = 5; + f->partition_spec_id = spec_->spec_id(); + return f; + }; + auto present_dv = make_dv(0); + auto missing_dv = make_dv(10); + + auto manifest_path = std::format("{}/metadata/dv-manifest-{}.avro", table_location_, + manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteDeleteManifest(present_dv, 3L, file_io_, + metadata, manifest_path)); + + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + EXPECT_THAT(mgr->DeleteFile(missing_dv), IsOk()); + mgr->FailMissingDeletePaths(); + + std::vector manifests{&manifest}; + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata.Schema()); + EXPECT_THAT(mgr->FilterManifests(schema, SpecsById(metadata), manifests, factory), + IsError(ErrorKind::kValidationFailed)); +} + TEST_F(ManifestFilterManagerTest, DuplicateDeletesCountRepeatedDeletedFiles) { auto* metadata = table_->metadata().get(); auto factory = MakeWriterFactory(*metadata); @@ -630,16 +1135,59 @@ TEST_F(ManifestFilterManagerTest, DuplicateDeletesCountRepeatedDeletedFiles) { WriteDeleteManifest({{del_file, 3L}, {del_file, 3L}}, file_io_, *metadata, manifest_path)); - ManifestFilterManager mgr(ManifestContent::kDeletes, file_io_); - EXPECT_THAT(mgr.DeleteFile(del_file), IsOk()); + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + EXPECT_THAT(mgr->DeleteFile(del_file), IsOk()); std::vector manifests{&manifest}; ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->FilterManifests(schema, SpecsById(*metadata), + manifests, factory)); + + EXPECT_EQ(mgr->DeletedFiles().size(), 1U); + EXPECT_EQ(mgr->DuplicateDeletesCount(), 1); +} + +TEST_F(ManifestFilterManagerTest, DuplicateDeletesCountDoesNotCrossManifestBoundary) { + auto* metadata = table_->metadata().get(); + auto factory = MakeWriterFactory(*metadata); + + auto del_file = std::make_shared(); + del_file->content = DataFile::Content::kPositionDeletes; + del_file->file_path = table_location_ + "/delete/reused.parquet"; + del_file->file_format = FileFormatType::kParquet; + del_file->partition = PartitionValues(std::vector{Literal::Long(1L)}); + del_file->file_size_in_bytes = 512; + del_file->record_count = 10; + del_file->partition_spec_id = spec_->spec_id(); + + auto first_manifest_path = + std::format("{}/metadata/reused-a-{}.avro", table_location_, manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto first_manifest, + WriteDeleteManifest(del_file, 3L, file_io_, *metadata, first_manifest_path)); + auto second_manifest_path = + std::format("{}/metadata/reused-b-{}.avro", table_location_, manifest_counter_++); + ICEBERG_UNWRAP_OR_FAIL( + auto second_manifest, + WriteDeleteManifest(del_file, 3L, file_io_, *metadata, second_manifest_path)); + ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.FilterManifests(schema, SpecsById(*metadata), manifests, factory)); + auto mgr, ManifestFilterManager::Make(ManifestContent::kDeletes, file_io_)); + EXPECT_THAT(mgr->DeleteFile(del_file), IsOk()); - EXPECT_EQ(mgr.DeletedFiles().size(), 1U); - EXPECT_EQ(mgr.DuplicateDeletesCount(), 1); + std::vector manifests{&first_manifest, &second_manifest}; + ICEBERG_UNWRAP_OR_FAIL(auto schema, metadata->Schema()); + auto specs = SpecsById(*metadata); + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr->FilterManifests(schema, specs, manifests, factory)); + ICEBERG_UNWRAP_OR_FAIL(auto summary, mgr->BuildSummary(result, specs)); + + EXPECT_EQ(mgr->DeletedFiles().size(), 1U); + EXPECT_EQ(mgr->DuplicateDeletesCount(), 0); + auto summary_map = summary.Build(); + EXPECT_EQ(summary_map.at(SnapshotSummaryFields::kRemovedDeleteFiles), "2"); + EXPECT_EQ(summary_map.count(SnapshotSummaryFields::kDeletedDuplicatedFiles), 0U); } } // namespace iceberg diff --git a/src/iceberg/test/manifest_merge_manager_test.cc b/src/iceberg/test/manifest_merge_manager_test.cc index d06ff0842..18d24db29 100644 --- a/src/iceberg/test/manifest_merge_manager_test.cc +++ b/src/iceberg/test/manifest_merge_manager_test.cc @@ -20,6 +20,7 @@ #include "iceberg/manifest/manifest_merge_manager.h" #include +#include #include #include #include @@ -51,6 +52,7 @@ namespace iceberg { namespace { constexpr int8_t kFormatVersion = 2; +constexpr int8_t kRowIdFormatVersion = 3; constexpr int64_t kSnapshotId = 12345L; constexpr int32_t kSpecId0 = 0; constexpr int32_t kSpecId1 = 1; @@ -85,6 +87,17 @@ class ManifestMergeManagerTest : public ::testing::Test { metadata_ = std::shared_ptr(std::move(metadata)); } + Result> BuildV3Metadata() { + auto builder = TableMetadataBuilder::BuildFromEmpty(kRowIdFormatVersion); + builder->SetCurrentSchema(schema_, schema_->HighestFieldId().value_or(0)); + builder->SetDefaultPartitionSpec(spec0_); + builder->AddPartitionSpec(spec1_); + builder->SetDefaultSortOrder(SortOrder::Unsorted()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, builder->Build()); + metadata->next_row_id = 1000; + return std::shared_ptr(std::move(metadata)); + } + // Write a small manifest with N data files and return the ManifestFile descriptor. Result WriteManifest(int32_t spec_id, int num_files, int64_t file_size_override = 512, @@ -116,14 +129,61 @@ class ManifestMergeManagerTest : public ::testing::Test { return manifest_file; } - ManifestWriterFactory MakeWriterFactory() { - return [this](int32_t spec_id, - ManifestContent content) -> Result> { + Result WriteDataManifestWithFileRowIds( + std::optional manifest_first_row_id, + std::optional entry_first_row_id, int64_t snapshot_id, + int64_t file_size_override = 512) { + auto path = std::format("manifest-{}.avro", manifest_counter_++); + ICEBERG_ASSIGN_OR_RAISE(auto writer, + ManifestWriter::MakeWriter( + kRowIdFormatVersion, snapshot_id, path, file_io_, spec0_, + schema_, ManifestContent::kData, entry_first_row_id)); + auto f = std::make_shared(); + f->content = DataFile::Content::kData; + f->file_path = std::format("data/row-id-file-{}.parquet", manifest_counter_); + f->file_format = FileFormatType::kParquet; + f->partition = PartitionValues(std::vector{Literal::Long(0)}); + f->file_size_in_bytes = 1024; + f->record_count = 10; + f->partition_spec_id = kSpecId0; + f->first_row_id = entry_first_row_id; + ICEBERG_RETURN_UNEXPECTED( + writer->WriteExistingEntry(ManifestEntry{.status = ManifestStatus::kExisting, + .snapshot_id = snapshot_id, + .sequence_number = kSnapshotId, + .file_sequence_number = kSnapshotId, + .data_file = std::move(f)})); + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto manifest_file, writer->ToManifestFile()); + manifest_file.manifest_length = file_size_override; + manifest_file.first_row_id = manifest_first_row_id; + manifest_file.added_snapshot_id = snapshot_id; + return manifest_file; + } + + ManifestWriterFactory MakeWriterFactory() { return MakeWriterFactory(kFormatVersion); } + + Result> MakeMergeManager( + int64_t target_size_bytes, int32_t min_count_to_merge, bool merge_enabled, + ManifestContent content = ManifestContent::kData) { + return ManifestMergeManager::Make( + content, target_size_bytes, min_count_to_merge, merge_enabled, file_io_, + [] { return kSnapshotId; }, + [this](const std::string& location) { return file_io_->DeleteFile(location); }); + } + + ManifestWriterFactory MakeWriterFactory(int8_t format_version) { + return [this, format_version]( + int32_t spec_id, + ManifestContent content) -> Result> { ++factory_call_count_; auto spec = spec_id == kSpecId0 ? spec0_ : spec1_; auto path = std::format("merged-{}.avro", manifest_counter_++); - return ManifestWriter::MakeWriter(kFormatVersion, kSnapshotId, path, file_io_, spec, - schema_, content); + return ManifestWriter::MakeWriter( + format_version, kSnapshotId, path, file_io_, spec, schema_, content, + content == ManifestContent::kData && format_version >= 3 + ? std::make_optional(1000) + : std::nullopt); }; } @@ -140,6 +200,12 @@ class ManifestMergeManagerTest : public ::testing::Test { return total; } + Result> ReadEntries(const ManifestFile& manifest) { + ICEBERG_ASSIGN_OR_RAISE(auto reader, + ManifestReader::Make(manifest, file_io_, schema_, spec0_)); + return reader->Entries(); + } + std::shared_ptr file_io_; std::shared_ptr schema_; std::shared_ptr spec0_; @@ -154,39 +220,46 @@ TEST_F(ManifestMergeManagerTest, MergeDisabled) { ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1)); ICEBERG_UNWRAP_OR_FAIL(auto m2, WriteManifest(kSpecId0, 1)); - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/false); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/false)); ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.MergeManifests({m0, m1}, {m2}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); - // merge disabled → all 3 manifests returned, factory never called + auto result, mgr->MergeManifests({m0, m1}, {m2}, *metadata_, MakeWriterFactory())); EXPECT_EQ(result.size(), 3U); EXPECT_EQ(factory_call_count_, 0); } +TEST_F(ManifestMergeManagerTest, MakeRejectsNullParameters) { + EXPECT_THAT( + ManifestMergeManager::Make(ManifestContent::kData, /*target=*/1024, /*min_count=*/2, + /*enabled=*/true, nullptr, [] { return kSnapshotId; }), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT( + ManifestMergeManager::Make(ManifestContent::kData, /*target=*/1024, /*min_count=*/2, + /*enabled=*/true, file_io_, {}), + IsError(ErrorKind::kInvalidArgument)); +} + TEST_F(ManifestMergeManagerTest, BelowMinCountThreshold) { ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1)); ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1)); - // min_count=3, only 2 manifests total → no merge - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/3, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.MergeManifests({m0}, {m1}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/3, + /*enabled=*/true)); + ICEBERG_UNWRAP_OR_FAIL( + auto result, mgr->MergeManifests({m0}, {m1}, *metadata_, MakeWriterFactory())); EXPECT_EQ(result.size(), 2U); EXPECT_EQ(factory_call_count_, 0); } TEST_F(ManifestMergeManagerTest, MergeOccursAtThreshold) { - // 3 small manifests (each 100 bytes), target=1024 → all fit in one bin ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m2, WriteManifest(kSpecId0, 1, /*size=*/100)); - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/3, /*enabled=*/true); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/3, + /*enabled=*/true)); ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.MergeManifests({m0, m1}, {m2}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); - // All 3 merged into 1 manifest (total 3 entries) + auto result, mgr->MergeManifests({m0, m1}, {m2}, *metadata_, MakeWriterFactory())); EXPECT_EQ(result.size(), 1U); ICEBERG_UNWRAP_OR_FAIL(auto count1, CountEntries(result)); EXPECT_EQ(count1, 3); @@ -198,144 +271,225 @@ TEST_F(ManifestMergeManagerTest, ReplacedManifestCountTracksPreviousSnapshotInpu m0.added_snapshot_id = kSnapshotId - 1; m1.added_snapshot_id = kSnapshotId - 2; - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.MergeManifests({m0, m1}, {}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); + auto result, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); EXPECT_EQ(result.size(), 1U); - EXPECT_EQ(mgr.ReplacedManifestsCount(), 2); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 2); +} + +TEST_F(ManifestMergeManagerTest, MergeManifestsCachesMergedBinAcrossRetries) { + ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); + m0.added_snapshot_id = kSnapshotId - 1; + m1.added_snapshot_id = kSnapshotId - 2; + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); + ICEBERG_UNWRAP_OR_FAIL( + auto first, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(first.size(), 1U); + EXPECT_EQ(factory_call_count_, 1); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 2); + + ICEBERG_UNWRAP_OR_FAIL( + auto second, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(second.size(), 1U); + EXPECT_EQ(second[0].manifest_path, first[0].manifest_path); + EXPECT_EQ(factory_call_count_, 1); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 2); +} + +TEST_F(ManifestMergeManagerTest, CleanUncommittedDropsMergeCacheAndRollsBackCount) { + ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); + m0.added_snapshot_id = kSnapshotId - 1; + m1.added_snapshot_id = kSnapshotId - 2; + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); + ICEBERG_UNWRAP_OR_FAIL( + auto first, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(first.size(), 1U); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 2); + + EXPECT_THAT(mgr->CleanUncommitted({}), IsOk()); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 0); + + ICEBERG_UNWRAP_OR_FAIL( + auto second, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(second.size(), 1U); + EXPECT_NE(second[0].manifest_path, first[0].manifest_path); + EXPECT_EQ(factory_call_count_, 2); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 2); +} + +TEST_F(ManifestMergeManagerTest, CleanUncommittedDeletesMergedManifestWithCallback) { + ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); + m0.added_snapshot_id = kSnapshotId - 1; + m1.added_snapshot_id = kSnapshotId - 2; + + std::vector deleted_paths; + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + ManifestMergeManager::Make( + ManifestContent::kData, /*target=*/1024, /*min_count=*/2, + /*enabled=*/true, file_io_, [] { return kSnapshotId; }, + [&deleted_paths](const std::string& path) { + deleted_paths.push_back(path); + return Status{}; + })); + ICEBERG_UNWRAP_OR_FAIL( + auto result, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(result.size(), 1U); + + EXPECT_THAT(mgr->CleanUncommitted({}), IsOk()); + + EXPECT_THAT(deleted_paths, ::testing::ElementsAre(result[0].manifest_path)); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 0); +} + +TEST_F(ManifestMergeManagerTest, CleanUncommittedIgnoresDeleteCallbackError) { + ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); + m0.added_snapshot_id = kSnapshotId - 1; + m1.added_snapshot_id = kSnapshotId - 2; + + ICEBERG_UNWRAP_OR_FAIL( + auto mgr, ManifestMergeManager::Make( + ManifestContent::kData, /*target=*/1024, /*min_count=*/2, + /*enabled=*/true, file_io_, [] { return kSnapshotId; }, + [](const std::string&) { return IOError("delete failed"); })); + ICEBERG_UNWRAP_OR_FAIL( + auto result, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(result.size(), 1U); + + EXPECT_THAT(mgr->CleanUncommitted({}), IsOk()); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 0); +} + +TEST_F(ManifestMergeManagerTest, CleanUncommittedKeepsCommittedMergeCache) { + ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); + ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); + m0.added_snapshot_id = kSnapshotId - 1; + m1.added_snapshot_id = kSnapshotId - 2; + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); + ICEBERG_UNWRAP_OR_FAIL( + auto first, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(first.size(), 1U); + + EXPECT_THAT(mgr->CleanUncommitted({first[0].manifest_path}), IsOk()); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 2); + + ICEBERG_UNWRAP_OR_FAIL( + auto second, mgr->MergeManifests({m0, m1}, {}, *metadata_, MakeWriterFactory())); + ASSERT_EQ(second.size(), 1U); + EXPECT_EQ(second[0].manifest_path, first[0].manifest_path); + EXPECT_EQ(factory_call_count_, 1); } TEST_F(ManifestMergeManagerTest, ReplacedManifestCountIgnoresCurrentSnapshotInputs) { ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.MergeManifests({}, {m0, m1}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); + auto result, mgr->MergeManifests({}, {m0, m1}, *metadata_, MakeWriterFactory())); EXPECT_EQ(result.size(), 1U); - EXPECT_EQ(mgr.ReplacedManifestsCount(), 0); + EXPECT_EQ(mgr->ReplacedManifestsCount(), 0); +} + +TEST_F(ManifestMergeManagerTest, + MergePreservesCurrentSnapshotFileFirstRowIdsWhenManifestFirstRowIdIsNull) { + constexpr int64_t kEntryFirstRowId = 1234; + ICEBERG_UNWRAP_OR_FAIL(auto current, WriteDataManifestWithFileRowIds( + /*manifest_first_row_id=*/std::nullopt, + kEntryFirstRowId, kSnapshotId)); + ICEBERG_UNWRAP_OR_FAIL(auto previous, WriteManifest(kSpecId0, 1, /*size=*/512)); + previous.added_snapshot_id = 7; + + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); + ICEBERG_UNWRAP_OR_FAIL(auto v3_metadata, BuildV3Metadata()); + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr->MergeManifests({previous}, {current}, *v3_metadata, + MakeWriterFactory(kRowIdFormatVersion))); + ASSERT_EQ(result.size(), 1U); + + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadEntries(result[0])); + auto current_entry = std::ranges::find_if(entries, [](const ManifestEntry& entry) { + return entry.data_file != nullptr && + entry.data_file->file_path.find("row-id-file") != std::string::npos; + }); + ASSERT_NE(current_entry, entries.end()); + ASSERT_TRUE(current_entry->data_file->first_row_id.has_value()); + EXPECT_EQ(*current_entry->data_file->first_row_id, kEntryFirstRowId); } TEST_F(ManifestMergeManagerTest, OversizedManifestPassedThrough) { - // m_large exceeds target → must not be merged; m_small fits ICEBERG_UNWRAP_OR_FAIL(auto m_large, WriteManifest(kSpecId0, 2, /*size=*/2000)); ICEBERG_UNWRAP_OR_FAIL(auto m_small, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m_small2, WriteManifest(kSpecId0, 1, /*size=*/100)); - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.MergeManifests({m_large, m_small}, {m_small2}, kSnapshotId, - *metadata_, file_io_, MakeWriterFactory())); - // m_large is oversized and acts as a bin boundary — the two small manifests on either - // side of it are never merged together. m_small2 (the newest) is also protected by - // minCountToMerge (size 1 < 2). All three remain separate. + mgr->MergeManifests({m_large, m_small}, {m_small2}, *metadata_, + MakeWriterFactory())); EXPECT_EQ(result.size(), 3U); ICEBERG_UNWRAP_OR_FAIL(auto count2, CountEntries(result)); - EXPECT_EQ(count2, 4); // 2 + 1 + 1 + EXPECT_EQ(count2, 4); } TEST_F(ManifestMergeManagerTest, CrossSpecManifestsNotMerged) { - // Manifests with different spec IDs must never be merged together ICEBERG_UNWRAP_OR_FAIL(auto m_spec0a, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m_spec0b, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m_spec1a, WriteManifest(kSpecId1, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m_spec1b, WriteManifest(kSpecId1, 1, /*size=*/100)); - // With 4 manifests (target large enough for each pair), we get 2 merged outputs - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL( - auto result, - mgr.MergeManifests({m_spec0a, m_spec1a}, {m_spec0b, m_spec1b}, kSnapshotId, - *metadata_, file_io_, MakeWriterFactory())); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); + ICEBERG_UNWRAP_OR_FAIL(auto result, + mgr->MergeManifests({m_spec0a, m_spec1a}, {m_spec0b, m_spec1b}, + *metadata_, MakeWriterFactory())); EXPECT_EQ(result.size(), 2U); - // Verify spec IDs are preserved per output manifest for (const auto& m : result) { EXPECT_THAT(m.partition_spec_id, ::testing::AnyOf(kSpecId0, kSpecId1)); } } TEST_F(ManifestMergeManagerTest, WriterFactoryCalledOncePerMergedManifest) { - // 4 small manifests in two groups → 2 merged outputs → factory called twice ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m2, WriteManifest(kSpecId1, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m3, WriteManifest(kSpecId1, 1, /*size=*/100)); - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.MergeManifests({m0, m2}, {m1, m3}, kSnapshotId, *metadata_, - file_io_, MakeWriterFactory())); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true)); + ICEBERG_UNWRAP_OR_FAIL(auto result, mgr->MergeManifests({m0, m2}, {m1, m3}, *metadata_, + MakeWriterFactory())); EXPECT_EQ(result.size(), 2U); EXPECT_EQ(factory_call_count_, 2); } -TEST_F(ManifestMergeManagerTest, MixedContentManifestsNotMerged) { - // Data and delete manifests sharing the same spec_id must never be merged together. - // The grouping key is (spec_id, content), so they land in separate bins. +TEST_F(ManifestMergeManagerTest, UnexpectedContentRejected) { ICEBERG_UNWRAP_OR_FAIL( auto d0, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kData)); - ICEBERG_UNWRAP_OR_FAIL( - auto d1, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kData)); - ICEBERG_UNWRAP_OR_FAIL( - auto del0, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kDeletes)); - ICEBERG_UNWRAP_OR_FAIL( - auto del1, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kDeletes)); - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/2, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.MergeManifests({d0, del0}, {d1, del1}, kSnapshotId, *metadata_, - file_io_, MakeWriterFactory())); - // 2 data → 1 merged data manifest; 2 delete → 1 merged delete manifest - EXPECT_EQ(result.size(), 2U); - int data_count = 0; - int delete_count = 0; - for (const auto& m : result) { - if (m.content == ManifestContent::kData) ++data_count; - if (m.content == ManifestContent::kDeletes) ++delete_count; - } - EXPECT_EQ(data_count, 1); - EXPECT_EQ(delete_count, 1); -} - -TEST_F(ManifestMergeManagerTest, MixedContentUsesFirstManifestPerContent) { - ICEBERG_UNWRAP_OR_FAIL( - auto d0, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kData)); - ICEBERG_UNWRAP_OR_FAIL( - auto d1, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kData)); - ICEBERG_UNWRAP_OR_FAIL( - auto del0, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kDeletes)); - ICEBERG_UNWRAP_OR_FAIL( - auto del1, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kDeletes)); - - // Each content type's newest manifest must be protected by the threshold - // independently. - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/3, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.MergeManifests({d0, del0}, {d1, del1}, kSnapshotId, *metadata_, - file_io_, MakeWriterFactory())); - - // Each content type has exactly two manifests, below min_count=3, so neither pair - // should be merged. - ASSERT_EQ(result.size(), 4U); - int data_count = 0; - int delete_count = 0; - for (const auto& manifest : result) { - if (manifest.content == ManifestContent::kData) { - ++data_count; - } else if (manifest.content == ManifestContent::kDeletes) { - ++delete_count; - } - } - EXPECT_EQ(data_count, 2); - EXPECT_EQ(delete_count, 2); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + MakeMergeManager(/*target=*/1024, /*min_count=*/2, + /*enabled=*/true, ManifestContent::kDeletes)); + EXPECT_THAT(mgr->MergeManifests({}, {d0}, *metadata_, MakeWriterFactory()), + IsError(ErrorKind::kInvalidArgument)); } TEST_F(ManifestMergeManagerTest, DeleteManifestsMerged) { - // Delete manifests are bin-packed and merged just like data manifests. ICEBERG_UNWRAP_OR_FAIL( auto del0, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kDeletes)); ICEBERG_UNWRAP_OR_FAIL( @@ -343,10 +497,12 @@ TEST_F(ManifestMergeManagerTest, DeleteManifestsMerged) { ICEBERG_UNWRAP_OR_FAIL( auto del2, WriteManifest(kSpecId0, 1, /*size=*/100, ManifestContent::kDeletes)); - ManifestMergeManager mgr(/*target=*/1024, /*min_count=*/3, /*enabled=*/true); - ICEBERG_UNWRAP_OR_FAIL(auto result, - mgr.MergeManifests({del0, del1}, {del2}, kSnapshotId, *metadata_, - file_io_, MakeWriterFactory())); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, + MakeMergeManager(/*target=*/1024, /*min_count=*/3, + /*enabled=*/true, ManifestContent::kDeletes)); + ICEBERG_UNWRAP_OR_FAIL( + auto result, + mgr->MergeManifests({del0, del1}, {del2}, *metadata_, MakeWriterFactory())); EXPECT_EQ(result.size(), 1U); EXPECT_EQ(result[0].content, ManifestContent::kDeletes); ICEBERG_UNWRAP_OR_FAIL(auto count, CountEntries(result)); @@ -354,26 +510,16 @@ TEST_F(ManifestMergeManagerTest, DeleteManifestsMerged) { } TEST_F(ManifestMergeManagerTest, PackEndOlderManifestsMergedNotNewest) { - // packEnd semantics: for [m0_new, m1_old, m2_old] with target=250 (pairs fit but - // triples don't), packing from the end merges m1+m2 (the older pair) and leaves - // m0 (the newest) in its own under-filled bin at the front of the output. - // This is the opposite of naive forward packing, which would merge m0+m1. ICEBERG_UNWRAP_OR_FAIL(auto m1, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m2, WriteManifest(kSpecId0, 1, /*size=*/100)); ICEBERG_UNWRAP_OR_FAIL(auto m0, WriteManifest(kSpecId0, 1, /*size=*/100)); - // target=250 fits two 100-byte manifests but not three. - // min_count=3 so m0's single-element bin is kept as-is (below threshold). - ManifestMergeManager mgr(/*target=*/250, /*min_count=*/3, /*enabled=*/true); + ICEBERG_UNWRAP_OR_FAIL(auto mgr, MakeMergeManager(/*target=*/250, /*min_count=*/3, + /*enabled=*/true)); ICEBERG_UNWRAP_OR_FAIL( - auto result, mgr.MergeManifests({m1, m2}, {m0}, kSnapshotId, *metadata_, file_io_, - MakeWriterFactory())); - // Expected: [m0 (pass-through), merged(m1+m2)] + auto result, mgr->MergeManifests({m1, m2}, {m0}, *metadata_, MakeWriterFactory())); ASSERT_EQ(result.size(), 2U); - // First output is the newest manifest m0, passed through unchanged (under-filled bin). EXPECT_EQ(result[0].manifest_length, m0.manifest_length); - // Second output is the merged older pair — it must be a newly written manifest - // (different path than either original). EXPECT_NE(result[1].manifest_path, m1.manifest_path); EXPECT_NE(result[1].manifest_path, m2.manifest_path); ICEBERG_UNWRAP_OR_FAIL(auto count, CountEntries(result)); diff --git a/src/iceberg/test/manifest_reader_test.cc b/src/iceberg/test/manifest_reader_test.cc index 3f85729a7..f2c4ef406 100644 --- a/src/iceberg/test/manifest_reader_test.cc +++ b/src/iceberg/test/manifest_reader_test.cc @@ -190,6 +190,41 @@ TEST_P(TestManifestReader, TestManifestReaderWithEmptyInheritableMetadata) { EXPECT_EQ(read_entry.snapshot_id, 1000L); } +TEST_P(TestManifestReader, DeletedEntriesDoNotInheritFirstRowId) { + auto version = GetParam(); + if (version < 3) { + GTEST_SKIP() << "first_row_id is only assigned in V3 manifests"; + } + + auto deleted_file = + MakeDataFile("/path/to/deleted.parquet", PartitionValues({Literal::Int(0)}), + /*record_count=*/10); + auto added_file = + MakeDataFile("/path/to/added.parquet", PartitionValues({Literal::Int(1)}), + /*record_count=*/5); + + auto deleted_entry = + MakeEntry(ManifestStatus::kDeleted, /*snapshot_id=*/1000L, std::move(deleted_file)); + deleted_entry.sequence_number = 0; + deleted_entry.file_sequence_number = 0; + + std::vector entries; + entries.push_back(std::move(deleted_entry)); + entries.push_back( + MakeEntry(ManifestStatus::kAdded, /*snapshot_id=*/1000L, std::move(added_file))); + auto manifest = WriteManifest(version, /*snapshot_id=*/1000L, std::move(entries)); + + ICEBERG_UNWRAP_OR_FAIL(auto reader, + ManifestReader::Make(manifest, file_io_, schema_, spec_)); + ICEBERG_UNWRAP_OR_FAIL(auto read_entries, reader->Entries()); + + ASSERT_EQ(read_entries.size(), 2U); + EXPECT_EQ(read_entries[0].status, ManifestStatus::kDeleted); + EXPECT_EQ(read_entries[0].data_file->first_row_id, std::nullopt); + EXPECT_EQ(read_entries[1].status, ManifestStatus::kAdded); + EXPECT_EQ(read_entries[1].data_file->first_row_id, 0); +} + TEST_P(TestManifestReader, TestReaderWithFilterWithoutSelect) { auto version = GetParam(); auto file_a = diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 76b9ac32c..69ee10b91 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -70,6 +70,16 @@ class TestMergeAppend : public MergingSnapshotUpdate { Status AddDelete(std::shared_ptr file, int64_t data_sequence_number) { return AddDeleteFile(std::move(file), data_sequence_number); } + Status ValidateNoNewDeletesForDataFiles(const TableMetadata& metadata, + int64_t starting_snapshot_id, + std::shared_ptr data_filter, + const DataFileSet& replaced_files, + const std::shared_ptr& parent, + std::shared_ptr io) const { + return MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( + metadata, starting_snapshot_id, std::move(data_filter), replaced_files, parent, + std::move(io), IsCaseSensitive()); + } Status RemoveDataFile(std::shared_ptr file) { return DeleteDataFile(std::move(file)); } @@ -84,6 +94,7 @@ class TestMergeAppend : public MergingSnapshotUpdate { } int64_t GeneratedSnapshotId() { return SnapshotId(); } void SetDataSeqNumber(int64_t seq) { SetNewDataFilesDataSequenceNumber(seq); } + void SetCaseSensitive(bool case_sensitive) { CaseSensitive(case_sensitive); } static Status ValidateAddedDataFilesForTest(const TableMetadata& metadata, std::optional starting_snapshot_id, const std::shared_ptr& parent, @@ -99,6 +110,23 @@ class TestMergeAppend : public MergingSnapshotUpdate { return MergingSnapshotUpdate::ValidateAddedDataFiles( metadata, starting_snapshot_id, partition_set, parent, std::move(io)); } + static Status ValidateDataFilesExistForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + const std::unordered_set& file_paths, bool skip_deletes, + std::shared_ptr filter, const std::shared_ptr& parent, + std::shared_ptr io, bool case_sensitive = true) { + return MergingSnapshotUpdate::ValidateDataFilesExist( + metadata, starting_snapshot_id, file_paths, skip_deletes, std::move(filter), + parent, std::move(io), case_sensitive); + } + static Status ValidateNoNewDeletesForDataFilesForTest( + const TableMetadata& metadata, int64_t starting_snapshot_id, + const DataFileSet& replaced_files, const std::shared_ptr& parent, + std::shared_ptr io, bool ignore_equality_deletes = false) { + return MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( + metadata, starting_snapshot_id, replaced_files, parent, std::move(io), + ignore_equality_deletes); + } static Status ValidateNoNewDeletesForDataFilesForTest( const TableMetadata& metadata, int64_t starting_snapshot_id, std::shared_ptr data_filter, const DataFileSet& replaced_files, @@ -397,6 +425,25 @@ TEST_F(MergingSnapshotUpdateTest, CommitNewDataFile) { EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedRecords), "100"); } +TEST_F(MergingSnapshotUpdateTest, CommitV3NewDataFileAssignsRowLineage) { + ICEBERG_UNWRAP_OR_FAIL(auto props, table_->NewUpdateProperties()); + props->Set(TableProperties::kFormatVersion.key(), "3"); + EXPECT_THAT(props->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + ICEBERG_UNWRAP_OR_FAIL(auto first_row_id, snapshot->FirstRowId()); + ICEBERG_UNWRAP_OR_FAIL(auto added_rows, snapshot->AddedRows()); + EXPECT_EQ(first_row_id, std::make_optional(0)); + EXPECT_EQ(added_rows, std::make_optional(100)); + EXPECT_EQ(table_->metadata()->next_row_id, 100); +} + TEST_F(MergingSnapshotUpdateTest, CommitMultipleDataFiles) { ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddFile(file_a_), IsOk()); @@ -434,7 +481,6 @@ TEST_F(MergingSnapshotUpdateTest, CommitPreservesExistingManifests) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); - // Both data files should be visible — 1 existing + 1 new EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "2"); } @@ -461,6 +507,23 @@ TEST_F(MergingSnapshotUpdateTest, SetNewDataFilesDataSequenceNumber) { EXPECT_THAT(table_->Refresh(), IsOk()); ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); + + auto snapshot_cache = SnapshotCache(snapshot.get()); + ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(table_->io())); + std::vector manifests(data_manifests.begin(), data_manifests.end()); + ICEBERG_UNWRAP_OR_FAIL(auto entries, ReadAllEntries(manifests, *table_->metadata())); + ASSERT_EQ(entries.size(), 1U); + EXPECT_EQ(entries[0].sequence_number, 42); +} + +TEST_F(MergingSnapshotUpdateTest, AddDataFileDoesNotMutateCallerFile) { + auto file = MakeDataFile("/data/with-row-id.parquet", 1L); + file->first_row_id = 42; + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file), IsOk()); + + EXPECT_EQ(file->first_row_id, 42); } TEST_F(MergingSnapshotUpdateTest, CustomSummaryPropertySurvivesApplyRebuild) { @@ -475,6 +538,19 @@ TEST_F(MergingSnapshotUpdateTest, CustomSummaryPropertySurvivesApplyRebuild) { EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); } +TEST_F(MergingSnapshotUpdateTest, BaseSetCustomSummaryPropertySurvivesApplyRebuild) { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + SnapshotUpdate& base_update = *op; + base_update.Set("custom-prop", "custom-value"); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.at("custom-prop"), "custom-value"); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "1"); +} + // ------------------------------------------------------------------------- // CleanUncommitted test // ------------------------------------------------------------------------- @@ -484,11 +560,32 @@ TEST_F(MergingSnapshotUpdateTest, CleanUncommittedAfterSuccessfulCommitDoesNotCr EXPECT_THAT(op->AddFile(file_a_), IsOk()); EXPECT_THAT(op->Commit(), IsOk()); - // Simulate a caller invoking CleanUncommitted after a commit (e.g. cleanup - // in an error handler that runs regardless of success). Passing an empty set - // means no manifests are considered committed, so CleanUncommitted attempts - // to delete all written manifests. This should not crash. - op->CleanUncommitted({}); + // Cleanup may run from an error handler even after commit success. + EXPECT_THAT(op->CleanUncommitted({}), IsOk()); +} + +TEST_F(MergingSnapshotUpdateTest, + CleanUncommittedDeletesManagerOutputsWithDeleteCallback) { + ICEBERG_UNWRAP_OR_FAIL(auto initial, NewMergeAppend()); + EXPECT_THAT(initial->AddFile(file_a_), IsOk()); + EXPECT_THAT(initial->AddFile(file_b_), IsOk()); + EXPECT_THAT(initial->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + + std::vector deleted_paths; + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + op->DeleteWith([&deleted_paths](const std::string& path) { + deleted_paths.push_back(path); + return Status{}; + }); + EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL( + auto manifests, op->Apply(*table_->metadata(), table_->current_snapshot().value())); + EXPECT_THAT(manifests, ::testing::SizeIs(1)); + + EXPECT_THAT(op->CleanUncommitted({}), IsOk()); + EXPECT_THAT(deleted_paths, ::testing::Contains(::testing::HasSubstr("/metadata/"))); } // ------------------------------------------------------------------------- @@ -542,6 +639,26 @@ TEST_F(MergingSnapshotUpdateTest, AddDeleteFileWithExplicitSequenceWritesSequenc EXPECT_EQ(entries[0].sequence_number.value(), 17); } +TEST_F(MergingSnapshotUpdateTest, ApplyRebuildsDeleteSummaryAfterPreparingDeletes) { + auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto first_manifests, op->Apply(*table_->metadata(), nullptr)); + EXPECT_THAT(first_manifests, ::testing::Contains(::testing::Field( + &ManifestFile::content, ManifestContent::kDeletes))); + + ICEBERG_UNWRAP_OR_FAIL(auto second_manifests, op->Apply(*table_->metadata(), nullptr)); + EXPECT_THAT(second_manifests, ::testing::Contains(::testing::Field( + &ManifestFile::content, ManifestContent::kDeletes))); + + auto summary = op->Summary(); + EXPECT_EQ(summary.at(SnapshotSummaryFields::kAddedDeleteFiles), "1"); + EXPECT_EQ(summary.at(SnapshotSummaryFields::kAddedPosDeleteFiles), "1"); +} + // Covers the bug where deleted delete files were not tracked in the snapshot summary. TEST_F(MergingSnapshotUpdateTest, CommitDeletesDeleteFileSummaryHasRemovedDeleteFiles) { // Step 1: commit a delete file. @@ -572,7 +689,7 @@ TEST_F(MergingSnapshotUpdateTest, CommitDeletesDeleteFileSummaryHasRemovedDelete TEST_F(MergingSnapshotUpdateTest, DuplicateDataFileOnlyCountedOnce) { ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddFile(file_a_), IsOk()); - EXPECT_THAT(op->AddFile(file_a_), IsOk()); // duplicate — should be ignored + EXPECT_THAT(op->AddFile(file_a_), IsOk()); EXPECT_THAT(op->Commit(), IsOk()); EXPECT_THAT(table_->Refresh(), IsOk()); @@ -581,11 +698,32 @@ TEST_F(MergingSnapshotUpdateTest, DuplicateDataFileOnlyCountedOnce) { EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kTotalDataFiles), "1"); } +TEST_F(MergingSnapshotUpdateTest, CommitSkipsMalformedPreviousSummaryTotal) { + { + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_a_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + EXPECT_THAT(table_->Refresh(), IsOk()); + } + + ICEBERG_UNWRAP_OR_FAIL(auto previous_snapshot, table_->current_snapshot()); + previous_snapshot->summary[SnapshotSummaryFields::kTotalRecords] = "not-a-number"; + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddFile(file_b_), IsOk()); + EXPECT_THAT(op->Commit(), IsOk()); + + EXPECT_THAT(table_->Refresh(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); + EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kTotalRecords), 0U); + EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedRecords), "100"); +} + // ------------------------------------------------------------------------- // ValidateNewDeleteFile format version tests // ------------------------------------------------------------------------- -/// \brief V1-table test fixture — deletes are not supported in format v1. +/// \brief V1-table test fixture. class MergingSnapshotUpdateV1Test : public UpdateTestBase { protected: std::string MetadataResource() const override { return "TableMetadataV1Valid.json"; } @@ -667,6 +805,19 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV3AllowsDeletionVector) { EXPECT_THAT(op->AddDelete(del_file), IsOk()); } +TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileRejectsUnsupportedVersion) { + SetTableFormatVersion(TableMetadata::kSupportedTableFormatVersion + 1); + + auto del_file = MakeDeleteFile("/delete/dv_a.puffin", 1L); + del_file->file_format = FileFormatType::kPuffin; + del_file->referenced_data_file = file_a_->file_path; + del_file->content_offset = 0; + del_file->content_size_in_bytes = 10; + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + EXPECT_THAT(op->AddDelete(del_file), IsError(ErrorKind::kInvalidArgument)); +} + TEST_F(MergingSnapshotUpdateTest, ApplyRejectsV2StagedPositionDeleteAfterV3Upgrade) { auto del_file = MakeDeleteFile("/delete/del_a.parquet", 1L); @@ -679,11 +830,10 @@ TEST_F(MergingSnapshotUpdateTest, ApplyRejectsV2StagedPositionDeleteAfterV3Upgra } // ------------------------------------------------------------------------- -// AddManifest — invalid manifest rejection +// AddManifest protected primitive behavior // ------------------------------------------------------------------------- TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsDeleteManifest) { - // Build a ManifestFile with content = kDeletes ManifestFile del_manifest; del_manifest.manifest_path = table_location_ + "/metadata/del.avro"; del_manifest.content = ManifestContent::kDeletes; @@ -693,7 +843,7 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsDeleteManifest) { EXPECT_THAT(op->AppendManifest(del_manifest), IsError(ErrorKind::kInvalidArgument)); } -TEST_F(MergingSnapshotUpdateTest, AddManifestAllowsManifestWithExistingFilesCount) { +TEST_F(MergingSnapshotUpdateTest, AddManifestPrimitiveAllowsExistingFilesCount) { ManifestFile manifest; manifest.manifest_path = table_location_ + "/metadata/existing.avro"; manifest.content = ManifestContent::kData; @@ -704,7 +854,7 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestAllowsManifestWithExistingFilesCoun EXPECT_THAT(op->AppendManifest(manifest), IsOk()); } -TEST_F(MergingSnapshotUpdateTest, AddManifestAllowsManifestWithDeletedFilesCount) { +TEST_F(MergingSnapshotUpdateTest, AddManifestPrimitiveAllowsDeletedFilesCount) { ManifestFile manifest; manifest.manifest_path = table_location_ + "/metadata/deleted.avro"; manifest.content = ManifestContent::kData; @@ -742,7 +892,7 @@ TEST_F(MergingSnapshotUpdateTest, AddManifestRejectsManifestWithFirstRowId) { } // ------------------------------------------------------------------------- -// AddManifest — basic commit (inherit path: v2 with can_inherit_snapshot_id) +// AddManifest basic commit behavior // ------------------------------------------------------------------------- TEST_F(MergingSnapshotUpdateTest, AppendManifestEmptyTable) { @@ -766,7 +916,6 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestEmptyTable) { } TEST_F(MergingSnapshotUpdateTest, AppendManifestWithDataFiles) { - // Mix AddDataFile + AddManifest — should produce 2 manifests. auto path = table_location_ + "/metadata/input.avro"; ICEBERG_UNWRAP_OR_FAIL(auto manifest, WriteManifest(path, {file_a_, file_b_})); @@ -785,7 +934,7 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestWithDataFiles) { } // ------------------------------------------------------------------------- -// AddManifest — merge behavior +// AddManifest merge behavior // ------------------------------------------------------------------------- TEST_F(MergingSnapshotUpdateTest, AppendManifestMergeWithMinCountOne) { @@ -837,13 +986,12 @@ TEST_F(MergingSnapshotUpdateTest, AppendManifestDoNotMergeMinCount) { ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot()); SnapshotCache snapshot_cache(snapshot.get()); ICEBERG_UNWRAP_OR_FAIL(auto data_manifests, snapshot_cache.DataManifests(file_io_)); - // Below min-count-to-merge threshold — all 3 pass through unchanged. EXPECT_EQ(data_manifests.size(), 3); EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kAddedDataFiles), "3"); } // ------------------------------------------------------------------------- -// Manifest merge — data files only +// Manifest merge data files only // ------------------------------------------------------------------------- TEST_F(MergingSnapshotUpdateTest, ManifestMergeMergesIntoOne) { @@ -856,7 +1004,6 @@ TEST_F(MergingSnapshotUpdateTest, ManifestMergeMergesIntoOne) { // Snapshot 1: file_a_ CommitFileA(); - // Snapshot 2: file_b_ — should merge with existing manifest. ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->AddFile(file_b_), IsOk()); EXPECT_THAT(op->Commit(), IsOk()); @@ -965,7 +1112,6 @@ TEST_F(MergingSnapshotUpdateTest, SummaryManifestCountsAfterDelete) { CommitFileA(); - // Delete file_a_ — filter manager rewrites the manifest. ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); EXPECT_THAT(op->Commit(), IsOk()); @@ -1022,7 +1168,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesFailsForTruncatedHistory EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest(*metadata, /*starting=*/2, branch_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, @@ -1046,7 +1192,7 @@ TEST_F(MergingSnapshotUpdateTest, EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest(*metadata, std::nullopt, snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithNoStartingSnapshotChecksAll) { @@ -1055,7 +1201,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithNoStartingSnapshotCh EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest( *table_->metadata(), std::nullopt, snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest( *table_->metadata(), snapshot->snapshot_id, snapshot, file_io_), IsOk()); @@ -1076,7 +1222,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesWithPartitionSetDetectsC EXPECT_THAT(TestMergeAppend::ValidateAddedDataFilesForTest( *table_->metadata(), first_snapshot->snapshot_id, partition_set, second_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesIgnoresOldEntrySnapshotId) { @@ -1109,6 +1255,38 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDataFilesIgnoresOldEntrySnapshotI IsOk()); } +TEST_F(MergingSnapshotUpdateTest, ValidateDataFilesExistUsesRowFilter) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->RemoveDataFile(file_a_), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + std::unordered_set required_files{file_a_->file_path}; + EXPECT_THAT(TestMergeAppend::ValidateDataFilesExistForTest( + *metadata, first_snapshot->snapshot_id, required_files, + /*skip_deletes=*/false, Expressions::Equal("x", Literal::Long(1L)), + second_snapshot, file_io_), + IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(TestMergeAppend::ValidateDataFilesExistForTest( + *metadata, first_snapshot->snapshot_id, required_files, + /*skip_deletes=*/false, Expressions::Equal("x", Literal::Long(2L)), + second_snapshot, file_io_), + IsOk()); +} + TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesWithFilterDetectsConflict) { CommitFileA(); @@ -1135,7 +1313,97 @@ TEST_F(MergingSnapshotUpdateTest, EXPECT_THAT(TestMergeAppend::ValidateNoNewDeletesForDataFilesForTest( *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), replaced_files, second_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); +} + +TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeletesForDataFilesDetectsConflict) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + DataFileSet replaced_files; + replaced_files.insert(file_a_); + EXPECT_THAT(TestMergeAppend::ValidateNoNewDeletesForDataFilesForTest( + *metadata, first_snapshot->snapshot_id, replaced_files, second_snapshot, + file_io_), + IsError(ErrorKind::kValidationFailed)); +} + +TEST_F(MergingSnapshotUpdateTest, + ValidateNoNewDeletesForDataFilesFailsOnPositionDeleteWhenIgnoringEqualityDeletes) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeDeleteFile("/delete/pos_del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewOverwriteUpdate()); + EXPECT_THAT(op->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = op->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + DataFileSet replaced_files; + replaced_files.insert(file_a_); + EXPECT_THAT(TestMergeAppend::ValidateNoNewDeletesForDataFilesForTest( + *metadata, first_snapshot->snapshot_id, replaced_files, second_snapshot, + file_io_, /*ignore_equality_deletes=*/true), + IsError(ErrorKind::kValidationFailed)); +} + +TEST_F(MergingSnapshotUpdateTest, + ValidateNoNewDeletesForDataFilesUsesConfiguredCaseSensitivity) { + CommitFileA(); + ICEBERG_UNWRAP_OR_FAIL(auto first_snapshot, table_->current_snapshot()); + + auto del_file = MakeEqualityDeleteFile("/delete/del_a.parquet", 1L); + ICEBERG_UNWRAP_OR_FAIL(auto overwrite, NewOverwriteUpdate()); + EXPECT_THAT(overwrite->AddDelete(del_file), IsOk()); + const int64_t second_snapshot_id = overwrite->GeneratedSnapshotId(); + ICEBERG_UNWRAP_OR_FAIL(auto manifests, + overwrite->Apply(*table_->metadata(), first_snapshot)); + ICEBERG_UNWRAP_OR_FAIL( + auto second_snapshot, + MakeSyntheticSnapshot(DataOperation::kOverwrite, second_snapshot_id, + first_snapshot->snapshot_id, + first_snapshot->sequence_number + 1, manifests)); + + auto metadata = std::make_shared(*table_->metadata()); + metadata->snapshots.push_back(second_snapshot); + metadata->current_snapshot_id = second_snapshot->snapshot_id; + metadata->last_sequence_number = second_snapshot->sequence_number; + + DataFileSet replaced_files; + replaced_files.insert(file_a_); + ICEBERG_UNWRAP_OR_FAIL(auto validate, NewMergeAppend()); + validate->SetCaseSensitive(false); + EXPECT_THAT(validate->ValidateNoNewDeletesForDataFiles( + *metadata, first_snapshot->snapshot_id, + Expressions::Equal("X", Literal::Long(1L)), replaced_files, + second_snapshot, file_io_), + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, @@ -1190,7 +1458,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNoNewDeleteFilesWithExpressionDetectsC EXPECT_THAT(TestMergeAppend::ValidateNoNewDeleteFilesForTest( *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), second_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, @@ -1246,7 +1514,7 @@ TEST_F(MergingSnapshotUpdateTest, EXPECT_THAT(TestMergeAppend::ValidateNoNewDeleteFilesForTest( *metadata, first_snapshot->snapshot_id, partition_set, second_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsDetectsConflict) { @@ -1285,7 +1553,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsDetectsConflict) { EXPECT_THAT(TestMergeAppend::ValidateAddedDVsForTest( *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), referenced_data_files, second_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, ValidateAddedDVsIgnoresUnrelatedDVs) { @@ -1388,7 +1656,7 @@ TEST_F(MergingSnapshotUpdateTest, ValidateDeletedDataFilesWithExpressionDetectsC EXPECT_THAT(TestMergeAppend::ValidateDeletedDataFilesForTest( *metadata, first_snapshot->snapshot_id, Expressions::AlwaysTrue(), second_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } TEST_F(MergingSnapshotUpdateTest, @@ -1416,16 +1684,15 @@ TEST_F(MergingSnapshotUpdateTest, EXPECT_THAT(TestMergeAppend::ValidateDeletedDataFilesForTest( *metadata, first_snapshot->snapshot_id, partition_set, second_snapshot, file_io_), - IsError(ErrorKind::kInvalidArgument)); + IsError(ErrorKind::kValidationFailed)); } // ------------------------------------------------------------------------- -// DataSpec — multiple partition specs +// DataSpec multiple partition specs // ------------------------------------------------------------------------- TEST_F(MergingSnapshotUpdateTest, DataSpecThrowsWithMultipleSpecs) { ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); - // file_a_ and file_b_ both use spec_id 0 — DataSpec() should succeed. EXPECT_THAT(op->AddFile(file_a_), IsOk()); EXPECT_THAT(op->AddFile(file_b_), IsOk()); EXPECT_THAT(op->DataSpec(), IsOk()); @@ -1433,7 +1700,6 @@ TEST_F(MergingSnapshotUpdateTest, DataSpecThrowsWithMultipleSpecs) { TEST_F(MergingSnapshotUpdateTest, DataSpecThrowsWhenEmpty) { ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); - // No files added — DataSpec() should fail. EXPECT_THAT(op->DataSpec(), IsError(ErrorKind::kInvalidArgument)); } diff --git a/src/iceberg/update/fast_append.cc b/src/iceberg/update/fast_append.cc index d08f497cf..24f8e744f 100644 --- a/src/iceberg/update/fast_append.cc +++ b/src/iceberg/update/fast_append.cc @@ -131,7 +131,7 @@ std::unordered_map FastAppend::Summary() { return summary_.Build(); } -void FastAppend::CleanUncommitted(const std::unordered_set& committed) { +Status FastAppend::CleanUncommitted(const std::unordered_set& committed) { // Clean up new manifests that were written but not committed if (!new_manifests_.empty()) { for (const auto& manifest : new_manifests_) { @@ -152,6 +152,7 @@ void FastAppend::CleanUncommitted(const std::unordered_set& committ } } } + return {}; } bool FastAppend::CleanupAfterCommit() const { diff --git a/src/iceberg/update/fast_append.h b/src/iceberg/update/fast_append.h index 580fa4722..a04786c88 100644 --- a/src/iceberg/update/fast_append.h +++ b/src/iceberg/update/fast_append.h @@ -72,7 +72,7 @@ class ICEBERG_EXPORT FastAppend : public SnapshotUpdate { const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) override; std::unordered_map Summary() override; - void CleanUncommitted(const std::unordered_set& committed) override; + Status CleanUncommitted(const std::unordered_set& committed) override; bool CleanupAfterCommit() const override; private: diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index acf4cd171..a0d882d14 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -20,7 +20,9 @@ #include "iceberg/update/merging_snapshot_update.h" #include +#include #include +#include #include #include #include @@ -28,8 +30,10 @@ #include "iceberg/constants.h" #include "iceberg/delete_file_index.h" #include "iceberg/expression/expressions.h" -#include "iceberg/expression/inclusive_metrics_evaluator.h" +#include "iceberg/expression/manifest_evaluator.h" +#include "iceberg/expression/projections.h" #include "iceberg/manifest/manifest_entry.h" +#include "iceberg/manifest/manifest_group.h" #include "iceberg/manifest/manifest_list.h" #include "iceberg/manifest/manifest_reader.h" #include "iceberg/manifest/manifest_util_internal.h" @@ -49,8 +53,19 @@ namespace iceberg { namespace { +const std::array kValidateAddedFilesOperations = { + DataOperation::kAppend, DataOperation::kOverwrite}; +const std::array kValidateDataFilesExistOperations = { + DataOperation::kOverwrite, DataOperation::kReplace, DataOperation::kDelete}; +const std::array kValidateDataFilesExistSkipDeleteOperations = { + DataOperation::kOverwrite, DataOperation::kReplace}; +const std::array kValidateAddedDeleteFilesOperations = { + DataOperation::kOverwrite, DataOperation::kDelete}; +const std::array kValidateAddedDVsOperations = { + DataOperation::kOverwrite, DataOperation::kDelete, DataOperation::kReplace}; + bool MatchesOperation(std::optional operation, - std::initializer_list expected) { + std::span expected) { return operation.has_value() && std::ranges::find(expected, operation.value()) != expected.end(); } @@ -60,33 +75,6 @@ struct ValidationHistoryResult { std::unordered_set snapshot_ids; }; -struct DeleteFileObjectKey { - std::string path; - std::optional content_offset; - std::optional content_size_in_bytes; - - bool operator==(const DeleteFileObjectKey& other) const = default; -}; - -struct DeleteFileObjectKeyHash { - size_t operator()(const DeleteFileObjectKey& key) const { - size_t hash = std::hash{}(key.path); - auto combine = [&hash](const auto& value) { - size_t value_hash = value.has_value() ? std::hash{}(*value) : 0; - hash ^= value_hash + 0x9e3779b9 + (hash << 6) + (hash >> 2); - }; - combine(key.content_offset); - combine(key.content_size_in_bytes); - return hash; - } -}; - -DeleteFileObjectKey MakeDeleteFileObjectKey(const DataFile& file) { - return DeleteFileObjectKey{.path = file.file_path, - .content_offset = file.content_offset, - .content_size_in_bytes = file.content_size_in_bytes}; -} - Result>> ValidationAncestorsBetween( const TableMetadata& metadata, int64_t latest_snapshot_id, std::optional starting_snapshot_id) { @@ -97,7 +85,7 @@ Result>> ValidationAncestorsBetween( if (!ancestors.empty()) { const auto& oldest_checked = ancestors.back(); if (oldest_checked == nullptr || oldest_checked->parent_snapshot_id.has_value()) { - return InvalidArgument( + return ValidationFailed( "Cannot validate history: cannot determine complete history for snapshot {}", latest_snapshot_id); } @@ -109,7 +97,7 @@ Result>> ValidationAncestorsBetween( return ancestors; } if (ancestors.empty()) { - return InvalidArgument( + return ValidationFailed( "Cannot validate history: starting snapshot {} is not an ancestor " "of snapshot {}", starting_snapshot_id.value(), latest_snapshot_id); @@ -118,7 +106,7 @@ Result>> ValidationAncestorsBetween( const auto& oldest_checked = ancestors.back(); if (oldest_checked == nullptr || !oldest_checked->parent_snapshot_id.has_value() || oldest_checked->parent_snapshot_id.value() != starting_snapshot_id.value()) { - return InvalidArgument( + return ValidationFailed( "Cannot validate history: starting snapshot {} is not an ancestor " "of snapshot {}", starting_snapshot_id.value(), latest_snapshot_id); @@ -129,7 +117,7 @@ Result>> ValidationAncestorsBetween( Result ValidationHistory( const TableMetadata& metadata, int64_t latest_snapshot_id, std::optional starting_snapshot_id, - std::initializer_list matching_operations, ManifestContent content, + std::span matching_operations, ManifestContent content, const std::shared_ptr& io) { ICEBERG_ASSIGN_OR_RAISE( auto ancestors, @@ -156,42 +144,255 @@ Result ValidationHistory( return result; } -Result> FindMatchingDataFile( - const TableMetadata& metadata, const std::vector& manifests, - const std::unordered_set& snapshot_ids, ManifestStatus status, - std::shared_ptr filter, const PartitionSet* partition_set, - const std::shared_ptr& io, bool case_sensitive) { +Result>> PartitionSpecsByIdMap( + const TableMetadata& metadata) { + TableMetadataCache metadata_cache(&metadata); + ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, metadata_cache.GetPartitionSpecsById()); + return std::unordered_map>( + specs_ref.get().begin(), specs_ref.get().end()); +} + +Result> MakeValidationManifestGroup( + const TableMetadata& metadata, const std::shared_ptr& io, + std::vector manifests) { + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, PartitionSpecsByIdMap(metadata)); + return ManifestGroup::Make(io, std::move(schema), std::move(specs_by_id), + std::move(manifests)); +} + +Result StartingSequenceNumber(const TableMetadata& metadata, + std::optional starting_snapshot_id) { + if (starting_snapshot_id.has_value()) { + auto snapshot = metadata.SnapshotById(starting_snapshot_id.value()); + if (snapshot.has_value()) { + return snapshot.value()->sequence_number; + } + } + return TableMetadata::kInitialSequenceNumber; +} + +Result> BuildDeleteFileIndex( + const TableMetadata& metadata, const std::shared_ptr& io, + std::vector delete_manifests, int64_t starting_sequence_number, + std::shared_ptr data_filter, std::shared_ptr partition_set, + bool case_sensitive) { ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); - auto partition_filter = partition_set != nullptr - ? std::make_shared(*partition_set) - : std::shared_ptr{}; + ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, PartitionSpecsByIdMap(metadata)); + ICEBERG_ASSIGN_OR_RAISE(auto builder, DeleteFileIndex::BuilderFor( + io, std::move(schema), std::move(specs_by_id), + std::move(delete_manifests))); + builder.AfterSequenceNumber(starting_sequence_number); + builder.CaseSensitive(case_sensitive); + if (data_filter != nullptr) { + builder.DataFilter(std::move(data_filter)); + } + if (partition_set != nullptr) { + builder.FilterPartitions(std::move(partition_set)); + } + return builder.Build(); +} + +Result> FilterManifestsByPartition( + const TableMetadata& metadata, std::shared_ptr conflict_detection_filter, + const std::vector& manifests, bool case_sensitive) { + if (conflict_detection_filter == nullptr || + conflict_detection_filter->op() == Expression::Operation::kTrue) { + return manifests; + } + + const int32_t default_spec_id = metadata.default_spec_id; + if (std::ranges::any_of(manifests, [default_spec_id](const ManifestFile& manifest) { + return manifest.partition_spec_id != default_spec_id; + })) { + return manifests; + } + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, PartitionSpecsByIdMap(metadata)); + std::unordered_map> eval_cache; + std::vector matching_manifests; for (const auto& manifest : manifests) { - ICEBERG_ASSIGN_OR_RAISE(auto spec, - metadata.PartitionSpecById(manifest.partition_spec_id)); - ICEBERG_ASSIGN_OR_RAISE(auto reader, - ManifestReader::Make(manifest, io, schema, spec)); - reader->CaseSensitive(case_sensitive); - if (filter != nullptr) { - reader->FilterRows(filter); + auto it = eval_cache.find(manifest.partition_spec_id); + if (it == eval_cache.end()) { + auto spec_it = specs_by_id.find(manifest.partition_spec_id); + if (spec_it == specs_by_id.end()) { + return InvalidArgument("Cannot find partition spec ID {}", + manifest.partition_spec_id); + } + + auto projector = Projections::Inclusive(*spec_it->second, *schema, case_sensitive); + ICEBERG_ASSIGN_OR_RAISE(auto partition_filter, + projector->Project(conflict_detection_filter)); + ICEBERG_ASSIGN_OR_RAISE( + auto evaluator, + ManifestEvaluator::MakePartitionFilter( + std::move(partition_filter), spec_it->second, *schema, case_sensitive)); + it = eval_cache.emplace(manifest.partition_spec_id, std::move(evaluator)).first; } - if (partition_filter != nullptr) { - reader->FilterPartitions(partition_filter); + + ICEBERG_ASSIGN_OR_RAISE(auto matches, it->second->Evaluate(manifest)); + if (matches) { + matching_manifests.push_back(manifest); } + } + return matching_manifests; +} - ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); - for (const auto& entry : entries) { - if (!entry.snapshot_id.has_value() || - !snapshot_ids.contains(entry.snapshot_id.value())) { - continue; - } - if (entry.status == status && entry.data_file != nullptr) { - return entry.data_file->file_path; - } +void FilterManifestEntriesByPartitionSet(ManifestGroup& group, + const PartitionSet* partition_set) { + if (partition_set != nullptr) { + auto partitions = std::make_shared(*partition_set); + group.FilterManifestEntries([partitions](const ManifestEntry& entry) { + return entry.data_file != nullptr && + entry.data_file->partition_spec_id.has_value() && + partitions->contains(entry.data_file->partition_spec_id.value(), + entry.data_file->partition); + }); + } +} + +Result> MatchingAddedDataFiles( + const TableMetadata& metadata, std::optional starting_snapshot_id, + std::shared_ptr data_filter, const PartitionSet* partition_set, + const std::shared_ptr& parent, const std::shared_ptr& io, + bool case_sensitive) { + if (parent == nullptr) { + return std::vector{}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + kValidateAddedFilesOperations, ManifestContent::kData, io)); + auto new_snapshots = + std::make_shared>(std::move(history.snapshot_ids)); + ICEBERG_ASSIGN_OR_RAISE(auto group, MakeValidationManifestGroup( + metadata, io, std::move(history.manifests))); + group->CaseSensitive(case_sensitive) + .FilterManifestEntries([new_snapshots](const ManifestEntry& entry) { + return entry.snapshot_id.has_value() && + new_snapshots->contains(entry.snapshot_id.value()) && + entry.data_file != nullptr; + }) + .IgnoreDeleted() + .IgnoreExisting(); + if (data_filter != nullptr) { + group->FilterData(std::move(data_filter)); + } + FilterManifestEntriesByPartitionSet(*group, partition_set); + return group->Entries(); +} + +Result> MatchingDeletedDataFiles( + const TableMetadata& metadata, std::optional starting_snapshot_id, + std::shared_ptr data_filter, const PartitionSet* partition_set, + const std::shared_ptr& parent, const std::shared_ptr& io, + bool case_sensitive) { + if (parent == nullptr) { + return std::vector{}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + kValidateDataFilesExistOperations, ManifestContent::kData, io)); + auto new_snapshots = + std::make_shared>(std::move(history.snapshot_ids)); + ICEBERG_ASSIGN_OR_RAISE(auto group, MakeValidationManifestGroup( + metadata, io, std::move(history.manifests))); + group->CaseSensitive(case_sensitive) + .FilterManifestEntries([new_snapshots](const ManifestEntry& entry) { + return entry.snapshot_id.has_value() && + new_snapshots->contains(entry.snapshot_id.value()); + }) + .FilterManifestEntries([](const ManifestEntry& entry) { + return entry.status == ManifestStatus::kDeleted && entry.data_file != nullptr; + }) + .IgnoreExisting(); + if (data_filter != nullptr) { + group->FilterData(std::move(data_filter)); + } + FilterManifestEntriesByPartitionSet(*group, partition_set); + return group->Entries(); +} + +std::string FormatLocations(std::vector locations) { + std::string result = "["; + for (size_t i = 0; i < locations.size(); ++i) { + if (i > 0) { + result += ", "; } + result += locations[i]; } + result += "]"; + return result; +} - return std::optional{}; +std::vector DataFilePaths(const std::vector& entries) { + std::vector paths; + paths.reserve(entries.size()); + for (const auto& entry : entries) { + if (entry.data_file != nullptr) { + paths.push_back(entry.data_file->file_path); + } + } + return paths; +} + +std::optional DataFileLocations(const std::vector& entries) { + auto paths = DataFilePaths(entries); + if (paths.empty()) { + return std::optional{}; + } + return FormatLocations(std::move(paths)); +} + +std::optional DeleteFileLocations( + const std::vector>& delete_files) { + std::vector paths; + paths.reserve(delete_files.size()); + for (const auto& delete_file : delete_files) { + if (delete_file != nullptr) { + paths.push_back(delete_file->file_path); + } + } + if (paths.empty()) { + return std::optional{}; + } + return FormatLocations(std::move(paths)); +} + +Status ValidateAddedDVsInManifest( + const TableMetadata& metadata, const ManifestFile& manifest, + std::shared_ptr conflict_detection_filter, + const std::unordered_set& new_snapshot_ids, + const std::unordered_set& referenced_data_files, + const std::shared_ptr& io, const std::shared_ptr& schema, + bool case_sensitive) { + ICEBERG_ASSIGN_OR_RAISE(auto spec, + metadata.PartitionSpecById(manifest.partition_spec_id)); + ICEBERG_ASSIGN_OR_RAISE(auto reader, ManifestReader::Make(manifest, io, schema, spec)); + reader->CaseSensitive(case_sensitive); + reader->FilterRows(std::move(conflict_detection_filter)); + ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); + + for (const auto& entry : entries) { + if (!entry.snapshot_id.has_value() || + !new_snapshot_ids.contains(entry.snapshot_id.value())) { + continue; + } + if (entry.data_file == nullptr || !ContentFileUtil::IsDV(*entry.data_file) || + !entry.data_file->referenced_data_file.has_value()) { + continue; + } + if (referenced_data_files.contains(*entry.data_file->referenced_data_file)) { + return ValidationFailed("Found concurrently added DV for {}: {}", + *entry.data_file->referenced_data_file, + ContentFileUtil::DVDesc(*entry.data_file)); + } + } + return {}; } } // namespace @@ -200,17 +401,52 @@ MergingSnapshotUpdate::MergingSnapshotUpdate(std::string table_name, std::shared_ptr ctx) : SnapshotUpdate(std::move(ctx)), table_name_(std::move(table_name)), - delete_expression_(Expressions::AlwaysFalse()), - data_filter_manager_(ManifestContent::kData, ctx_->table->io()), - delete_filter_manager_(ManifestContent::kDeletes, ctx_->table->io()), - data_merge_manager_( - base().properties.Get(TableProperties::kManifestTargetSizeBytes), - base().properties.Get(TableProperties::kManifestMinMergeCount), - base().properties.Get(TableProperties::kManifestMergeEnabled)), - delete_merge_manager_( - base().properties.Get(TableProperties::kManifestTargetSizeBytes), - base().properties.Get(TableProperties::kManifestMinMergeCount), - base().properties.Get(TableProperties::kManifestMergeEnabled)) {} + delete_expression_(Expressions::AlwaysFalse()) { + auto file_io = ctx_->table->io(); + auto data_filter_manager = ManifestFilterManager::Make( + ManifestContent::kData, file_io, + [this](const std::string& location) { return DeleteFile(location); }); + if (!data_filter_manager.has_value()) { + AddError(data_filter_manager.error()); + } else { + data_filter_manager_ = std::move(data_filter_manager.value()); + } + + auto delete_filter_manager = ManifestFilterManager::Make( + ManifestContent::kDeletes, file_io, + [this](const std::string& location) { return DeleteFile(location); }); + if (!delete_filter_manager.has_value()) { + AddError(delete_filter_manager.error()); + } else { + delete_filter_manager_ = std::move(delete_filter_manager.value()); + } + + const int64_t target_size_bytes = + base().properties.Get(TableProperties::kManifestTargetSizeBytes); + const int32_t min_count_to_merge = + base().properties.Get(TableProperties::kManifestMinMergeCount); + const bool merge_enabled = + base().properties.Get(TableProperties::kManifestMergeEnabled); + auto data_merge_manager = ManifestMergeManager::Make( + ManifestContent::kData, target_size_bytes, min_count_to_merge, merge_enabled, + file_io, [this] { return SnapshotId(); }, + [this](const std::string& location) { return DeleteFile(location); }); + if (!data_merge_manager.has_value()) { + AddError(data_merge_manager.error()); + } else { + data_merge_manager_ = std::move(data_merge_manager.value()); + } + + auto delete_merge_manager = ManifestMergeManager::Make( + ManifestContent::kDeletes, target_size_bytes, min_count_to_merge, merge_enabled, + file_io, [this] { return SnapshotId(); }, + [this](const std::string& location) { return DeleteFile(location); }); + if (!delete_merge_manager.has_value()) { + AddError(delete_merge_manager.error()); + } else { + delete_merge_manager_ = std::move(delete_merge_manager.value()); + } +} // ------------------------------------------------------------------------- // Primitive API @@ -227,15 +463,16 @@ Status MergingSnapshotUpdate::AddDataFile(std::shared_ptr file) { int32_t spec_id = file->partition_spec_id.value(); ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(spec_id)); - // Suppress first_row_id — it will be assigned by the commit, not inherited from the - // source file. - file->first_row_id = std::nullopt; + // Suppress first_row_id in the staged copy. The commit assigns row IDs for newly + // added files and must not mutate the caller-owned file object. + auto staged_file = std::make_shared(*file); + staged_file->first_row_id = std::nullopt; auto& data_files = new_data_files_by_spec_[spec_id]; - auto [it, inserted] = data_files.insert(file); + auto [it, inserted] = data_files.insert(staged_file); if (inserted) { has_new_data_files_ = true; - ICEBERG_RETURN_UNEXPECTED(added_data_files_summary_.AddedFile(*spec, *file)); + ICEBERG_RETURN_UNEXPECTED(added_data_files_summary_.AddedFile(*spec, *staged_file)); } return {}; } @@ -259,7 +496,8 @@ Status MergingSnapshotUpdate::ValidateNewDeleteFile(const TableMetadata& metadat } break; default: - if (format_version >= 3) { + if (format_version >= 3 && + format_version <= TableMetadata::kSupportedTableFormatVersion) { // Position deletes MUST be DVs in v3+. if (file.content == DataFile::Content::kPositionDeletes && !is_dv) { return InvalidArgument("Must use DVs for position deletes in V{}: {}", @@ -282,6 +520,16 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, return AddDeleteFile(std::move(file), std::optional(data_sequence_number)); } +void MergingSnapshotUpdate::PendingDeleteFilesByReferencedFile::Add( + std::string referenced_file, PendingDeleteFile file) { + auto [iter, inserted] = + index_by_referenced_file_.try_emplace(referenced_file, entries_.size()); + if (inserted) { + entries_.push_back(Entry{.referenced_file = std::move(referenced_file), .files = {}}); + } + entries_[iter->second].files.push_back(std::move(file)); +} + Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, std::optional data_sequence_number) { if (!file) { @@ -293,8 +541,17 @@ Status MergingSnapshotUpdate::AddDeleteFile(std::shared_ptr file, } ICEBERG_RETURN_UNEXPECTED(base().PartitionSpecById(file->partition_spec_id.value())); has_new_delete_files_ = true; - new_delete_files_.push_back(PendingDeleteFile{ - .file = std::move(file), .data_sequence_number = std::move(data_sequence_number)}); + PendingDeleteFile pending_file{.file = std::move(file), + .data_sequence_number = std::move(data_sequence_number)}; + if (ContentFileUtil::IsDV(*pending_file.file)) { + ICEBERG_PRECHECK(pending_file.file->referenced_data_file.has_value(), + "DV must have a referenced data file: {}", + pending_file.file->file_path); + auto referenced_data_file = *pending_file.file->referenced_data_file; + dvs_by_referenced_file_.Add(std::move(referenced_data_file), std::move(pending_file)); + } else { + v2_deletes_.push_back(std::move(pending_file)); + } return {}; } @@ -302,42 +559,44 @@ Status MergingSnapshotUpdate::DeleteDataFile(std::shared_ptr file) { if (!file) { return InvalidArgument("Cannot delete a null data file"); } - return data_filter_manager_.DeleteFile(std::move(file)); + return data_filter_manager_->DeleteFile(std::move(file)); } Status MergingSnapshotUpdate::DeleteDeleteFile(std::shared_ptr file) { if (!file) { return InvalidArgument("Cannot delete a null delete file"); } - return delete_filter_manager_.DeleteFile(std::move(file)); + return delete_filter_manager_->DeleteFile(std::move(file)); } -void MergingSnapshotUpdate::DeleteByPath(std::string_view path) { - data_filter_manager_.DeleteFile(path); +Status MergingSnapshotUpdate::DeleteByPath(std::string_view path) { + return data_filter_manager_->DeleteFile(path); } Status MergingSnapshotUpdate::DeleteByRowFilter(std::shared_ptr expr) { // If a delete file matches the row filter, it can also be removed because the rows // it references will also be deleted. Both filter managers receive the expression. delete_expression_ = expr; - ICEBERG_RETURN_UNEXPECTED(data_filter_manager_.DeleteByRowFilter(expr)); - return delete_filter_manager_.DeleteByRowFilter(std::move(expr)); + ICEBERG_RETURN_UNEXPECTED(data_filter_manager_->DeleteByRowFilter(expr)); + return delete_filter_manager_->DeleteByRowFilter(std::move(expr)); } -void MergingSnapshotUpdate::DropPartition(int32_t spec_id, PartitionValues partition) { +Status MergingSnapshotUpdate::DropPartition(int32_t spec_id, PartitionValues partition) { // Dropping data in a partition also drops all delete files in that partition. - data_filter_manager_.DropPartition(spec_id, partition); - delete_filter_manager_.DropPartition(spec_id, std::move(partition)); + ICEBERG_RETURN_UNEXPECTED(data_filter_manager_->DropPartition(spec_id, partition)); + ICEBERG_RETURN_UNEXPECTED( + delete_filter_manager_->DropPartition(spec_id, std::move(partition))); + return {}; } void MergingSnapshotUpdate::FailMissingDeletePaths() { - data_filter_manager_.FailMissingDeletePaths(); - delete_filter_manager_.FailMissingDeletePaths(); + data_filter_manager_->FailMissingDeletePaths(); + delete_filter_manager_->FailMissingDeletePaths(); } void MergingSnapshotUpdate::FailAnyDelete() { - data_filter_manager_.FailAnyDelete(); - delete_filter_manager_.FailAnyDelete(); + data_filter_manager_->FailAnyDelete(); + delete_filter_manager_->FailAnyDelete(); } void MergingSnapshotUpdate::SetNewDataFilesDataSequenceNumber(int64_t sequence_number) { @@ -346,13 +605,14 @@ void MergingSnapshotUpdate::SetNewDataFilesDataSequenceNumber(int64_t sequence_n void MergingSnapshotUpdate::CaseSensitive(bool case_sensitive) { case_sensitive_ = case_sensitive; - data_filter_manager_.CaseSensitive(case_sensitive); - delete_filter_manager_.CaseSensitive(case_sensitive); + data_filter_manager_->CaseSensitive(case_sensitive); + delete_filter_manager_->CaseSensitive(case_sensitive); } -void MergingSnapshotUpdate::Set(const std::string& property, const std::string& value) { +void MergingSnapshotUpdate::SetSummaryProperty(const std::string& property, + const std::string& value) { custom_summary_properties_[property] = value; - summary_builder().Set(property, value); + SnapshotUpdate::SetSummaryProperty(property, value); } Result> MergingSnapshotUpdate::DataSpec() const { @@ -401,7 +661,6 @@ Result MergingSnapshotUpdate::CopyManifest(const ManifestFile& man ICEBERG_ASSIGN_OR_RAISE(auto spec, current.PartitionSpecById(manifest.partition_spec_id)); std::string path = ManifestPath(); - all_written_manifests_.insert(path); return CopyAppendManifest(manifest, ctx_->table->io(), schema, spec, SnapshotId(), path, current.format_version, &appended_manifests_summary_); } @@ -414,21 +673,34 @@ bool MergingSnapshotUpdate::AddsDataFiles() const { return !new_data_files_by_spec_.empty(); } -bool MergingSnapshotUpdate::AddsDeleteFiles() const { return !new_delete_files_.empty(); } +bool MergingSnapshotUpdate::AddsDeleteFiles() const { + return !v2_deletes_.empty() || !dvs_by_referenced_file_.empty(); +} bool MergingSnapshotUpdate::DeletesDataFiles() const { - return data_filter_manager_.ContainsDeletes(); + return data_filter_manager_->ContainsDeletes(); } bool MergingSnapshotUpdate::DeletesDeleteFiles() const { - return delete_filter_manager_.ContainsDeletes(); + return delete_filter_manager_->ContainsDeletes(); +} + +Status MergingSnapshotUpdate::ManagersReady() const { + ICEBERG_CHECK(data_filter_manager_ != nullptr, + "Data filter manager is not initialized"); + ICEBERG_CHECK(delete_filter_manager_ != nullptr, + "Delete filter manager is not initialized"); + ICEBERG_CHECK(data_merge_manager_ != nullptr, "Data merge manager is not initialized"); + ICEBERG_CHECK(delete_merge_manager_ != nullptr, + "Delete merge manager is not initialized"); + return {}; } // ------------------------------------------------------------------------- // Apply pipeline // ------------------------------------------------------------------------- -ManifestWriterFactory MergingSnapshotUpdate::MakeTrackedWriterFactory( +ManifestWriterFactory MergingSnapshotUpdate::MakeWriterFactory( const std::shared_ptr& schema) { return [this, schema](int32_t spec_id, @@ -436,7 +708,6 @@ ManifestWriterFactory MergingSnapshotUpdate::MakeTrackedWriterFactory( const TableMetadata& meta = base(); ICEBERG_ASSIGN_OR_RAISE(auto spec, meta.PartitionSpecById(spec_id)); std::string path = ManifestPath(); - all_written_manifests_.insert(path); return ManifestWriter::MakeWriter(meta.format_version, SnapshotId(), std::move(path), ctx_->table->io(), std::move(spec), schema, content); @@ -445,15 +716,15 @@ ManifestWriterFactory MergingSnapshotUpdate::MakeTrackedWriterFactory( Result> MergingSnapshotUpdate::WriteNewDataManifests() { // If new files were staged after the cache was populated (commit retry), invalidate. - if (has_new_data_files_ && cached_new_data_manifests_.has_value()) { - for (const auto& m : *cached_new_data_manifests_) { + if (has_new_data_files_ && !cached_new_data_manifests_.empty()) { + for (const auto& m : cached_new_data_manifests_) { std::ignore = DeleteFile(m.manifest_path); } - cached_new_data_manifests_.reset(); + cached_new_data_manifests_.clear(); } - if (cached_new_data_manifests_.has_value()) { - return *cached_new_data_manifests_; + if (!cached_new_data_manifests_.empty()) { + return cached_new_data_manifests_; } std::vector result; @@ -462,9 +733,6 @@ Result> MergingSnapshotUpdate::WriteNewDataManifests() ICEBERG_ASSIGN_OR_RAISE( auto written, WriteDataManifests(data_files.as_span(), spec, new_data_files_data_seq_number_)); - for (const auto& m : written) { - all_written_manifests_.insert(m.manifest_path); - } result.insert(result.end(), std::make_move_iterator(written.begin()), std::make_move_iterator(written.end())); } @@ -475,36 +743,26 @@ Result> MergingSnapshotUpdate::WriteNewDataManifests() } Result> -MergingSnapshotUpdate::NormalizeNewDeleteFiles() const { +MergingSnapshotUpdate::MergeDVs() const { std::vector result; - result.reserve(new_delete_files_.size()); - - std::unordered_set seen_delete_files; - std::unordered_map dv_by_referenced_data_file; - - for (const auto& pending_file : new_delete_files_) { - const auto& file = pending_file.file; - ICEBERG_PRECHECK(file != nullptr, "Cannot add a null delete file"); + result.reserve(dvs_by_referenced_file_.size()); - auto key = MakeDeleteFileObjectKey(*file); - if (!seen_delete_files.insert(key).second) { + for (const auto& entry : dvs_by_referenced_file_.entries()) { + const auto& referenced_file = entry.referenced_file; + const auto& dvs = entry.files; + if (dvs.empty()) { continue; } - - if (ContentFileUtil::IsDV(*file)) { - ICEBERG_PRECHECK(file->referenced_data_file.has_value(), - "DV must have a referenced data file: {}", file->file_path); - auto [it, inserted] = - dv_by_referenced_data_file.emplace(*file->referenced_data_file, key); - if (!inserted && it->second != key) { - return NotImplemented( - "Merging multiple deletion vectors is not supported yet for referenced " - "data file: {}", - *file->referenced_data_file); - } + if (dvs.size() > 1) { + // TODO(Guotao): Merge duplicate DVs for one referenced data file once C++ + // has DVUtil/Puffin DV rewriting; Java merges them before writing manifests. + return NotImplemented( + "Merging multiple deletion vectors is not supported yet for referenced " + "data file: {}", + referenced_file); } - result.push_back(pending_file); + result.push_back(dvs.front()); } return result; @@ -512,39 +770,53 @@ MergingSnapshotUpdate::NormalizeNewDeleteFiles() const { Result> MergingSnapshotUpdate::WriteNewDeleteManifests() { // If new files were staged after the cache was populated (commit retry), invalidate. - if (has_new_delete_files_ && cached_new_delete_manifests_.has_value()) { - for (const auto& m : *cached_new_delete_manifests_) { + if (has_new_delete_files_ && !cached_new_delete_manifests_.empty()) { + for (const auto& m : cached_new_delete_manifests_) { std::ignore = DeleteFile(m.manifest_path); } - cached_new_delete_manifests_.reset(); + cached_new_delete_manifests_.clear(); + added_delete_files_summary_.Clear(); } - if (cached_new_delete_manifests_.has_value()) { - return *cached_new_delete_manifests_; + if (!cached_new_delete_manifests_.empty()) { + return cached_new_delete_manifests_; } - // Group delete files by partition spec ID, mirroring WriteNewDataManifests(). + ICEBERG_ASSIGN_OR_RAISE(auto merged_dvs, MergeDVs()); + + std::vector new_delete_files; + new_delete_files.reserve(merged_dvs.size() + v2_deletes_.size()); + new_delete_files.insert(new_delete_files.end(), merged_dvs.begin(), merged_dvs.end()); + + DeleteFileSet v2_delete_set; + for (const auto& pending_file : v2_deletes_) { + if (v2_delete_set.insert(pending_file.file).second) { + new_delete_files.push_back(pending_file); + } + } + + // Group delete files by partition spec ID, mirroring Java newDeleteFilesAsManifests(). std::unordered_map> delete_files_by_spec; - for (const auto& pending_file : new_delete_files_) { + for (const auto& pending_file : new_delete_files) { delete_files_by_spec[pending_file.file->partition_spec_id.value()].push_back( pending_file); } std::vector result; + added_delete_files_summary_.Clear(); for (auto& [spec_id, delete_files] : delete_files_by_spec) { ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(spec_id)); - std::vector delete_entries; + std::vector delete_entries; delete_entries.reserve(delete_files.size()); for (const auto& pending_file : delete_files) { - delete_entries.push_back(DeleteManifestEntry{ + ICEBERG_RETURN_UNEXPECTED( + added_delete_files_summary_.AddedFile(*spec, *pending_file.file)); + delete_entries.push_back(ContentFileWithSequenceNumber{ .file = pending_file.file, .data_sequence_number = pending_file.data_sequence_number, }); } ICEBERG_ASSIGN_OR_RAISE(auto written, WriteDeleteManifests(delete_entries, spec)); - for (const auto& m : written) { - all_written_manifests_.insert(m.manifest_path); - } result.insert(result.end(), std::make_move_iterator(written.begin()), std::make_move_iterator(written.end())); } @@ -556,89 +828,58 @@ Result> MergingSnapshotUpdate::WriteNewDeleteManifests Result> MergingSnapshotUpdate::Apply( const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) { + ICEBERG_RETURN_UNEXPECTED(ManagersReady()); + // Re-validate buffered delete files against the current format version. A format // upgrade between staging and commit could make previously-valid files invalid. - for (const auto& pending_file : new_delete_files_) { + for (const auto& pending_file : v2_deletes_) { ICEBERG_RETURN_UNEXPECTED( ValidateNewDeleteFile(metadata_to_update, *pending_file.file)); } - ICEBERG_ASSIGN_OR_RAISE(auto normalized_delete_files, NormalizeNewDeleteFiles()); - new_delete_files_ = std::move(normalized_delete_files); - - added_delete_files_summary_.Clear(); - for (const auto& pending_file : new_delete_files_) { - ICEBERG_ASSIGN_OR_RAISE(auto spec, metadata_to_update.PartitionSpecById( - *pending_file.file->partition_spec_id)); - ICEBERG_RETURN_UNEXPECTED( - added_delete_files_summary_.AddedFile(*spec, *pending_file.file)); - } - - // Rebuild summary from stable sub-builders so that commit retries don't double-count. - summary_builder().Clear(); - summary_builder().Merge(added_data_files_summary_); - summary_builder().Merge(added_delete_files_summary_); - summary_builder().Merge(appended_manifests_summary_); - for (const auto& [property, value] : custom_summary_properties_) { - summary_builder().Set(property, value); + for (const auto& entry : dvs_by_referenced_file_.entries()) { + for (const auto& pending_file : entry.files) { + ICEBERG_RETURN_UNEXPECTED( + ValidateNewDeleteFile(metadata_to_update, *pending_file.file)); + } } ICEBERG_ASSIGN_OR_RAISE(auto target_schema, SnapshotUtil::SchemaFor(metadata_to_update, target_branch())); - auto tracked_factory = MakeTrackedWriterFactory(target_schema); + auto writer_factory = MakeWriterFactory(target_schema); // Step 1: Filter data manifests. - ICEBERG_ASSIGN_OR_RAISE(auto filtered_data, data_filter_manager_.FilterManifests( + ICEBERG_ASSIGN_OR_RAISE(auto filtered_data, data_filter_manager_->FilterManifests( target_schema, metadata_to_update, - snapshot, tracked_factory)); - - // Track deleted data files in the summary builder. - for (const auto& file : data_filter_manager_.DeletedFiles()) { - if (!file->partition_spec_id.has_value()) { - continue; - } - ICEBERG_ASSIGN_OR_RAISE( - auto spec, metadata_to_update.PartitionSpecById(*file->partition_spec_id)); - ICEBERG_RETURN_UNEXPECTED(summary_builder().DeletedFile(*spec, *file)); - } - summary_builder().IncrementDuplicateDeletes( - data_filter_manager_.DuplicateDeletesCount()); + snapshot, writer_factory)); // Step 2: Compute min data sequence number; set up delete filter cleanup. - // Use last_sequence_number as the initial value so that an empty filtered list - // produces a sensible minimum. Skip manifests with kUnassignedSequenceNumber — - // those are manifests written in the current Apply() call whose sequence number - // hasn't been assigned yet. If all filtered manifests are unassigned (e.g. the - // table has no pre-existing data manifests), the fallback to last_sequence_number - // is safe: any delete file with seq > 0 and seq <= last_sequence_number can no - // longer match live data rows, so cleaning them up is correct. + // Skip unassigned manifests written in this Apply() call. int64_t min_data_seq = metadata_to_update.last_sequence_number; for (const auto& manifest : filtered_data) { if (manifest.min_sequence_number != kUnassignedSequenceNumber) { min_data_seq = std::min(min_data_seq, manifest.min_sequence_number); } } - delete_filter_manager_.DropDeleteFilesOlderThan(min_data_seq); - delete_filter_manager_.RemoveDanglingDeletesFor( - data_filter_manager_.FilesToBeDeleted()); + ICEBERG_RETURN_UNEXPECTED( + delete_filter_manager_->DropDeleteFilesOlderThan(min_data_seq)); + delete_filter_manager_->RemoveDanglingDeletesFor( + data_filter_manager_->FilesToBeDeleted()); // Step 3: Filter delete manifests. - ICEBERG_ASSIGN_OR_RAISE(auto filtered_deletes, delete_filter_manager_.FilterManifests( + ICEBERG_ASSIGN_OR_RAISE(auto filtered_deletes, delete_filter_manager_->FilterManifests( target_schema, metadata_to_update, - snapshot, tracked_factory)); + snapshot, writer_factory)); - // Track deleted delete files in the summary builder. - for (const auto& file : delete_filter_manager_.DeletedFiles()) { - if (!file->partition_spec_id.has_value()) { - continue; - } - ICEBERG_ASSIGN_OR_RAISE( - auto spec, metadata_to_update.PartitionSpecById(*file->partition_spec_id)); - ICEBERG_RETURN_UNEXPECTED(summary_builder().DeletedFile(*spec, *file)); - } - summary_builder().IncrementDuplicateDeletes( - delete_filter_manager_.DuplicateDeletesCount()); + TableMetadataCache metadata_cache(&metadata_to_update); + ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, metadata_cache.GetPartitionSpecsById()); + ICEBERG_ASSIGN_OR_RAISE( + auto data_filter_summary, + data_filter_manager_->BuildSummary(filtered_data, specs_by_id.get())); + ICEBERG_ASSIGN_OR_RAISE( + auto delete_filter_summary, + delete_filter_manager_->BuildSummary(filtered_deletes, specs_by_id.get())); - // Drop manifests with no live files — they carry no data and should not be merged + // Drop manifests with no live files - they carry no data and should not be merged // into the new snapshot. Manifests written by the current snapshot are always kept // regardless of live-file counts; the merge stage handles any that are empty. int64_t snapshot_id = SnapshotId(); @@ -646,8 +887,6 @@ Result> MergingSnapshotUpdate::Apply( return m.has_added_files() || m.has_existing_files() || m.added_snapshot_id == snapshot_id; }; - std::erase_if(filtered_data, [&](const ManifestFile& m) { return !should_keep(m); }); - std::erase_if(filtered_deletes, [&](const ManifestFile& m) { return !should_keep(m); }); // Step 4: Write (or retrieve cached) new data manifests. ICEBERG_ASSIGN_OR_RAISE(auto written_data_manifests, WriteNewDataManifests()); @@ -670,17 +909,33 @@ Result> MergingSnapshotUpdate::Apply( // Step 5: Write (or retrieve cached) new delete manifests. ICEBERG_ASSIGN_OR_RAISE(auto new_delete_manifests, WriteNewDeleteManifests()); + std::erase_if(new_data_manifests, + [&](const ManifestFile& m) { return !should_keep(m); }); + std::erase_if(filtered_data, [&](const ManifestFile& m) { return !should_keep(m); }); + std::erase_if(new_delete_manifests, + [&](const ManifestFile& m) { return !should_keep(m); }); + std::erase_if(filtered_deletes, [&](const ManifestFile& m) { return !should_keep(m); }); + + // Rebuild summary from stable sub-builders so that commit retries don't double-count. + summary_builder().Clear(); + summary_builder().Merge(added_data_files_summary_); + summary_builder().Merge(added_delete_files_summary_); + summary_builder().Merge(appended_manifests_summary_); + for (const auto& [property, value] : custom_summary_properties_) { + summary_builder().Set(property, value); + } + summary_builder().Merge(data_filter_summary); + summary_builder().Merge(delete_filter_summary); + // Step 6: Merge data manifests. - ICEBERG_ASSIGN_OR_RAISE(auto merged_data, - data_merge_manager_.MergeManifests( - filtered_data, new_data_manifests, SnapshotId(), - metadata_to_update, ctx_->table->io(), tracked_factory)); + ICEBERG_ASSIGN_OR_RAISE(auto merged_data, data_merge_manager_->MergeManifests( + filtered_data, new_data_manifests, + metadata_to_update, writer_factory)); // Step 7: Merge delete manifests. - ICEBERG_ASSIGN_OR_RAISE(auto merged_deletes, - delete_merge_manager_.MergeManifests( - filtered_deletes, new_delete_manifests, SnapshotId(), - metadata_to_update, ctx_->table->io(), tracked_factory)); + ICEBERG_ASSIGN_OR_RAISE(auto merged_deletes, delete_merge_manager_->MergeManifests( + filtered_deletes, new_delete_manifests, + metadata_to_update, writer_factory)); std::vector result; result.reserve(merged_data.size() + merged_deletes.size()); @@ -689,56 +944,74 @@ Result> MergingSnapshotUpdate::Apply( result.insert(result.end(), std::make_move_iterator(merged_deletes.begin()), std::make_move_iterator(merged_deletes.end())); - // Manifest count summary. + // Manifest count summary: unassigned manifests count as neither created nor kept. int32_t manifests_created = 0; int32_t manifests_kept = 0; for (const auto& m : result) { if (m.added_snapshot_id == snapshot_id) { ++manifests_created; - } else { + } else if (m.added_snapshot_id != kInvalidSnapshotId) { ++manifests_kept; } } - int32_t replaced_manifests_count = data_filter_manager_.ReplacedManifestsCount() + - delete_filter_manager_.ReplacedManifestsCount() + - data_merge_manager_.ReplacedManifestsCount() + - delete_merge_manager_.ReplacedManifestsCount(); - summary_builder().SetManifestCounts(manifests_created, manifests_kept, - replaced_manifests_count); + int32_t replaced_manifests_count = data_filter_manager_->ReplacedManifestsCount() + + delete_filter_manager_->ReplacedManifestsCount() + + data_merge_manager_->ReplacedManifestsCount() + + delete_merge_manager_->ReplacedManifestsCount(); + summary_builder().Set(SnapshotSummaryFields::kManifestsCreated, + std::to_string(manifests_created)); + summary_builder().Set(SnapshotSummaryFields::kManifestsKept, + std::to_string(manifests_kept)); + summary_builder().Set(SnapshotSummaryFields::kManifestsReplaced, + std::to_string(replaced_manifests_count)); return result; } -void MergingSnapshotUpdate::CleanUncommitted( +Status MergingSnapshotUpdate::CleanUncommitted( const std::unordered_set& committed) { - for (const auto& path : all_written_manifests_) { - if (!committed.contains(path)) { - std::ignore = DeleteFile(path); - } + ICEBERG_RETURN_UNEXPECTED(ManagersReady()); + ICEBERG_RETURN_UNEXPECTED(data_merge_manager_->CleanUncommitted(committed)); + ICEBERG_RETURN_UNEXPECTED(data_filter_manager_->CleanUncommitted(committed)); + ICEBERG_RETURN_UNEXPECTED(delete_merge_manager_->CleanUncommitted(committed)); + ICEBERG_RETURN_UNEXPECTED(delete_filter_manager_->CleanUncommitted(committed)); + ICEBERG_RETURN_UNEXPECTED(CleanUncommittedAppends(committed)); + return {}; +} + +Status MergingSnapshotUpdate::CleanUncommittedAppends( + const std::unordered_set& committed) { + ICEBERG_RETURN_UNEXPECTED( + DeleteUncommitted(cached_new_data_manifests_, committed, /*clear=*/true)); + ICEBERG_RETURN_UNEXPECTED( + DeleteUncommitted(cached_new_delete_manifests_, committed, /*clear=*/true)); + // rewritten_append_manifests_ are always owned by the table. + ICEBERG_RETURN_UNEXPECTED( + DeleteUncommitted(rewritten_append_manifests_, committed, /*clear=*/false)); + + // append_manifests_ are only owned by the table if the commit succeeded. + if (!committed.empty()) { + ICEBERG_RETURN_UNEXPECTED( + DeleteUncommitted(append_manifests_, committed, /*clear=*/false)); } - all_written_manifests_.clear(); - cached_new_data_manifests_.reset(); - cached_new_delete_manifests_.reset(); + has_new_data_files_ = false; has_new_delete_files_ = false; + return {}; +} - // rewritten_append_manifests_ are always owned by the table (copied by us), - // so delete any that were not committed. - for (const auto& m : rewritten_append_manifests_) { - if (!committed.contains(m.manifest_path)) { - std::ignore = DeleteFile(m.manifest_path); +Status MergingSnapshotUpdate::DeleteUncommitted( + std::vector& manifests, + const std::unordered_set& committed, bool clear) { + for (const auto& manifest : manifests) { + if (!committed.contains(manifest.manifest_path)) { + std::ignore = DeleteFile(manifest.manifest_path); } } - - // append_manifests_ are only owned by the table if the commit succeeded - // (i.e., at least one manifest was committed). - if (!committed.empty()) { - for (const auto& m : append_manifests_) { - if (!committed.contains(m.manifest_path)) { - std::ignore = DeleteFile(m.manifest_path); - } - } + if (clear) { + manifests.clear(); } + return {}; } std::unordered_map MergingSnapshotUpdate::Summary() { @@ -753,105 +1026,57 @@ std::unordered_map MergingSnapshotUpdate::Summary() { Status MergingSnapshotUpdate::ValidateAddedDataFiles( const TableMetadata& metadata, std::optional starting_snapshot_id, - std::shared_ptr filter, const std::shared_ptr& parent, + std::shared_ptr data_filter, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive) { - if (parent == nullptr) { - return {}; - } - - ICEBERG_ASSIGN_OR_RAISE( - auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kAppend, DataOperation::kOverwrite}, - ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( - auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, - ManifestStatus::kAdded, filter, nullptr, io, case_sensitive)); - if (conflict_path.has_value()) { - return InvalidArgument( - "Found conflicting files that can contain rows matching {}: {}", - filter != nullptr ? filter->ToString() : "any expression", conflict_path.value()); + auto conflict_entries, + MatchingAddedDataFiles(metadata, starting_snapshot_id, data_filter, + /*partition_set=*/nullptr, parent, io, case_sensitive)); + auto conflict_paths = DataFileLocations(conflict_entries); + if (conflict_paths.has_value()) { + return ValidationFailed( + "Found conflicting files that can contain records matching {}: {}", + data_filter != nullptr ? data_filter->ToString() : "any expression", + conflict_paths.value()); } return {}; } Status MergingSnapshotUpdate::ValidateDataFilesExist( const TableMetadata& metadata, std::optional starting_snapshot_id, - const std::unordered_set& file_paths, bool allow_deletes, - std::shared_ptr filter, const std::shared_ptr& parent, - std::shared_ptr io, bool case_sensitive) { + const std::unordered_set& file_paths, bool skip_deletes, + std::shared_ptr conflict_detection_filter, + const std::shared_ptr& parent, std::shared_ptr io, + bool /*case_sensitive*/) { if (parent == nullptr || file_paths.empty()) { return {}; } - ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + std::span matching_operations = + skip_deletes + ? std::span(kValidateDataFilesExistSkipDeleteOperations) + : std::span(kValidateDataFilesExistOperations); ICEBERG_ASSIGN_OR_RAISE( - auto ancestors, - ValidationAncestorsBetween(metadata, parent->snapshot_id, starting_snapshot_id)); - - // Build the full set of matching snapshot IDs first, then scan their manifests. - // The full set must be known before filtering manifests, since a manifest may have - // been written by a different snapshot in the ancestry range. - // Included operations: OVERWRITE and REPLACE always; DELETE when allow_deletes is - // false. - std::unordered_set matching_snapshot_ids; - for (const auto& snap : ancestors) { - auto op = snap->Operation(); - if (op == DataOperation::kOverwrite || op == DataOperation::kReplace) { - matching_snapshot_ids.insert(snap->snapshot_id); - } else if (!allow_deletes && op == DataOperation::kDelete) { - matching_snapshot_ids.insert(snap->snapshot_id); - } - } - - // Build a metrics evaluator for the conflict-detection filter, if provided. - std::unique_ptr evaluator; - if (filter != nullptr) { - ICEBERG_ASSIGN_OR_RAISE( - evaluator, InclusiveMetricsEvaluator::Make(filter, *schema, case_sensitive)); - } - - for (const auto& snapshot : ancestors) { - if (!matching_snapshot_ids.contains(snapshot->snapshot_id)) { - continue; - } - auto cached = SnapshotCache(snapshot.get()); - ICEBERG_ASSIGN_OR_RAISE(auto data_manifests, cached.DataManifests(io)); - - for (const auto& manifest : data_manifests) { - if (!matching_snapshot_ids.contains(manifest.added_snapshot_id)) { - continue; - } - ICEBERG_ASSIGN_OR_RAISE(auto spec, - metadata.PartitionSpecById(manifest.partition_spec_id)); - ICEBERG_ASSIGN_OR_RAISE(auto reader, - ManifestReader::Make(manifest, io, schema, spec)); - ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->Entries()); - - for (const auto& entry : entries) { - if (!entry.snapshot_id.has_value() || - !matching_snapshot_ids.contains(entry.snapshot_id.value())) { - continue; - } - if (entry.status != ManifestStatus::kDeleted) { - continue; - } - if (entry.data_file == nullptr) { - continue; - } - if (!file_paths.contains(entry.data_file->file_path)) { - continue; - } - if (evaluator != nullptr) { - ICEBERG_ASSIGN_OR_RAISE(bool matches, evaluator->Evaluate(*entry.data_file)); - if (!matches) { - continue; - } - } - return InvalidArgument("Cannot commit, missing data files: {} in snapshot {}", - entry.data_file->file_path, snapshot->snapshot_id); - } - } + auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + matching_operations, ManifestContent::kData, io)); + + ICEBERG_ASSIGN_OR_RAISE(auto group, MakeValidationManifestGroup( + metadata, io, std::move(history.manifests))); + group->IgnoreExisting(); + group->FilterManifestEntries([&history, &file_paths](const ManifestEntry& entry) { + return entry.status != ManifestStatus::kAdded && entry.snapshot_id.has_value() && + history.snapshot_ids.contains(entry.snapshot_id.value()) && + entry.data_file != nullptr && file_paths.contains(entry.data_file->file_path); + }); + if (conflict_detection_filter != nullptr) { + group->FilterData(std::move(conflict_detection_filter)); + } + + ICEBERG_ASSIGN_OR_RAISE(auto entries, group->Entries()); + auto deleted_paths = DataFileLocations(entries); + if (deleted_paths.has_value()) { + return ValidationFailed("Cannot commit, missing data files: {}", + deleted_paths.value()); } return {}; } @@ -864,41 +1089,30 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( return {}; } - // Build an index of delete files added since starting_snapshot_id. - // Covers both position and equality deletes; the caller controls whether - // equality deletes are ignored. - ICEBERG_ASSIGN_OR_RAISE(auto deletes, AddedDeleteFiles(metadata, starting_snapshot_id, - nullptr, nullptr, parent, io)); + ICEBERG_ASSIGN_OR_RAISE(auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, + /*data_filter=*/nullptr, + /*partition_set=*/nullptr, parent, io)); - if (deletes->empty()) { - return {}; - } - - // Compute the starting sequence number for the data file check. - int64_t starting_seq = TableMetadata::kInitialSequenceNumber; - if (starting_snapshot_id.has_value()) { - if (auto snap_result = metadata.SnapshotById(starting_snapshot_id.value()); - snap_result.has_value()) { - starting_seq = snap_result.value()->sequence_number; - } - } + ICEBERG_ASSIGN_OR_RAISE(auto starting_sequence_number, + StartingSequenceNumber(metadata, starting_snapshot_id)); for (const auto& data_file : replaced_files) { ICEBERG_ASSIGN_OR_RAISE(auto delete_files, - deletes->ForDataFile(starting_seq, *data_file)); + deletes->ForDataFile(starting_sequence_number, *data_file)); if (ignore_equality_deletes) { // Only fail on position deletes — equality deletes at higher sequence numbers // still apply to the rewritten files and are not a conflict. for (const auto& df : delete_files) { if (df->content == DataFile::Content::kPositionDeletes) { - return InvalidArgument( + return ValidationFailed( "Cannot commit, found new position delete for replaced data file: {}", data_file->file_path); } } } else { if (!delete_files.empty()) { - return InvalidArgument( + return ValidationFailed( "Cannot commit, found new delete for replaced data file: {}", data_file->file_path); } @@ -911,23 +1125,17 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io) { - if (parent == nullptr) { - return {}; - } - - ICEBERG_ASSIGN_OR_RAISE( - auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kAppend, DataOperation::kOverwrite}, - ManifestContent::kData, io)); ICEBERG_ASSIGN_OR_RAISE( - auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, - ManifestStatus::kAdded, nullptr, &partition_set, io, - /*case_sensitive=*/true)); - if (conflict_path.has_value()) { - return InvalidArgument( - "Found conflicting files that can contain rows in validated partitions: {}", - conflict_path.value()); + auto conflict_entries, + MatchingAddedDataFiles(metadata, starting_snapshot_id, + /*data_filter=*/nullptr, &partition_set, parent, io, + /*case_sensitive=*/true)); + auto conflict_paths = DataFileLocations(conflict_entries); + if (conflict_paths.has_value()) { + return ValidationFailed( + "Found conflicting files that can contain records matching validated " + "partitions: {}", + conflict_paths.value()); } return {}; } @@ -935,32 +1143,27 @@ Status MergingSnapshotUpdate::ValidateAddedDataFiles( Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const DataFileSet& replaced_files, - const std::shared_ptr& parent, std::shared_ptr io) { + const std::shared_ptr& parent, std::shared_ptr io, + bool case_sensitive) { if (parent == nullptr || replaced_files.empty() || metadata.format_version < 2) { return {}; } - ICEBERG_ASSIGN_OR_RAISE(auto deletes, - AddedDeleteFiles(metadata, starting_snapshot_id, - std::move(data_filter), nullptr, parent, io)); - if (deletes->empty()) { - return {}; - } + ICEBERG_ASSIGN_OR_RAISE( + auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, std::move(data_filter), + /*partition_set=*/nullptr, parent, io, case_sensitive)); - int64_t starting_seq = TableMetadata::kInitialSequenceNumber; - if (starting_snapshot_id.has_value()) { - if (auto snap_result = metadata.SnapshotById(starting_snapshot_id.value()); - snap_result.has_value()) { - starting_seq = snap_result.value()->sequence_number; - } - } + ICEBERG_ASSIGN_OR_RAISE(auto starting_sequence_number, + StartingSequenceNumber(metadata, starting_snapshot_id)); for (const auto& data_file : replaced_files) { ICEBERG_ASSIGN_OR_RAISE(auto delete_files, - deletes->ForDataFile(starting_seq, *data_file)); - for (const auto& delete_file : delete_files) { - return InvalidArgument("Cannot commit, found new delete for replaced data file: {}", - data_file->file_path); + deletes->ForDataFile(starting_sequence_number, *data_file)); + if (!delete_files.empty()) { + return ValidationFailed( + "Cannot commit, found new delete for replaced data file: {}", + data_file->file_path); } } return {}; @@ -969,15 +1172,19 @@ Status MergingSnapshotUpdate::ValidateNoNewDeletesForDataFiles( Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, - std::shared_ptr io) { - ICEBERG_ASSIGN_OR_RAISE(auto deletes, - AddedDeleteFiles(metadata, starting_snapshot_id, - std::move(data_filter), nullptr, parent, io)); + std::shared_ptr io, bool case_sensitive) { + std::string data_filter_text = + data_filter != nullptr ? data_filter->ToString() : "any expression"; + ICEBERG_ASSIGN_OR_RAISE( + auto deletes, + AddedDeleteFiles(metadata, starting_snapshot_id, std::move(data_filter), + /*partition_set=*/nullptr, parent, io, case_sensitive)); auto referenced_delete_files = deletes->ReferencedDeleteFiles(); - - for (const auto& delete_file : referenced_delete_files) { - return InvalidArgument("Found new conflicting delete files: {}", - delete_file->file_path); + auto delete_paths = DeleteFileLocations(referenced_delete_files); + if (delete_paths.has_value()) { + return ValidationFailed( + "Found new conflicting delete files that can apply to records matching {}: {}", + data_filter_text, delete_paths.value()); } return {}; } @@ -988,13 +1195,16 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( std::shared_ptr io) { ICEBERG_ASSIGN_OR_RAISE( auto deletes, - AddedDeleteFiles(metadata, starting_snapshot_id, nullptr, + AddedDeleteFiles(metadata, starting_snapshot_id, + /*data_filter=*/nullptr, std::make_shared(partition_set), parent, io)); auto referenced_delete_files = deletes->ReferencedDeleteFiles(); - for (const auto& delete_file : referenced_delete_files) { - return InvalidArgument( - "Found new conflicting delete files in validated partitions: {}", - delete_file->file_path); + auto delete_paths = DeleteFileLocations(referenced_delete_files); + if (delete_paths.has_value()) { + return ValidationFailed( + "Found new conflicting delete files that can apply to records matching " + "validated partitions: {}", + delete_paths.value()); } return {}; } @@ -1002,26 +1212,17 @@ Status MergingSnapshotUpdate::ValidateNoNewDeleteFiles( Status MergingSnapshotUpdate::ValidateDeletedDataFiles( const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, - std::shared_ptr io) { - if (parent == nullptr) { - return {}; - } - - ICEBERG_ASSIGN_OR_RAISE( - auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kOverwrite, DataOperation::kReplace, - DataOperation::kDelete}, - ManifestContent::kData, io)); + std::shared_ptr io, bool case_sensitive) { ICEBERG_ASSIGN_OR_RAISE( - auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, - ManifestStatus::kDeleted, data_filter, nullptr, io, - /*case_sensitive=*/true)); - if (conflict_path.has_value()) { - return InvalidArgument( - "Found conflicting deleted files that can contain rows matching {}: {}", + auto conflict_entries, + MatchingDeletedDataFiles(metadata, starting_snapshot_id, data_filter, + /*partition_set=*/nullptr, parent, io, case_sensitive)); + auto conflict_paths = DataFileLocations(conflict_entries); + if (conflict_paths.has_value()) { + return ValidationFailed( + "Found conflicting deleted files that can contain records matching {}: {}", data_filter != nullptr ? data_filter->ToString() : "any expression", - conflict_path.value()); + conflict_paths.value()); } return {}; } @@ -1030,23 +1231,17 @@ Status MergingSnapshotUpdate::ValidateDeletedDataFiles( const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, const std::shared_ptr& parent, std::shared_ptr io) { - if (parent == nullptr) { - return {}; - } - ICEBERG_ASSIGN_OR_RAISE( - auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kOverwrite, DataOperation::kReplace, - DataOperation::kDelete}, - ManifestContent::kData, io)); - ICEBERG_ASSIGN_OR_RAISE( - auto conflict_path, - FindMatchingDataFile(metadata, history.manifests, history.snapshot_ids, - ManifestStatus::kDeleted, nullptr, &partition_set, io, - /*case_sensitive=*/true)); - if (conflict_path.has_value()) { - return InvalidArgument("Found conflicting deleted files in validated partitions: {}", - conflict_path.value()); + auto conflict_entries, + MatchingDeletedDataFiles(metadata, starting_snapshot_id, + /*data_filter=*/nullptr, &partition_set, parent, io, + /*case_sensitive=*/true)); + auto conflict_paths = DataFileLocations(conflict_entries); + if (conflict_paths.has_value()) { + return ValidationFailed( + "Found conflicting deleted files that can apply to records matching " + "validated partitions: {}", + conflict_paths.value()); } return {}; } @@ -1056,91 +1251,54 @@ Result> MergingSnapshotUpdate::AddedDeleteFiles std::shared_ptr data_filter, std::shared_ptr partition_set, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive) { - ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); - if (parent == nullptr || metadata.format_version < 2) { - TableMetadataCache metadata_cache(&metadata); - ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, metadata_cache.GetPartitionSpecsById()); - std::unordered_map> specs_by_id( - specs_ref.get().begin(), specs_ref.get().end()); - ICEBERG_ASSIGN_OR_RAISE(auto builder, DeleteFileIndex::BuilderFor( - io, schema, std::move(specs_by_id), {})); + ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); + ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, PartitionSpecsByIdMap(metadata)); + ICEBERG_ASSIGN_OR_RAISE( + auto builder, + DeleteFileIndex::BuilderFor(io, std::move(schema), std::move(specs_by_id), + /*delete_manifests=*/{})); return builder.Build(); } ICEBERG_ASSIGN_OR_RAISE( auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kOverwrite, DataOperation::kDelete}, + kValidateAddedDeleteFilesOperations, ManifestContent::kDeletes, io)); - // Compute the starting sequence number from the starting snapshot. - int64_t starting_seq = TableMetadata::kInitialSequenceNumber; - if (starting_snapshot_id.has_value()) { - if (auto snap_result = metadata.SnapshotById(starting_snapshot_id.value()); - snap_result.has_value()) { - starting_seq = snap_result.value()->sequence_number; - } - } - - TableMetadataCache metadata_cache(&metadata); - ICEBERG_ASSIGN_OR_RAISE(auto specs_ref, metadata_cache.GetPartitionSpecsById()); - std::unordered_map> specs_by_id( - specs_ref.get().begin(), specs_ref.get().end()); - - ICEBERG_ASSIGN_OR_RAISE(auto builder, - DeleteFileIndex::BuilderFor(io, schema, std::move(specs_by_id), - std::move(history.manifests))); - builder.AfterSequenceNumber(starting_seq); - builder.CaseSensitive(case_sensitive); - if (data_filter != nullptr) { - builder.DataFilter(std::move(data_filter)); - } - if (partition_set != nullptr) { - builder.FilterPartitions(std::move(partition_set)); - } - return builder.Build(); + ICEBERG_ASSIGN_OR_RAISE(auto starting_sequence_number, + StartingSequenceNumber(metadata, starting_snapshot_id)); + return BuildDeleteFileIndex(metadata, io, std::move(history.manifests), + starting_sequence_number, std::move(data_filter), + std::move(partition_set), case_sensitive); } Status MergingSnapshotUpdate::ValidateAddedDVs( const TableMetadata& metadata, std::optional starting_snapshot_id, - std::shared_ptr conflict_filter, + std::shared_ptr conflict_detection_filter, const std::unordered_set& referenced_data_files, - const std::shared_ptr& parent, std::shared_ptr io) { - if (parent == nullptr || referenced_data_files.empty() || metadata.format_version < 3) { + const std::shared_ptr& parent, std::shared_ptr io, + bool case_sensitive) { + if (parent == nullptr || referenced_data_files.empty()) { return {}; } ICEBERG_ASSIGN_OR_RAISE( - auto history, ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, - {DataOperation::kOverwrite, DataOperation::kDelete, - DataOperation::kReplace}, - ManifestContent::kDeletes, io)); + auto history, + ValidationHistory(metadata, parent->snapshot_id, starting_snapshot_id, + kValidateAddedDVsOperations, ManifestContent::kDeletes, io)); ICEBERG_ASSIGN_OR_RAISE(auto schema, metadata.Schema()); - for (const auto& manifest : history.manifests) { - ICEBERG_ASSIGN_OR_RAISE(auto spec, - metadata.PartitionSpecById(manifest.partition_spec_id)); - ICEBERG_ASSIGN_OR_RAISE(auto reader, - ManifestReader::Make(manifest, io, schema, spec)); - if (conflict_filter != nullptr) { - reader->FilterRows(conflict_filter); - } - ICEBERG_ASSIGN_OR_RAISE(auto entries, reader->LiveEntries()); - - for (const auto& entry : entries) { - if (!entry.snapshot_id.has_value() || - !history.snapshot_ids.contains(entry.snapshot_id.value())) { - continue; - } - if (entry.data_file == nullptr || !ContentFileUtil::IsDV(*entry.data_file) || - !entry.data_file->referenced_data_file.has_value()) { - continue; - } - if (referenced_data_files.contains(*entry.data_file->referenced_data_file)) { - return InvalidArgument("Cannot commit, found new deletion vector: {}", - ContentFileUtil::DVDesc(*entry.data_file)); - } + ICEBERG_ASSIGN_OR_RAISE(auto matching_manifests, + FilterManifestsByPartition(metadata, conflict_detection_filter, + history.manifests, case_sensitive)); + for (const auto& manifest : matching_manifests) { + if (!manifest.has_added_files()) { + continue; } + ICEBERG_RETURN_UNEXPECTED(ValidateAddedDVsInManifest( + metadata, manifest, conflict_detection_filter, history.snapshot_ids, + referenced_data_files, io, schema, case_sensitive)); } return {}; } @@ -1149,19 +1307,19 @@ Status MergingSnapshotUpdate::ValidateAddedDVs( const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr conflict_filter, const std::shared_ptr& parent, std::shared_ptr io) const { + if (parent == nullptr) { + return {}; + } + std::unordered_set referenced_data_files; - for (const auto& pending_file : new_delete_files_) { - if (pending_file.file == nullptr || !ContentFileUtil::IsDV(*pending_file.file) || - !pending_file.file->referenced_data_file.has_value()) { - continue; - } - referenced_data_files.insert(*pending_file.file->referenced_data_file); + for (const auto& entry : dvs_by_referenced_file_.entries()) { + referenced_data_files.insert(entry.referenced_file); } if (referenced_data_files.empty()) { return {}; } return ValidateAddedDVs(metadata, starting_snapshot_id, std::move(conflict_filter), - referenced_data_files, parent, std::move(io)); + referenced_data_files, parent, std::move(io), case_sensitive_); } } // namespace iceberg diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h index 3658269f9..5d4e128e9 100644 --- a/src/iceberg/update/merging_snapshot_update.h +++ b/src/iceberg/update/merging_snapshot_update.h @@ -56,6 +56,11 @@ namespace iceberg { /// 6. Merge data manifests (via data_merge_manager_) /// 7. Merge delete manifests (via delete_merge_manager_) /// +/// TODO(Guotao): Java MergingSnapshotProducer overrides updateEvent() to return a +/// CreateSnapshotEvent(tableName, operation, snapshotId, sequenceNumber, summary) +/// for commit listeners. The C++ update framework does not yet have an event +/// notification mechanism, so this is intentionally not implemented here. Add it +/// once an equivalent CreateSnapshotEvent / listener facility exists. class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { public: ~MergingSnapshotUpdate() override = default; @@ -65,13 +70,10 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { const TableMetadata& metadata_to_update, const std::shared_ptr& snapshot) override; - void CleanUncommitted(const std::unordered_set& committed) override; + Status CleanUncommitted(const std::unordered_set& committed) override; std::unordered_map Summary() override; - /// \brief Set a custom property in the snapshot summary. - void Set(const std::string& property, const std::string& value); - protected: /// \brief Constructor; reads merge configuration from table properties. explicit MergingSnapshotUpdate(std::string table_name, @@ -110,7 +112,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Register a data file path to be deleted from the table. /// /// \note Only applies to data files. To remove delete files, use DeleteDeleteFile(). - void DeleteByPath(std::string_view path); + Status DeleteByPath(std::string_view path); /// \brief Register an expression to delete matching rows. /// @@ -122,7 +124,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// /// Both data and delete filter managers receive the partition drop, since dropping /// data in a partition also drops all delete files in that partition. - void DropPartition(int32_t spec_id, PartitionValues partition); + Status DropPartition(int32_t spec_id, PartitionValues partition); /// \brief Fail if any registered delete path is not found in any manifest. void FailMissingDeletePaths(); @@ -175,7 +177,6 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, added a data file in any partition of the given partition /// set. - /// static Status ValidateAddedDataFiles(const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, @@ -184,10 +185,10 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, removed a file whose path is in file_paths (and - /// allow_deletes is false). + /// skip_deletes is false). static Status ValidateDataFilesExist( const TableMetadata& metadata, std::optional starting_snapshot_id, - const std::unordered_set& file_paths, bool allow_deletes, + const std::unordered_set& file_paths, bool skip_deletes, std::shared_ptr filter, const std::shared_ptr& parent, std::shared_ptr io, bool case_sensitive = true); @@ -205,9 +206,10 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { const DataFileSet& replaced_files, const std::shared_ptr& parent, std::shared_ptr io) const { + const bool ignore_equality_deletes = new_data_files_data_seq_number_.has_value(); return ValidateNoNewDeletesForDataFiles(metadata, starting_snapshot_id, replaced_files, parent, io, - new_data_files_data_seq_number_.has_value()); + ignore_equality_deletes); } /// \brief Return an error if any snapshot after starting_snapshot_id, or from @@ -225,11 +227,11 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, added a delete file matching the data filter that covers a /// file in replaced_files. - /// static Status ValidateNoNewDeletesForDataFiles( const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const DataFileSet& replaced_files, - const std::shared_ptr& parent, std::shared_ptr io); + const std::shared_ptr& parent, std::shared_ptr io, + bool case_sensitive = true); /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, added a delete file matching the given row filter. @@ -238,12 +240,12 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, - std::shared_ptr io); + std::shared_ptr io, + bool case_sensitive = true); /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, added a delete file matching any partition in the given /// partition set. - /// static Status ValidateNoNewDeleteFiles(const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, @@ -252,17 +254,16 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, deleted a data file matching the given row filter. - /// static Status ValidateDeletedDataFiles(const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr data_filter, const std::shared_ptr& parent, - std::shared_ptr io); + std::shared_ptr io, + bool case_sensitive = true); /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, deleted a data file in any partition of the given partition /// set. - /// static Status ValidateDeletedDataFiles(const TableMetadata& metadata, std::optional starting_snapshot_id, const PartitionSet& partition_set, @@ -280,12 +281,12 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Return an error if any snapshot after starting_snapshot_id, or from /// the beginning if unset, added a deletion vector that conflicts with DVs being /// written. - /// static Status ValidateAddedDVs( const TableMetadata& metadata, std::optional starting_snapshot_id, std::shared_ptr conflict_filter, const std::unordered_set& referenced_data_files, - const std::shared_ptr& parent, std::shared_ptr io); + const std::shared_ptr& parent, std::shared_ptr io, + bool case_sensitive = true); private: struct PendingDeleteFile { @@ -293,9 +294,27 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { std::optional data_sequence_number; }; - /// \brief Create a ManifestWriterFactory that records every path it creates in - /// all_written_manifests_. - ManifestWriterFactory MakeTrackedWriterFactory(const std::shared_ptr& schema); + /// \brief Ordered map from referenced data file path to pending DVs. + /// + /// Mirrors Java's LinkedHashMap-backed dvsByReferencedFile: lookup is by + /// referenced data file, and iteration preserves the first-seen key order. + struct PendingDeleteFilesByReferencedFile { + struct Entry { + std::string referenced_file; + std::vector files; + }; + + void Add(std::string referenced_file, PendingDeleteFile file); + bool empty() const { return entries_.empty(); } + size_t size() const { return entries_.size(); } + const std::vector& entries() const { return entries_; } + + private: + std::vector entries_; + std::unordered_map index_by_referenced_file_; + }; + + ManifestWriterFactory MakeWriterFactory(const std::shared_ptr& schema); /// \brief Copy a manifest with the current snapshot ID, for use when snapshot /// ID inheritance is not possible. @@ -310,7 +329,11 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { const std::shared_ptr& parent, std::shared_ptr io) const; - Result> NormalizeNewDeleteFiles() const; + Status ManagersReady() const; + + void SetSummaryProperty(const std::string& property, const std::string& value) override; + + Result> MergeDVs() const; /// \brief Write new data manifests for staged data files; caches the result. Result> WriteNewDataManifests(); @@ -318,6 +341,11 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { /// \brief Write new delete manifests for staged delete files; caches the result. Result> WriteNewDeleteManifests(); + Status CleanUncommittedAppends(const std::unordered_set& committed); + + Status DeleteUncommitted(std::vector& manifests, + const std::unordered_set& committed, bool clear); + // Used for commit event notifications and diagnostic log messages. std::string table_name_; std::shared_ptr delete_expression_; @@ -330,13 +358,14 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { SnapshotSummaryBuilder appended_manifests_summary_; std::unordered_map custom_summary_properties_; - ManifestFilterManager data_filter_manager_; - ManifestFilterManager delete_filter_manager_; - ManifestMergeManager data_merge_manager_; - ManifestMergeManager delete_merge_manager_; + std::unique_ptr data_filter_manager_; + std::unique_ptr delete_filter_manager_; + std::unique_ptr data_merge_manager_; + std::unique_ptr delete_merge_manager_; std::unordered_map new_data_files_by_spec_; - std::vector new_delete_files_; + std::vector v2_deletes_; + PendingDeleteFilesByReferencedFile dvs_by_referenced_file_; std::optional new_data_files_data_seq_number_; // Manifests passed via AddManifest(): inherit path (no copy needed) and @@ -349,12 +378,8 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { bool has_new_data_files_ = false; bool has_new_delete_files_ = false; - std::optional> cached_new_data_manifests_; - std::optional> cached_new_delete_manifests_; - - /// Tracks every manifest path created via MakeTrackedWriterFactory, plus the - /// paths in cached_new_*_manifests_. Used by CleanUncommitted(). - std::unordered_set all_written_manifests_; + std::vector cached_new_data_manifests_; + std::vector cached_new_delete_manifests_; }; } // namespace iceberg diff --git a/src/iceberg/update/meson.build b/src/iceberg/update/meson.build index 6acb007a1..6405f603f 100644 --- a/src/iceberg/update/meson.build +++ b/src/iceberg/update/meson.build @@ -19,6 +19,7 @@ install_headers( [ 'expire_snapshots.h', 'fast_append.h', + 'merging_snapshot_update.h', 'pending_update.h', 'set_snapshot.h', 'snapshot_manager.h', diff --git a/src/iceberg/update/snapshot_update.cc b/src/iceberg/update/snapshot_update.cc index b3ce0ff30..bb5376fa8 100644 --- a/src/iceberg/update/snapshot_update.cc +++ b/src/iceberg/update/snapshot_update.cc @@ -41,28 +41,34 @@ namespace iceberg { namespace { -// Java skips updating totals if parsing fails; C++ treats parse failures as errors. Status UpdateTotal(std::unordered_map& summary, const std::unordered_map& previous_summary, const std::string& total_property, const std::string& added_property, const std::string& deleted_property) { auto total_it = previous_summary.find(total_property); if (total_it != previous_summary.end()) { - ICEBERG_ASSIGN_OR_RAISE(auto new_total, - StringUtils::ParseNumber(total_it->second)); + auto parsed_total = StringUtils::ParseNumber(total_it->second); + if (!parsed_total.has_value()) { + return {}; + } + int64_t new_total = parsed_total.value(); auto added_it = summary.find(added_property); if (new_total >= 0 && added_it != summary.end()) { - ICEBERG_ASSIGN_OR_RAISE(auto added_value, - StringUtils::ParseNumber(added_it->second)); - new_total += added_value; + auto parsed_added = StringUtils::ParseNumber(added_it->second); + if (!parsed_added.has_value()) { + return {}; + } + new_total += parsed_added.value(); } auto deleted_it = summary.find(deleted_property); if (new_total >= 0 && deleted_it != summary.end()) { - ICEBERG_ASSIGN_OR_RAISE(auto deleted_value, - StringUtils::ParseNumber(deleted_it->second)); - new_total -= deleted_value; + auto parsed_deleted = StringUtils::ParseNumber(deleted_it->second); + if (!parsed_deleted.has_value()) { + return {}; + } + new_total -= parsed_deleted.value(); } if (new_total >= 0) { @@ -163,6 +169,11 @@ SnapshotUpdate::SnapshotUpdate(std::shared_ptr ctx) target_manifest_size_bytes_( base().properties.Get(TableProperties::kManifestTargetSizeBytes)) {} +void SnapshotUpdate::SetSummaryProperty(const std::string& property, + const std::string& value) { + summary_.Set(property, value); +} + // TODO(xxx): write manifests in parallel Result> SnapshotUpdate::WriteDataManifests( std::span> files, @@ -178,8 +189,7 @@ Result> SnapshotUpdate::WriteDataManifests( snapshot_id = SnapshotId()]() -> Result> { return ManifestWriter::MakeWriter( base().format_version, snapshot_id, ManifestPath(), ctx_->table->io(), - std::move(spec), std::move(schema), ManifestContent::kData, - /*first_row_id=*/base().next_row_id); + std::move(spec), std::move(schema), ManifestContent::kData); }, target_manifest_size_bytes_); @@ -192,20 +202,7 @@ Result> SnapshotUpdate::WriteDataManifests( // TODO(xxx): write manifests in parallel Result> SnapshotUpdate::WriteDeleteManifests( - std::span> files, - const std::shared_ptr& spec) { - std::vector delete_entries; - delete_entries.reserve(files.size()); - for (const auto& file : files) { - delete_entries.push_back( - DeleteManifestEntry{.file = file, .data_sequence_number = std::nullopt}); - } - return WriteDeleteManifests(delete_entries, spec); -} - -// TODO(xxx): write manifests in parallel -Result> SnapshotUpdate::WriteDeleteManifests( - std::span files, + std::span files, const std::shared_ptr& spec) { if (files.empty()) { return std::vector{}; @@ -244,7 +241,7 @@ Result SnapshotUpdate::Apply() { std::ignore = DeleteFile(manifest_list); } manifest_lists_.clear(); - CleanUncommitted(std::unordered_set{}); + ICEBERG_RETURN_UNEXPECTED(CleanUncommitted(std::unordered_set{})); staged_snapshot_ = nullptr; summary_.Clear(); @@ -257,9 +254,7 @@ Result SnapshotUpdate::Apply() { std::optional parent_snapshot_id = parent_snapshot ? std::make_optional(parent_snapshot->snapshot_id) : std::nullopt; - if (parent_snapshot) { - ICEBERG_RETURN_UNEXPECTED(Validate(base(), parent_snapshot)); - } + ICEBERG_RETURN_UNEXPECTED(Validate(base(), parent_snapshot)); ICEBERG_ASSIGN_OR_RAISE(auto manifests, Apply(base(), parent_snapshot)); for (auto& manifest : manifests) { @@ -326,7 +321,7 @@ Status SnapshotUpdate::Finalize(Result commit_result) { if (commit_result.error().kind == ErrorKind::kCommitStateUnknown) { return {}; } - CleanAll(); + std::ignore = CleanAll(); return {}; } @@ -334,11 +329,14 @@ Status SnapshotUpdate::Finalize(Result commit_result) { ICEBERG_CHECK(staged_snapshot_ != nullptr, "Staged snapshot is null during finalize after commit"); auto cached_snapshot = SnapshotCache(staged_snapshot_.get()); - ICEBERG_ASSIGN_OR_RAISE(auto manifests, cached_snapshot.Manifests(ctx_->table->io())); - CleanUncommitted(manifests | std::views::transform([](const auto& manifest) { - return manifest.manifest_path; - }) | - std::ranges::to>()); + if (auto manifests = cached_snapshot.Manifests(ctx_->table->io()); + manifests.has_value()) { + std::ignore = CleanUncommitted(manifests.value() | + std::views::transform([](const auto& manifest) { + return manifest.manifest_path; + }) | + std::ranges::to>()); + } } // Also clean up unused manifest lists created by multiple attempts @@ -401,12 +399,13 @@ Result> SnapshotUpdate::ComputeSumm return summary; } -void SnapshotUpdate::CleanAll() { +Status SnapshotUpdate::CleanAll() { for (const auto& manifest_list : manifest_lists_) { std::ignore = DeleteFile(manifest_list); } manifest_lists_.clear(); - CleanUncommitted(std::unordered_set{}); + std::ignore = CleanUncommitted(std::unordered_set{}); + return {}; } Status SnapshotUpdate::DeleteFile(const std::string& path) { diff --git a/src/iceberg/update/snapshot_update.h b/src/iceberg/update/snapshot_update.h index 42df70d61..03a74e788 100644 --- a/src/iceberg/update/snapshot_update.h +++ b/src/iceberg/update/snapshot_update.h @@ -105,7 +105,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \param value The property value /// \return Reference to this for method chaining auto& Set(this auto& self, const std::string& property, const std::string& value) { - self.summary_.Set(property, value); + static_cast(self).SetSummaryProperty(property, value); return self; } @@ -122,7 +122,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { Status Finalize(Result commit_result) override; protected: - struct DeleteManifestEntry { + struct ContentFileWithSequenceNumber { std::shared_ptr file; std::optional data_sequence_number; }; @@ -140,17 +140,8 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { const std::shared_ptr& spec, std::optional data_sequence_number = std::nullopt); - /// \brief Write delete manifests for the given delete files - /// - /// \param files Delete files to write - /// \param spec The partition spec to use - /// \return A vector of manifest files - Result> WriteDeleteManifests( - std::span> files, - const std::shared_ptr& spec); - Result> WriteDeleteManifests( - std::span files, + std::span files, const std::shared_ptr& spec); const std::string& target_branch() const { return target_branch_; } @@ -170,7 +161,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// actually committed. /// /// \param committed A set of manifest paths that were actually committed - virtual void CleanUncommitted(const std::unordered_set& committed) = 0; + virtual Status CleanUncommitted(const std::unordered_set& committed) = 0; /// \brief A string that describes the action that produced the new snapshot. /// @@ -202,6 +193,12 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { /// \return A map of summary properties virtual std::unordered_map Summary() = 0; + /// \brief Set a summary property. + /// + /// Implementations may override this to retain custom properties across + /// retry-safe summary rebuilds. + virtual void SetSummaryProperty(const std::string& property, const std::string& value); + /// \brief Check if cleanup should happen after commit /// /// \return True if cleanup should happen after commit @@ -226,7 +223,7 @@ class ICEBERG_EXPORT SnapshotUpdate : public PendingUpdate { const TableMetadata& previous); /// \brief Clean up all uncommitted files - void CleanAll(); + Status CleanAll(); protected: SnapshotSummaryBuilder summary_; diff --git a/src/iceberg/util/data_file_set.h b/src/iceberg/util/data_file_set.h index 741b34e56..93abdfff7 100644 --- a/src/iceberg/util/data_file_set.h +++ b/src/iceberg/util/data_file_set.h @@ -20,12 +20,15 @@ #pragma once /// \file iceberg/util/data_file_set.h -/// A set of DataFile pointers with insertion order preserved and deduplicated by file -/// path. +/// Sets of DataFile pointers with insertion order preserved and Iceberg-compatible +/// file identity deduplication. +#include #include #include +#include #include +#include #include #include #include @@ -46,6 +49,16 @@ class ICEBERG_EXPORT DataFileSet { using difference_type = typename std::vector::difference_type; DataFileSet() = default; + DataFileSet(const DataFileSet& other) : elements_(other.elements_) { RebuildIndex(); } + DataFileSet& operator=(const DataFileSet& other) { + if (this != &other) { + elements_ = other.elements_; + RebuildIndex(); + } + return *this; + } + DataFileSet(DataFileSet&&) noexcept = default; + DataFileSet& operator=(DataFileSet&&) noexcept = default; /// \brief Insert a data file into the set. /// \param file The data file to insert @@ -58,6 +71,16 @@ class ICEBERG_EXPORT DataFileSet { return InsertImpl(std::move(file)); } + /// \brief Returns whether an equivalent data file exists in the set. + bool contains(const DataFile& file) const { + return index_by_path_.contains(file.file_path); + } + + /// \brief Returns whether an equivalent data file exists in the set. + bool contains(const value_type& file) const { + return file != nullptr && contains(*file); + } + /// \brief Get the number of elements in the set. size_t size() const { return elements_.size(); } @@ -100,9 +123,146 @@ class ICEBERG_EXPORT DataFileSet { return {std::prev(elements_.end()), true}; } + void RebuildIndex() { + index_by_path_.clear(); + for (size_t i = 0; i < elements_.size(); ++i) { + if (elements_[i] != nullptr) { + index_by_path_.try_emplace(elements_[i]->file_path, i); + } + } + } + // Vector to preserve insertion order std::vector elements_; std::unordered_map index_by_path_; }; +/// \brief A set of delete-file pointers deduplicated by delete-file identity. +/// +/// Delete files, especially deletion vectors, are identified by location plus +/// content offset and content size. This mirrors Java's DeleteFileSet behavior. +class ICEBERG_EXPORT DeleteFileSet { + public: + using value_type = std::shared_ptr; + using iterator = typename std::vector::iterator; + using const_iterator = typename std::vector::const_iterator; + using difference_type = typename std::vector::difference_type; + + DeleteFileSet() = default; + DeleteFileSet(const DeleteFileSet& other) : elements_(other.elements_) { + RebuildIndex(); + } + DeleteFileSet& operator=(const DeleteFileSet& other) { + if (this != &other) { + elements_ = other.elements_; + RebuildIndex(); + } + return *this; + } + DeleteFileSet(DeleteFileSet&&) noexcept = default; + DeleteFileSet& operator=(DeleteFileSet&&) noexcept = default; + + /// \brief Insert a delete file into the set. + /// \param file The delete file to insert + /// \return A pair with an iterator to the inserted element (or the existing one) and + /// a bool indicating whether insertion took place + std::pair insert(const value_type& file) { return InsertImpl(file); } + + /// \brief Insert a delete file into the set (move version). + std::pair insert(value_type&& file) { + return InsertImpl(std::move(file)); + } + + /// \brief Returns whether an equivalent delete file exists in the set. + bool contains(const DataFile& file) const { + return index_by_file_.contains(DeleteFileKey(file)); + } + + /// \brief Returns whether an equivalent delete file exists in the set. + bool contains(const value_type& file) const { + return file != nullptr && contains(*file); + } + + /// \brief Get the number of elements in the set. + size_t size() const { return elements_.size(); } + + /// \brief Check if the set is empty. + bool empty() const { return elements_.empty(); } + + /// \brief Clear all elements from the set. + void clear() { + elements_.clear(); + index_by_file_.clear(); + } + + /// \brief Get iterator to the beginning. + iterator begin() { return elements_.begin(); } + const_iterator begin() const { return elements_.begin(); } + const_iterator cbegin() const { return elements_.cbegin(); } + + /// \brief Get iterator to the end. + iterator end() { return elements_.end(); } + const_iterator end() const { return elements_.end(); } + const_iterator cend() const { return elements_.cend(); } + + /// \brief Get a non-owning view of the delete files in insertion order. + std::span as_span() const { return elements_; } + + private: + struct DeleteFileKey { + explicit DeleteFileKey(const DataFile& file) + : path(file.file_path), + content_offset(file.content_offset), + content_size_in_bytes(file.content_size_in_bytes) {} + + std::string path; + std::optional content_offset; + std::optional content_size_in_bytes; + + bool operator==(const DeleteFileKey& other) const = default; + }; + + struct DeleteFileKeyHash { + size_t operator()(const DeleteFileKey& key) const { + size_t hash = std::hash{}(key.path); + auto combine = [&hash](const auto& value) { + size_t value_hash = value.has_value() ? std::hash{}(*value) : 0; + hash ^= value_hash + 0x9e3779b9 + (hash << 6) + (hash >> 2); + }; + combine(key.content_offset); + combine(key.content_size_in_bytes); + return hash; + } + }; + + std::pair InsertImpl(value_type file) { + if (!file) { + return {elements_.end(), false}; + } + + auto [index_iter, inserted] = + index_by_file_.try_emplace(DeleteFileKey(*file), elements_.size()); + if (!inserted) { + auto pos = static_cast(index_iter->second); + return {elements_.begin() + pos, false}; + } + + elements_.push_back(std::move(file)); + return {std::prev(elements_.end()), true}; + } + + void RebuildIndex() { + index_by_file_.clear(); + for (size_t i = 0; i < elements_.size(); ++i) { + if (elements_[i] != nullptr) { + index_by_file_.try_emplace(DeleteFileKey(*elements_[i]), i); + } + } + } + + // Vector to preserve insertion order. + std::vector elements_; + std::unordered_map index_by_file_; +}; + } // namespace iceberg