diff --git a/pyiceberg/table/update/snapshot.py b/pyiceberg/table/update/snapshot.py index 8feebb9901..4bae6e3a28 100644 --- a/pyiceberg/table/update/snapshot.py +++ b/pyiceberg/table/update/snapshot.py @@ -784,6 +784,10 @@ def _existing_manifests(self) -> list[ManifestFile]: manifest_evaluators: dict[int, Callable[[ManifestFile], bool]] = KeyDefaultDict(self._build_manifest_evaluator) if snapshot := self._transaction.table_metadata.snapshot_by_name(name=self._target_branch): for manifest_file in snapshot.manifests(io=self._io): + # Manifest only records files deleted by an earlier snapshot, so it is dropped + if not manifest_file.has_added_files() and not manifest_file.has_existing_files(): + continue + # Manifest does not contain rows that match the files to delete partitions if not manifest_evaluators[manifest_file.partition_spec_id](manifest_file): existing_files.append(manifest_file) diff --git a/tests/table/test_snapshots.py b/tests/table/test_snapshots.py index 39aeb7b349..af8cb0b296 100644 --- a/tests/table/test_snapshots.py +++ b/tests/table/test_snapshots.py @@ -783,6 +783,29 @@ def test_overwrite_replaces_existing_file(overwrite_table: Table, arrow_table_si assert _total_data_file_count(overwrite_table) == 1 +def test_overwrite_drops_manifests_without_live_files(overwrite_table: Table, arrow_table_simple: pa.Table) -> None: + manifest_counts = [] + snapshot_ids = [] + for index in range(3): + current_file = next(iter(overwrite_table.scan().plan_files())).file + replacement = _write_data_file(overwrite_table, arrow_table_simple.slice(index, 1)) + + with overwrite_table.transaction() as tx: + with tx.update_snapshot().overwrite() as overwrite: + overwrite.delete_data_file(current_file) + overwrite.append_data_file(replacement) + + snapshot = overwrite_table.current_snapshot() + assert snapshot is not None + manifest_counts.append(len(snapshot.manifests(overwrite_table.io))) + snapshot_ids.append(snapshot.snapshot_id) + + # Each overwrite lists its added file and its own deletions, never an earlier commit's deletions + assert manifest_counts == [2, 2, 2] + assert overwrite_table.scan(snapshot_id=snapshot_ids[0]).to_arrow()["foo"].to_pylist() == ["a"] + assert overwrite_table.scan().to_arrow()["foo"].to_pylist() == ["c"] + + def test_overwrite_rejects_explicit_delete_missing_from_base_snapshot(catalog: Catalog, overwrite_table: Table) -> None: stale_file = next(iter(overwrite_table.scan().plan_files())).file stale_rows = overwrite_table.scan().to_arrow()