Drop deleted-only manifests when overwriting files

pyiceberg/table/update/snapshot.py

@@ -784,6 +784,10 @@ def _existing_manifests(self) -> list[ManifestFile]:
         manifest_evaluators: dict[int, Callable[[ManifestFile], bool]] = KeyDefaultDict(self._build_manifest_evaluator)
         if snapshot := self._transaction.table_metadata.snapshot_by_name(name=self._target_branch):
             for manifest_file in snapshot.manifests(io=self._io):
+                # Manifest only records files deleted by an earlier snapshot, so it is dropped
+                if not manifest_file.has_added_files() and not manifest_file.has_existing_files():
+                    continue
+
                 # Manifest does not contain rows that match the files to delete partitions
                 if not manifest_evaluators[manifest_file.partition_spec_id](manifest_file):
                     existing_files.append(manifest_file)

tests/table/test_snapshots.py

@@ -783,6 +783,29 @@ def test_overwrite_replaces_existing_file(overwrite_table: Table, arrow_table_si
     assert _total_data_file_count(overwrite_table) == 1
 
 
+def test_overwrite_drops_manifests_without_live_files(overwrite_table: Table, arrow_table_simple: pa.Table) -> None:
+    manifest_counts = []
+    snapshot_ids = []
+    for index in range(3):
+        current_file = next(iter(overwrite_table.scan().plan_files())).file
+        replacement = _write_data_file(overwrite_table, arrow_table_simple.slice(index, 1))
+
+        with overwrite_table.transaction() as tx:
+            with tx.update_snapshot().overwrite() as overwrite:
+                overwrite.delete_data_file(current_file)
+                overwrite.append_data_file(replacement)
+
+        snapshot = overwrite_table.current_snapshot()
+        assert snapshot is not None
+        manifest_counts.append(len(snapshot.manifests(overwrite_table.io)))
+        snapshot_ids.append(snapshot.snapshot_id)
+
+    # Each overwrite lists its added file and its own deletions, never an earlier commit's deletions
+    assert manifest_counts == [2, 2, 2]
+    assert overwrite_table.scan(snapshot_id=snapshot_ids[0]).to_arrow()["foo"].to_pylist() == ["a"]
+    assert overwrite_table.scan().to_arrow()["foo"].to_pylist() == ["c"]
+
+
 def test_overwrite_rejects_explicit_delete_missing_from_base_snapshot(catalog: Catalog, overwrite_table: Table) -> None:
     stale_file = next(iter(overwrite_table.scan().plan_files())).file
     stale_rows = overwrite_table.scan().to_arrow()