From 2cc3c88ff2cdc984703aeb8e1be44127b72bb7db Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:11:57 +0100 Subject: [PATCH 1/3] chore(nifi): Update the backport of NIFI-15958 with the upstream version --- CHANGELOG.md | 3 +- ...-progress-logging-for-repository-sca.patch | 100 ++++++++++++ ...eriodic-progress-while-waiting-for-c.patch | 152 ------------------ ...ed-custom-Iceberg-REST-Client-to-fix.patch | 2 +- ...-HashiCorp-Vault-Parameter-Provider-.patch | 2 +- ...10-Remove-Develocity-Maven-extension.patch | 6 +- ...-progress-logging-for-repository-sca.patch | 100 ++++++++++++ ...eriodic-progress-while-waiting-for-c.patch | 152 ------------------ ...11-Remove-Develocity-Maven-extension.patch | 6 +- ...-progress-logging-for-repository-sca.patch | 100 ++++++++++++ ...eriodic-progress-while-waiting-for-c.patch | 152 ------------------ ...-HashiCorp-Vault-Parameter-Provider-.patch | 2 +- ...12-Remove-Develocity-Maven-extension.patch | 6 +- 13 files changed, 314 insertions(+), 469 deletions(-) create mode 100644 nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch delete mode 100644 nifi/stackable/patches/2.12.0/0007-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch create mode 100644 nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch delete mode 100644 nifi/stackable/patches/2.6.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch create mode 100644 nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch delete mode 100644 nifi/stackable/patches/2.9.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch diff --git a/CHANGELOG.md b/CHANGELOG.md index bcdfc15c9..97526092d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to this project will be documented in this file. - zookeeper: Add 3.9.6 ([#1662]). - airflow, superset, druid, nifi: Add SBOMs for the frontend (npm) dependencies ([#1600]). -- nifi: Backport NIFI-15958 to log periodic progress while waiting for the content archive scan and provenance re-index, for `2.6.0`, `2.7.2`, and `2.9.0` ([#1611]). +- nifi: Backport [NIFI-15958](https://issues.apache.org/jira/browse/NIFI-15958) to log progress during the content archive scan and provenance re-index ([#1611] and [#1726]). - hbase: Add an SBOM for the web UI (npm) dependencies, which are unpacked from webjars and therefore not covered by the CycloneDX Maven plugin ([#1620]). - trino: Add SBOMs for the web UI, both for the two npm projects behind it and for the pre-built JavaScript vendored into the source tree ([#1620]). - hive: Restore the `get_table` and `get_table_objects_by_name` Thrift methods that HIVE-26537 removed in Hive 4.0.1, for `4.2.0` ([#1636]). @@ -187,6 +187,7 @@ All notable changes to this project will be documented in this file. [#1721]: https://github.com/stackabletech/docker-images/pull/1721 [#1723]: https://github.com/stackabletech/docker-images/pull/1723 [#1725]: https://github.com/stackabletech/docker-images/pull/1725 +[#1726]: https://github.com/stackabletech/docker-images/pull/1726 ## [26.7.0] - 2026-07-21 diff --git a/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch b/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch new file mode 100644 index 000000000..c36f40765 --- /dev/null +++ b/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch @@ -0,0 +1,100 @@ +From fed2334144a3f2e7a3114ee67061ab6d9c1d18d1 Mon Sep 17 00:00:00 2001 +From: exceptionfactory +Date: Fri, 11 Sep 2026 11:09:10 -0500 +Subject: =?UTF-8?q?NIFI-15958=20Added=20progress=20logging=20for=20reposit?= + =?UTF-8?q?ory=20scanning=0AThis=20closes=20#11671?= + +--- + .../provenance/store/WriteAheadStorePartition.java | 10 ++++++++++ + .../repository/FileSystemRepository.java | 14 ++++++++++++-- + 2 files changed, 22 insertions(+), 2 deletions(-) + +diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java +index 688ba9da577bf3e2f2779afa35551f4f56e3a81b..5aeff416a9cad646887833270ecf434742909e08 100644 +--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java ++++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java +@@ -59,6 +59,7 @@ import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; + import java.util.concurrent.Future; + import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.atomic.AtomicLong; + import java.util.concurrent.atomic.AtomicReference; + import java.util.stream.Collectors; +@@ -66,6 +67,7 @@ import java.util.stream.Stream; + + public class WriteAheadStorePartition implements EventStorePartition { + private static final Logger logger = LoggerFactory.getLogger(WriteAheadStorePartition.class); ++ private static final int REINDEX_FILE_PROGRESS_LOG_INTERVAL = 25; + + private final RepositoryConfiguration config; + private final File partitionDirectory; +@@ -611,6 +613,8 @@ public class WriteAheadStorePartition implements EventStorePartition { + final ExecutorService executor = Executors.newFixedThreadPool(Math.min(4, eventFilesToReindex.size()), new NamedThreadFactory("Re-Index Provenance Events", true)); + final List> futures = new ArrayList<>(eventFilesToReindex.size()); + final AtomicLong reindexedCount = new AtomicLong(0L); ++ final AtomicInteger filesReindexed = new AtomicInteger(0); ++ final int totalFilesToReindex = eventFilesToReindex.size(); + + // Re-Index the last bunch of events. + // We don't use an Event Iterator here because it's possible that one of the event files could be corrupt (for example, if NiFi does while +@@ -659,6 +663,12 @@ public class WriteAheadStorePartition implements EventStorePartition { + logger.warn("Failed to find event with ID {} in Event File {}", minEventIdToReindex, eventFile, eof); + } catch (final Exception e) { + logger.error("Failed to index Provenance Events found in {}", eventFile, e); ++ } finally { ++ final int filesCompleted = filesReindexed.incrementAndGet(); ++ if (filesCompleted == 1 || filesCompleted == totalFilesToReindex || filesCompleted % REINDEX_FILE_PROGRESS_LOG_INTERVAL == 0) { ++ logger.info("Re-indexed {} of {} Files for Partition [{}] including {} Events", ++ filesCompleted, totalFilesToReindex, partitionName, reindexedCount.get()); ++ } + } + }; + +diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java +index ddcb048da27251a94ca7968336435fe9ff93564a..7a9fc1eda6f07cb3849c6091f188b380234bfadb 100644 +--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java ++++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java +@@ -89,6 +89,7 @@ public class FileSystemRepository implements ContentRepository { + public static final long MIN_CLEANUP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(1L); + public static final long DEFAULT_CLEANUP_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(1L); + public static final String ARCHIVE_DIR_NAME = "archive"; ++ private static final int ARCHIVE_SCAN_SECTION_LOG_INTERVAL = 256; + // 100 MB cap for the configurable NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE property to prevent + // unnecessarily large resource claim files + public static final String APPENDABLE_CLAIM_LENGTH_CAP = "100 MB"; +@@ -326,7 +327,7 @@ public class FileSystemRepository implements ContentRepository { + + // If the path didn't exist to begin with, there's no archive directory, so don't bother scanning. + if (pathExists) { +- futures.add(executor.submit(() -> scanArchiveDirectories(realPath.toFile(), containerState))); ++ futures.add(executor.submit(() -> scanArchiveDirectories(containerName, realPath.toFile(), containerState))); + } + } + +@@ -349,7 +350,8 @@ public class FileSystemRepository implements ContentRepository { + containers.putAll(realPathMap); + } + +- private void scanArchiveDirectories(final File containerDir, final ContainerState containerState) { ++ private void scanArchiveDirectories(final String containerName, final File containerDir, final ContainerState containerState) { ++ long archivedFilesFound = 0L; + for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) { + final File sectionDir = new File(containerDir, String.valueOf(i)); + final File archiveDir = new File(sectionDir, ARCHIVE_DIR_NAME); +@@ -363,7 +365,15 @@ public class FileSystemRepository implements ContentRepository { + } + + containerState.incrementArchiveCount(filenames.length); ++ archivedFilesFound += filenames.length; ++ ++ final int sectionsScanned = i + 1; ++ if (sectionsScanned % ARCHIVE_SCAN_SECTION_LOG_INTERVAL == 0 && sectionsScanned < SECTIONS_PER_CONTAINER) { ++ LOG.info("Scanned {} Sections for [{}] found {} Archived Files", sectionsScanned, containerName, archivedFilesFound); ++ } + } ++ ++ LOG.info("Finished scanning {} Sections for [{}] found {} Archived Files", SECTIONS_PER_CONTAINER, containerName, archivedFilesFound); + } + + // Visible for testing diff --git a/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch b/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch deleted file mode 100644 index 37241af82..000000000 --- a/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch +++ /dev/null @@ -1,152 +0,0 @@ -From a82c1cda1048a94e281f0ea68f21efb658739fd5 Mon Sep 17 00:00:00 2001 -From: Lars Francke -Date: Wed, 20 May 2026 16:33:13 +0200 -Subject: NIFI-15958 Log periodic progress while waiting for content archive - scan and provenance re-index - -Previously these calls blocked indefinitely on a Future.get() call. Now they wake up once a minute to report progress. - -Related: NIFI-4737 ---- - .../PartitionedWriteAheadEventStore.java | 41 ++++++++++++++---- - .../repository/FileSystemRepository.java | 42 +++++++++++++++---- - 2 files changed, 68 insertions(+), 15 deletions(-) - -diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -index 749ca1d04e99ccd4f1a792decca847281697c6b6..f242a01976c7531f281d27b49b17ca1a3dc80a1c 100644 ---- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -@@ -24,6 +24,8 @@ import org.apache.nifi.provenance.index.EventIndex; - import org.apache.nifi.provenance.serialization.EventFileCompressor; - import org.apache.nifi.provenance.store.iterator.AggregateEventIterator; - import org.apache.nifi.provenance.store.iterator.EventIterator; -+import org.slf4j.Logger; -+import org.slf4j.LoggerFactory; - - import java.io.File; - import java.io.IOException; -@@ -37,9 +39,13 @@ import java.util.concurrent.ExecutorService; - import java.util.concurrent.Executors; - import java.util.concurrent.Future; - import java.util.concurrent.LinkedBlockingQueue; -+import java.util.concurrent.TimeUnit; -+import java.util.concurrent.TimeoutException; - import java.util.concurrent.atomic.AtomicLong; - - public class PartitionedWriteAheadEventStore extends PartitionedEventStore { -+ private static final Logger logger = LoggerFactory.getLogger(PartitionedWriteAheadEventStore.class); -+ private static final long REINDEX_PROGRESS_LOG_INTERVAL_MINUTES = 1; - private final BlockingQueue filesToCompress; - private final List partitions; - private final RepositoryConfiguration repoConfig; -@@ -125,14 +131,35 @@ public class PartitionedWriteAheadEventStore extends PartitionedEventStore { - } - - executor.shutdown(); -+ -+ final long startNanos = System.nanoTime(); -+ final long warnIntervalNanos = TimeUnit.MINUTES.toNanos(REINDEX_PROGRESS_LOG_INTERVAL_MINUTES); -+ long nextWarnAtNanos = warnIntervalNanos; -+ - for (final Future future : futures) { -- try { -- future.get(); -- } catch (InterruptedException e) { -- Thread.currentThread().interrupt(); -- throw new RuntimeException("Failed to re-index events because Thread was interrupted", e); -- } catch (ExecutionException e) { -- throw new RuntimeException("Failed to re-index events", e); -+ while (true) { -+ try { -+ future.get(REINDEX_PROGRESS_LOG_INTERVAL_MINUTES, TimeUnit.MINUTES); -+ break; -+ } catch (final TimeoutException e) { -+ final long elapsedNanos = System.nanoTime() - startNanos; -+ if (elapsedNanos >= nextWarnAtNanos) { -+ long pending = 0; -+ for (final Future f : futures) { -+ if (!f.isDone()) { -+ pending++; -+ } -+ } -+ logger.info("Provenance re-indexing still in progress after {} seconds; {} of {} partitions still pending", -+ TimeUnit.NANOSECONDS.toSeconds(elapsedNanos), pending, numPartitions); -+ nextWarnAtNanos = elapsedNanos + warnIntervalNanos; -+ } -+ } catch (final InterruptedException e) { -+ Thread.currentThread().interrupt(); -+ throw new RuntimeException("Failed to re-index events because Thread was interrupted", e); -+ } catch (final ExecutionException e) { -+ throw new RuntimeException("Failed to re-index events", e); -+ } - } - } - } -diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -index ddcb048da27251a94ca7968336435fe9ff93564a..ff1c0a6527603daf940a24960e58efcbcffcd29c 100644 ---- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -@@ -74,6 +74,7 @@ import java.util.concurrent.Future; - import java.util.concurrent.LinkedBlockingQueue; - import java.util.concurrent.ScheduledExecutorService; - import java.util.concurrent.TimeUnit; -+import java.util.concurrent.TimeoutException; - import java.util.concurrent.atomic.AtomicLong; - import java.util.concurrent.locks.Condition; - import java.util.concurrent.locks.Lock; -@@ -89,6 +90,7 @@ public class FileSystemRepository implements ContentRepository { - public static final long MIN_CLEANUP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(1L); - public static final long DEFAULT_CLEANUP_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(1L); - public static final String ARCHIVE_DIR_NAME = "archive"; -+ private static final long ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES = 1L; - // 100 MB cap for the configurable NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE property to prevent - // unnecessarily large resource claim files - public static final String APPENDABLE_CLAIM_LENGTH_CAP = "100 MB"; -@@ -332,15 +334,39 @@ public class FileSystemRepository implements ContentRepository { - - executor.shutdown(); - -- // Wait for all futures to complete -+ // Wait for all futures to complete, logging a periodic warning if the archive scan is taking a long time. -+ // On busy installs the archive directories can hold large numbers of files. -+ final long startNanos = System.nanoTime(); -+ final long warnIntervalNanos = TimeUnit.MINUTES.toNanos(ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES); -+ long nextWarnAtNanos = warnIntervalNanos; -+ - for (final Future future : futures) { -- try { -- future.get(); -- } catch (final ExecutionException | InterruptedException e) { -- if (e.getCause() instanceof final IOException ioException) { -- throw ioException; -- } else { -- throw new RuntimeException(e); -+ while (true) { -+ try { -+ future.get(ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES, TimeUnit.MINUTES); -+ break; -+ } catch (final TimeoutException e) { -+ final long elapsedNanos = System.nanoTime() - startNanos; -+ if (elapsedNanos >= nextWarnAtNanos) { -+ long pending = 0; -+ for (final Future f : futures) { -+ if (!f.isDone()) { -+ pending++; -+ } -+ } -+ LOG.info("Content repository archive directory scan still in progress after {} seconds; {} of {} containers still pending", -+ TimeUnit.NANOSECONDS.toSeconds(elapsedNanos), pending, futures.size()); -+ nextWarnAtNanos = elapsedNanos + warnIntervalNanos; -+ } -+ } catch (final InterruptedException e) { -+ Thread.currentThread().interrupt(); -+ throw new RuntimeException("Failed to scan archive directories because Thread was interrupted", e); -+ } catch (final ExecutionException e) { -+ if (e.getCause() instanceof final IOException ioException) { -+ throw ioException; -+ } else { -+ throw new RuntimeException("Failed to scan archive directories", e); -+ } - } - } - } diff --git a/nifi/stackable/patches/2.12.0/0008-NIFI-16369-Removed-custom-Iceberg-REST-Client-to-fix.patch b/nifi/stackable/patches/2.12.0/0008-NIFI-16369-Removed-custom-Iceberg-REST-Client-to-fix.patch index dbb8433a3..6af03f0b3 100644 --- a/nifi/stackable/patches/2.12.0/0008-NIFI-16369-Removed-custom-Iceberg-REST-Client-to-fix.patch +++ b/nifi/stackable/patches/2.12.0/0008-NIFI-16369-Removed-custom-Iceberg-REST-Client-to-fix.patch @@ -1,4 +1,4 @@ -From c8d8dbf331280344b8beac46e00c1c26dd6ba0c9 Mon Sep 17 00:00:00 2001 +From a29d0205697f62f2426979fda1a5b8407b414a8a Mon Sep 17 00:00:00 2001 From: David Handermann Date: Tue, 22 Sep 2026 04:10:27 -0500 Subject: NIFI-16369 Removed custom Iceberg REST Client to fix OAuth 2 diff --git a/nifi/stackable/patches/2.12.0/0009-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch b/nifi/stackable/patches/2.12.0/0009-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch index b5f30d4e5..e074fd13e 100644 --- a/nifi/stackable/patches/2.12.0/0009-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch +++ b/nifi/stackable/patches/2.12.0/0009-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch @@ -1,4 +1,4 @@ -From c0c336f0bb4f460487d1183629d31a4c281ecfbb Mon Sep 17 00:00:00 2001 +From 40f64728560fae1b6c6337e453dae05f40f7f889 Mon Sep 17 00:00:00 2001 From: Fabian Grosch Date: Tue, 6 Oct 2026 17:33:25 +0200 Subject: NIFI-16431 Allow HashiCorp Vault Parameter Provider to list secrets diff --git a/nifi/stackable/patches/2.12.0/0010-Remove-Develocity-Maven-extension.patch b/nifi/stackable/patches/2.12.0/0010-Remove-Develocity-Maven-extension.patch index e07babe14..bbeaffc01 100644 --- a/nifi/stackable/patches/2.12.0/0010-Remove-Develocity-Maven-extension.patch +++ b/nifi/stackable/patches/2.12.0/0010-Remove-Develocity-Maven-extension.patch @@ -1,4 +1,4 @@ -From 4ad1abb2b2827d8a54f85413bba9aa0d2b5da727 Mon Sep 17 00:00:00 2001 +From 4601c5c63cc15cc2a6d94cf576a7fb5eab7b158e Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 15 Oct 2026 13:36:40 +0200 Subject: Remove Develocity Maven extension @@ -19,7 +19,7 @@ instead of bumping it. diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml deleted file mode 100644 -index f320f96953..0000000000 +index f320f96953a712dbd490b57906d014a90b320530..0000000000000000000000000000000000000000 --- a/.mvn/develocity.xml +++ /dev/null @@ -1,47 +0,0 @@ @@ -72,7 +72,7 @@ index f320f96953..0000000000 - diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml deleted file mode 100644 -index 3eddf27e9f..0000000000 +index 3eddf27e9f83028669ab4d4c6383c1a8c263b6dc..0000000000000000000000000000000000000000 --- a/.mvn/extensions.xml +++ /dev/null @@ -1,34 +0,0 @@ diff --git a/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch b/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch new file mode 100644 index 000000000..19591208e --- /dev/null +++ b/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch @@ -0,0 +1,100 @@ +From 4892c479a14454865e5748e20c39b814045f5fc3 Mon Sep 17 00:00:00 2001 +From: exceptionfactory +Date: Fri, 11 Sep 2026 11:09:10 -0500 +Subject: =?UTF-8?q?NIFI-15958=20Added=20progress=20logging=20for=20reposit?= + =?UTF-8?q?ory=20scanning=0AThis=20closes=20#11671?= + +--- + .../provenance/store/WriteAheadStorePartition.java | 10 ++++++++++ + .../repository/FileSystemRepository.java | 14 ++++++++++++-- + 2 files changed, 22 insertions(+), 2 deletions(-) + +diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java +index fb97f29bb85e252dd36d7f3fbb71d657a35cba60..1a849576aa95109253893ced1ae8b9b2f1b2dfc4 100644 +--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java ++++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java +@@ -57,6 +57,7 @@ import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; + import java.util.concurrent.Future; + import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.atomic.AtomicLong; + import java.util.concurrent.atomic.AtomicReference; + import java.util.stream.Collectors; +@@ -64,6 +65,7 @@ import java.util.stream.Stream; + + public class WriteAheadStorePartition implements EventStorePartition { + private static final Logger logger = LoggerFactory.getLogger(WriteAheadStorePartition.class); ++ private static final int REINDEX_FILE_PROGRESS_LOG_INTERVAL = 25; + + private final RepositoryConfiguration config; + private final File partitionDirectory; +@@ -608,6 +610,8 @@ public class WriteAheadStorePartition implements EventStorePartition { + final ExecutorService executor = Executors.newFixedThreadPool(Math.min(4, eventFilesToReindex.size()), new NamedThreadFactory("Re-Index Provenance Events", true)); + final List> futures = new ArrayList<>(eventFilesToReindex.size()); + final AtomicLong reindexedCount = new AtomicLong(0L); ++ final AtomicInteger filesReindexed = new AtomicInteger(0); ++ final int totalFilesToReindex = eventFilesToReindex.size(); + + // Re-Index the last bunch of events. + // We don't use an Event Iterator here because it's possible that one of the event files could be corrupt (for example, if NiFi does while +@@ -656,6 +660,12 @@ public class WriteAheadStorePartition implements EventStorePartition { + logger.warn("Failed to find event with ID {} in Event File {}", minEventIdToReindex, eventFile, eof); + } catch (final Exception e) { + logger.error("Failed to index Provenance Events found in {}", eventFile, e); ++ } finally { ++ final int filesCompleted = filesReindexed.incrementAndGet(); ++ if (filesCompleted == 1 || filesCompleted == totalFilesToReindex || filesCompleted % REINDEX_FILE_PROGRESS_LOG_INTERVAL == 0) { ++ logger.info("Re-indexed {} of {} Files for Partition [{}] including {} Events", ++ filesCompleted, totalFilesToReindex, partitionName, reindexedCount.get()); ++ } + } + }; + +diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java +index c2d21a5565747ee52f41d5886ae9b05b5b0db127..03744ccd634704a1d08df05fd1f1f71072e9f9a0 100644 +--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java ++++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java +@@ -87,6 +87,7 @@ public class FileSystemRepository implements ContentRepository { + public static final long MIN_CLEANUP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(1L); + public static final long DEFAULT_CLEANUP_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(1L); + public static final String ARCHIVE_DIR_NAME = "archive"; ++ private static final int ARCHIVE_SCAN_SECTION_LOG_INTERVAL = 256; + // 100 MB cap for the configurable NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE property to prevent + // unnecessarily large resource claim files + public static final String APPENDABLE_CLAIM_LENGTH_CAP = "100 MB"; +@@ -315,7 +316,7 @@ public class FileSystemRepository implements ContentRepository { + + // If the path didn't exist to begin with, there's no archive directory, so don't bother scanning. + if (pathExists) { +- futures.add(executor.submit(() -> scanArchiveDirectories(realPath.toFile(), containerState))); ++ futures.add(executor.submit(() -> scanArchiveDirectories(containerName, realPath.toFile(), containerState))); + } + } + +@@ -338,7 +339,8 @@ public class FileSystemRepository implements ContentRepository { + containers.putAll(realPathMap); + } + +- private void scanArchiveDirectories(final File containerDir, final ContainerState containerState) { ++ private void scanArchiveDirectories(final String containerName, final File containerDir, final ContainerState containerState) { ++ long archivedFilesFound = 0L; + for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) { + final File sectionDir = new File(containerDir, String.valueOf(i)); + final File archiveDir = new File(sectionDir, ARCHIVE_DIR_NAME); +@@ -352,7 +354,15 @@ public class FileSystemRepository implements ContentRepository { + } + + containerState.incrementArchiveCount(filenames.length); ++ archivedFilesFound += filenames.length; ++ ++ final int sectionsScanned = i + 1; ++ if (sectionsScanned % ARCHIVE_SCAN_SECTION_LOG_INTERVAL == 0 && sectionsScanned < SECTIONS_PER_CONTAINER) { ++ LOG.info("Scanned {} Sections for [{}] found {} Archived Files", sectionsScanned, containerName, archivedFilesFound); ++ } + } ++ ++ LOG.info("Finished scanning {} Sections for [{}] found {} Archived Files", SECTIONS_PER_CONTAINER, containerName, archivedFilesFound); + } + + // Visible for testing diff --git a/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch b/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch deleted file mode 100644 index 5bca9c0cf..000000000 --- a/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch +++ /dev/null @@ -1,152 +0,0 @@ -From a8d7887240e96c281a30cf051db85d588f7d6e80 Mon Sep 17 00:00:00 2001 -From: Lars Francke -Date: Wed, 20 May 2026 16:33:13 +0200 -Subject: NIFI-15958 Log periodic progress while waiting for content archive - scan and provenance re-index - -Previously these calls blocked indefinitely on a Future.get() call. Now they wake up once a minute to report progress. - -Related: NIFI-4737 ---- - .../PartitionedWriteAheadEventStore.java | 41 ++++++++++++++---- - .../repository/FileSystemRepository.java | 42 +++++++++++++++---- - 2 files changed, 68 insertions(+), 15 deletions(-) - -diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -index 749ca1d04e99ccd4f1a792decca847281697c6b6..f242a01976c7531f281d27b49b17ca1a3dc80a1c 100644 ---- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -@@ -24,6 +24,8 @@ import org.apache.nifi.provenance.index.EventIndex; - import org.apache.nifi.provenance.serialization.EventFileCompressor; - import org.apache.nifi.provenance.store.iterator.AggregateEventIterator; - import org.apache.nifi.provenance.store.iterator.EventIterator; -+import org.slf4j.Logger; -+import org.slf4j.LoggerFactory; - - import java.io.File; - import java.io.IOException; -@@ -37,9 +39,13 @@ import java.util.concurrent.ExecutorService; - import java.util.concurrent.Executors; - import java.util.concurrent.Future; - import java.util.concurrent.LinkedBlockingQueue; -+import java.util.concurrent.TimeUnit; -+import java.util.concurrent.TimeoutException; - import java.util.concurrent.atomic.AtomicLong; - - public class PartitionedWriteAheadEventStore extends PartitionedEventStore { -+ private static final Logger logger = LoggerFactory.getLogger(PartitionedWriteAheadEventStore.class); -+ private static final long REINDEX_PROGRESS_LOG_INTERVAL_MINUTES = 1; - private final BlockingQueue filesToCompress; - private final List partitions; - private final RepositoryConfiguration repoConfig; -@@ -125,14 +131,35 @@ public class PartitionedWriteAheadEventStore extends PartitionedEventStore { - } - - executor.shutdown(); -+ -+ final long startNanos = System.nanoTime(); -+ final long warnIntervalNanos = TimeUnit.MINUTES.toNanos(REINDEX_PROGRESS_LOG_INTERVAL_MINUTES); -+ long nextWarnAtNanos = warnIntervalNanos; -+ - for (final Future future : futures) { -- try { -- future.get(); -- } catch (InterruptedException e) { -- Thread.currentThread().interrupt(); -- throw new RuntimeException("Failed to re-index events because Thread was interrupted", e); -- } catch (ExecutionException e) { -- throw new RuntimeException("Failed to re-index events", e); -+ while (true) { -+ try { -+ future.get(REINDEX_PROGRESS_LOG_INTERVAL_MINUTES, TimeUnit.MINUTES); -+ break; -+ } catch (final TimeoutException e) { -+ final long elapsedNanos = System.nanoTime() - startNanos; -+ if (elapsedNanos >= nextWarnAtNanos) { -+ long pending = 0; -+ for (final Future f : futures) { -+ if (!f.isDone()) { -+ pending++; -+ } -+ } -+ logger.info("Provenance re-indexing still in progress after {} seconds; {} of {} partitions still pending", -+ TimeUnit.NANOSECONDS.toSeconds(elapsedNanos), pending, numPartitions); -+ nextWarnAtNanos = elapsedNanos + warnIntervalNanos; -+ } -+ } catch (final InterruptedException e) { -+ Thread.currentThread().interrupt(); -+ throw new RuntimeException("Failed to re-index events because Thread was interrupted", e); -+ } catch (final ExecutionException e) { -+ throw new RuntimeException("Failed to re-index events", e); -+ } - } - } - } -diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -index c2d21a5565747ee52f41d5886ae9b05b5b0db127..b3c0a8dcf681c964f8de47556e973a6435e6633d 100644 ---- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -@@ -72,6 +72,7 @@ import java.util.concurrent.Future; - import java.util.concurrent.LinkedBlockingQueue; - import java.util.concurrent.ScheduledExecutorService; - import java.util.concurrent.TimeUnit; -+import java.util.concurrent.TimeoutException; - import java.util.concurrent.atomic.AtomicLong; - import java.util.concurrent.locks.Condition; - import java.util.concurrent.locks.Lock; -@@ -87,6 +88,7 @@ public class FileSystemRepository implements ContentRepository { - public static final long MIN_CLEANUP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(1L); - public static final long DEFAULT_CLEANUP_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(1L); - public static final String ARCHIVE_DIR_NAME = "archive"; -+ private static final long ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES = 1L; - // 100 MB cap for the configurable NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE property to prevent - // unnecessarily large resource claim files - public static final String APPENDABLE_CLAIM_LENGTH_CAP = "100 MB"; -@@ -321,15 +323,39 @@ public class FileSystemRepository implements ContentRepository { - - executor.shutdown(); - -- // Wait for all futures to complete -+ // Wait for all futures to complete, logging a periodic warning if the archive scan is taking a long time. -+ // On busy installs the archive directories can hold large numbers of files. -+ final long startNanos = System.nanoTime(); -+ final long warnIntervalNanos = TimeUnit.MINUTES.toNanos(ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES); -+ long nextWarnAtNanos = warnIntervalNanos; -+ - for (final Future future : futures) { -- try { -- future.get(); -- } catch (final ExecutionException | InterruptedException e) { -- if (e.getCause() instanceof IOException) { -- throw (IOException) e.getCause(); -- } else { -- throw new RuntimeException(e); -+ while (true) { -+ try { -+ future.get(ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES, TimeUnit.MINUTES); -+ break; -+ } catch (final TimeoutException e) { -+ final long elapsedNanos = System.nanoTime() - startNanos; -+ if (elapsedNanos >= nextWarnAtNanos) { -+ long pending = 0; -+ for (final Future f : futures) { -+ if (!f.isDone()) { -+ pending++; -+ } -+ } -+ LOG.info("Content repository archive directory scan still in progress after {} seconds; {} of {} containers still pending", -+ TimeUnit.NANOSECONDS.toSeconds(elapsedNanos), pending, futures.size()); -+ nextWarnAtNanos = elapsedNanos + warnIntervalNanos; -+ } -+ } catch (final InterruptedException e) { -+ Thread.currentThread().interrupt(); -+ throw new RuntimeException("Failed to scan archive directories because Thread was interrupted", e); -+ } catch (final ExecutionException e) { -+ if (e.getCause() instanceof final IOException ioException) { -+ throw ioException; -+ } else { -+ throw new RuntimeException("Failed to scan archive directories", e); -+ } - } - } - } diff --git a/nifi/stackable/patches/2.6.0/0011-Remove-Develocity-Maven-extension.patch b/nifi/stackable/patches/2.6.0/0011-Remove-Develocity-Maven-extension.patch index 2ecf94d2b..3a1d11795 100644 --- a/nifi/stackable/patches/2.6.0/0011-Remove-Develocity-Maven-extension.patch +++ b/nifi/stackable/patches/2.6.0/0011-Remove-Develocity-Maven-extension.patch @@ -1,4 +1,4 @@ -From 69258be36cd38d7bfb45809f9e29f46a5319df40 Mon Sep 17 00:00:00 2001 +From 1a1f44e9401bd6816117894fd81c53dccb081200 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 15 Oct 2026 13:36:39 +0200 Subject: Remove Develocity Maven extension @@ -19,7 +19,7 @@ instead of bumping it. diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml deleted file mode 100644 -index 8e73aa0558..0000000000 +index 8e73aa05580f981bc53502ec97f2c74003e03088..0000000000000000000000000000000000000000 --- a/.mvn/develocity.xml +++ /dev/null @@ -1,46 +0,0 @@ @@ -71,7 +71,7 @@ index 8e73aa0558..0000000000 - diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml deleted file mode 100644 -index 3eddf27e9f..0000000000 +index 3eddf27e9f83028669ab4d4c6383c1a8c263b6dc..0000000000000000000000000000000000000000 --- a/.mvn/extensions.xml +++ /dev/null @@ -1,34 +0,0 @@ diff --git a/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch b/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch new file mode 100644 index 000000000..00c5784f0 --- /dev/null +++ b/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch @@ -0,0 +1,100 @@ +From 5f93946bccf8c77d9aacf548f32a44ea151aaefc Mon Sep 17 00:00:00 2001 +From: exceptionfactory +Date: Fri, 11 Sep 2026 11:09:10 -0500 +Subject: =?UTF-8?q?NIFI-15958=20Added=20progress=20logging=20for=20reposit?= + =?UTF-8?q?ory=20scanning=0AThis=20closes=20#11671?= + +--- + .../provenance/store/WriteAheadStorePartition.java | 10 ++++++++++ + .../repository/FileSystemRepository.java | 14 ++++++++++++-- + 2 files changed, 22 insertions(+), 2 deletions(-) + +diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java +index 1ccc25beb289d3056dc9d55e50eee372988e4ebc..a6dee4e4c1e43c6709aa824e275b0aaff95eec34 100644 +--- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java ++++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/WriteAheadStorePartition.java +@@ -57,6 +57,7 @@ import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; + import java.util.concurrent.Future; + import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicInteger; + import java.util.concurrent.atomic.AtomicLong; + import java.util.concurrent.atomic.AtomicReference; + import java.util.stream.Collectors; +@@ -64,6 +65,7 @@ import java.util.stream.Stream; + + public class WriteAheadStorePartition implements EventStorePartition { + private static final Logger logger = LoggerFactory.getLogger(WriteAheadStorePartition.class); ++ private static final int REINDEX_FILE_PROGRESS_LOG_INTERVAL = 25; + + private final RepositoryConfiguration config; + private final File partitionDirectory; +@@ -605,6 +607,8 @@ public class WriteAheadStorePartition implements EventStorePartition { + final ExecutorService executor = Executors.newFixedThreadPool(Math.min(4, eventFilesToReindex.size()), new NamedThreadFactory("Re-Index Provenance Events", true)); + final List> futures = new ArrayList<>(eventFilesToReindex.size()); + final AtomicLong reindexedCount = new AtomicLong(0L); ++ final AtomicInteger filesReindexed = new AtomicInteger(0); ++ final int totalFilesToReindex = eventFilesToReindex.size(); + + // Re-Index the last bunch of events. + // We don't use an Event Iterator here because it's possible that one of the event files could be corrupt (for example, if NiFi does while +@@ -653,6 +657,12 @@ public class WriteAheadStorePartition implements EventStorePartition { + logger.warn("Failed to find event with ID {} in Event File {}", minEventIdToReindex, eventFile, eof); + } catch (final Exception e) { + logger.error("Failed to index Provenance Events found in {}", eventFile, e); ++ } finally { ++ final int filesCompleted = filesReindexed.incrementAndGet(); ++ if (filesCompleted == 1 || filesCompleted == totalFilesToReindex || filesCompleted % REINDEX_FILE_PROGRESS_LOG_INTERVAL == 0) { ++ logger.info("Re-indexed {} of {} Files for Partition [{}] including {} Events", ++ filesCompleted, totalFilesToReindex, partitionName, reindexedCount.get()); ++ } + } + }; + +diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java +index 15ec9e78cddf4adf493ffa7a82e4cfef7e2c15ac..336130e3814474f8f30462d1dace3ffb5b4179a7 100644 +--- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java ++++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java +@@ -87,6 +87,7 @@ public class FileSystemRepository implements ContentRepository { + public static final long MIN_CLEANUP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(1L); + public static final long DEFAULT_CLEANUP_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(1L); + public static final String ARCHIVE_DIR_NAME = "archive"; ++ private static final int ARCHIVE_SCAN_SECTION_LOG_INTERVAL = 256; + // 100 MB cap for the configurable NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE property to prevent + // unnecessarily large resource claim files + public static final String APPENDABLE_CLAIM_LENGTH_CAP = "100 MB"; +@@ -313,7 +314,7 @@ public class FileSystemRepository implements ContentRepository { + + // If the path didn't exist to begin with, there's no archive directory, so don't bother scanning. + if (pathExists) { +- futures.add(executor.submit(() -> scanArchiveDirectories(realPath.toFile(), containerState))); ++ futures.add(executor.submit(() -> scanArchiveDirectories(containerName, realPath.toFile(), containerState))); + } + } + +@@ -336,7 +337,8 @@ public class FileSystemRepository implements ContentRepository { + containers.putAll(realPathMap); + } + +- private void scanArchiveDirectories(final File containerDir, final ContainerState containerState) { ++ private void scanArchiveDirectories(final String containerName, final File containerDir, final ContainerState containerState) { ++ long archivedFilesFound = 0L; + for (int i = 0; i < SECTIONS_PER_CONTAINER; i++) { + final File sectionDir = new File(containerDir, String.valueOf(i)); + final File archiveDir = new File(sectionDir, ARCHIVE_DIR_NAME); +@@ -350,7 +352,15 @@ public class FileSystemRepository implements ContentRepository { + } + + containerState.incrementArchiveCount(filenames.length); ++ archivedFilesFound += filenames.length; ++ ++ final int sectionsScanned = i + 1; ++ if (sectionsScanned % ARCHIVE_SCAN_SECTION_LOG_INTERVAL == 0 && sectionsScanned < SECTIONS_PER_CONTAINER) { ++ LOG.info("Scanned {} Sections for [{}] found {} Archived Files", sectionsScanned, containerName, archivedFilesFound); ++ } + } ++ ++ LOG.info("Finished scanning {} Sections for [{}] found {} Archived Files", SECTIONS_PER_CONTAINER, containerName, archivedFilesFound); + } + + // Visible for testing diff --git a/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch b/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch deleted file mode 100644 index f0e80e64d..000000000 --- a/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Log-periodic-progress-while-waiting-for-c.patch +++ /dev/null @@ -1,152 +0,0 @@ -From ab12fadcf4b9b20e448098d6f6bbaf2e8ab6af87 Mon Sep 17 00:00:00 2001 -From: Lars Francke -Date: Wed, 20 May 2026 16:33:13 +0200 -Subject: NIFI-15958 Log periodic progress while waiting for content archive - scan and provenance re-index - -Previously these calls blocked indefinitely on a Future.get() call. Now they wake up once a minute to report progress. - -Related: NIFI-4737 ---- - .../PartitionedWriteAheadEventStore.java | 41 ++++++++++++++---- - .../repository/FileSystemRepository.java | 42 +++++++++++++++---- - 2 files changed, 68 insertions(+), 15 deletions(-) - -diff --git a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -index 749ca1d04e99ccd4f1a792decca847281697c6b6..f242a01976c7531f281d27b49b17ca1a3dc80a1c 100644 ---- a/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -+++ b/nifi-framework-bundle/nifi-framework-extensions/nifi-provenance-repository-bundle/nifi-persistent-provenance-repository/src/main/java/org/apache/nifi/provenance/store/PartitionedWriteAheadEventStore.java -@@ -24,6 +24,8 @@ import org.apache.nifi.provenance.index.EventIndex; - import org.apache.nifi.provenance.serialization.EventFileCompressor; - import org.apache.nifi.provenance.store.iterator.AggregateEventIterator; - import org.apache.nifi.provenance.store.iterator.EventIterator; -+import org.slf4j.Logger; -+import org.slf4j.LoggerFactory; - - import java.io.File; - import java.io.IOException; -@@ -37,9 +39,13 @@ import java.util.concurrent.ExecutorService; - import java.util.concurrent.Executors; - import java.util.concurrent.Future; - import java.util.concurrent.LinkedBlockingQueue; -+import java.util.concurrent.TimeUnit; -+import java.util.concurrent.TimeoutException; - import java.util.concurrent.atomic.AtomicLong; - - public class PartitionedWriteAheadEventStore extends PartitionedEventStore { -+ private static final Logger logger = LoggerFactory.getLogger(PartitionedWriteAheadEventStore.class); -+ private static final long REINDEX_PROGRESS_LOG_INTERVAL_MINUTES = 1; - private final BlockingQueue filesToCompress; - private final List partitions; - private final RepositoryConfiguration repoConfig; -@@ -125,14 +131,35 @@ public class PartitionedWriteAheadEventStore extends PartitionedEventStore { - } - - executor.shutdown(); -+ -+ final long startNanos = System.nanoTime(); -+ final long warnIntervalNanos = TimeUnit.MINUTES.toNanos(REINDEX_PROGRESS_LOG_INTERVAL_MINUTES); -+ long nextWarnAtNanos = warnIntervalNanos; -+ - for (final Future future : futures) { -- try { -- future.get(); -- } catch (InterruptedException e) { -- Thread.currentThread().interrupt(); -- throw new RuntimeException("Failed to re-index events because Thread was interrupted", e); -- } catch (ExecutionException e) { -- throw new RuntimeException("Failed to re-index events", e); -+ while (true) { -+ try { -+ future.get(REINDEX_PROGRESS_LOG_INTERVAL_MINUTES, TimeUnit.MINUTES); -+ break; -+ } catch (final TimeoutException e) { -+ final long elapsedNanos = System.nanoTime() - startNanos; -+ if (elapsedNanos >= nextWarnAtNanos) { -+ long pending = 0; -+ for (final Future f : futures) { -+ if (!f.isDone()) { -+ pending++; -+ } -+ } -+ logger.info("Provenance re-indexing still in progress after {} seconds; {} of {} partitions still pending", -+ TimeUnit.NANOSECONDS.toSeconds(elapsedNanos), pending, numPartitions); -+ nextWarnAtNanos = elapsedNanos + warnIntervalNanos; -+ } -+ } catch (final InterruptedException e) { -+ Thread.currentThread().interrupt(); -+ throw new RuntimeException("Failed to re-index events because Thread was interrupted", e); -+ } catch (final ExecutionException e) { -+ throw new RuntimeException("Failed to re-index events", e); -+ } - } - } - } -diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -index 15ec9e78cddf4adf493ffa7a82e4cfef7e2c15ac..9204a496b34ad19fb1734836e08beda18ad8373c 100644 ---- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -+++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/repository/FileSystemRepository.java -@@ -72,6 +72,7 @@ import java.util.concurrent.Future; - import java.util.concurrent.LinkedBlockingQueue; - import java.util.concurrent.ScheduledExecutorService; - import java.util.concurrent.TimeUnit; -+import java.util.concurrent.TimeoutException; - import java.util.concurrent.atomic.AtomicLong; - import java.util.concurrent.locks.Condition; - import java.util.concurrent.locks.Lock; -@@ -87,6 +88,7 @@ public class FileSystemRepository implements ContentRepository { - public static final long MIN_CLEANUP_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(1L); - public static final long DEFAULT_CLEANUP_INTERVAL_MILLIS = TimeUnit.MINUTES.toMillis(1L); - public static final String ARCHIVE_DIR_NAME = "archive"; -+ private static final long ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES = 1L; - // 100 MB cap for the configurable NiFiProperties.MAX_APPENDABLE_CLAIM_SIZE property to prevent - // unnecessarily large resource claim files - public static final String APPENDABLE_CLAIM_LENGTH_CAP = "100 MB"; -@@ -319,15 +321,39 @@ public class FileSystemRepository implements ContentRepository { - - executor.shutdown(); - -- // Wait for all futures to complete -+ // Wait for all futures to complete, logging a periodic warning if the archive scan is taking a long time. -+ // On busy installs the archive directories can hold large numbers of files. -+ final long startNanos = System.nanoTime(); -+ final long warnIntervalNanos = TimeUnit.MINUTES.toNanos(ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES); -+ long nextWarnAtNanos = warnIntervalNanos; -+ - for (final Future future : futures) { -- try { -- future.get(); -- } catch (final ExecutionException | InterruptedException e) { -- if (e.getCause() instanceof IOException) { -- throw (IOException) e.getCause(); -- } else { -- throw new RuntimeException(e); -+ while (true) { -+ try { -+ future.get(ARCHIVE_SCAN_PROGRESS_LOG_INTERVAL_MINUTES, TimeUnit.MINUTES); -+ break; -+ } catch (final TimeoutException e) { -+ final long elapsedNanos = System.nanoTime() - startNanos; -+ if (elapsedNanos >= nextWarnAtNanos) { -+ long pending = 0; -+ for (final Future f : futures) { -+ if (!f.isDone()) { -+ pending++; -+ } -+ } -+ LOG.info("Content repository archive directory scan still in progress after {} seconds; {} of {} containers still pending", -+ TimeUnit.NANOSECONDS.toSeconds(elapsedNanos), pending, futures.size()); -+ nextWarnAtNanos = elapsedNanos + warnIntervalNanos; -+ } -+ } catch (final InterruptedException e) { -+ Thread.currentThread().interrupt(); -+ throw new RuntimeException("Failed to scan archive directories because Thread was interrupted", e); -+ } catch (final ExecutionException e) { -+ if (e.getCause() instanceof final IOException ioException) { -+ throw ioException; -+ } else { -+ throw new RuntimeException("Failed to scan archive directories", e); -+ } - } - } - } diff --git a/nifi/stackable/patches/2.9.0/0011-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch b/nifi/stackable/patches/2.9.0/0011-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch index 295a4594f..d768dcc75 100644 --- a/nifi/stackable/patches/2.9.0/0011-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch +++ b/nifi/stackable/patches/2.9.0/0011-NIFI-16431-Allow-HashiCorp-Vault-Parameter-Provider-.patch @@ -1,4 +1,4 @@ -From 7487b9ea17719b795c752f5cbd9ccd3acdcabd83 Mon Sep 17 00:00:00 2001 +From ef5b7b7d0b55647592b1f586288b1d26f6bc5f0b Mon Sep 17 00:00:00 2001 From: Fabian Grosch Date: Tue, 6 Oct 2026 17:33:25 +0200 Subject: NIFI-16431 Allow HashiCorp Vault Parameter Provider to list secrets diff --git a/nifi/stackable/patches/2.9.0/0012-Remove-Develocity-Maven-extension.patch b/nifi/stackable/patches/2.9.0/0012-Remove-Develocity-Maven-extension.patch index 1b3c5c57e..08b6a22bb 100644 --- a/nifi/stackable/patches/2.9.0/0012-Remove-Develocity-Maven-extension.patch +++ b/nifi/stackable/patches/2.9.0/0012-Remove-Develocity-Maven-extension.patch @@ -1,4 +1,4 @@ -From 4863150986fcdcdd335c6c87d34de04f58aca7a3 Mon Sep 17 00:00:00 2001 +From 2fdf71b8cd45ec2ec0f6fec3a4fd1aa173e20ab1 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 15 Oct 2026 13:36:39 +0200 Subject: Remove Develocity Maven extension @@ -19,7 +19,7 @@ instead of bumping it. diff --git a/.mvn/develocity.xml b/.mvn/develocity.xml deleted file mode 100644 -index f320f96953..0000000000 +index f320f96953a712dbd490b57906d014a90b320530..0000000000000000000000000000000000000000 --- a/.mvn/develocity.xml +++ /dev/null @@ -1,47 +0,0 @@ @@ -72,7 +72,7 @@ index f320f96953..0000000000 - diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml deleted file mode 100644 -index 3eddf27e9f..0000000000 +index 3eddf27e9f83028669ab4d4c6383c1a8c263b6dc..0000000000000000000000000000000000000000 --- a/.mvn/extensions.xml +++ /dev/null @@ -1,34 +0,0 @@ From 729c74ab9cf4481efe4f7f605b8f298757178b6d Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:36:42 +0100 Subject: [PATCH 2/3] Apply batched suggestions from code review Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97526092d..0333e2f84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to this project will be documented in this file. - zookeeper: Add 3.9.6 ([#1662]). - airflow, superset, druid, nifi: Add SBOMs for the frontend (npm) dependencies ([#1600]). -- nifi: Backport [NIFI-15958](https://issues.apache.org/jira/browse/NIFI-15958) to log progress during the content archive scan and provenance re-index ([#1611] and [#1726]). +- nifi: Backport [NIFI-15958](https://issues.apache.org/jira/browse/NIFI-15958) to log progress during the content archive scan and provenance re-index ([#1611] and [#1727]). - hbase: Add an SBOM for the web UI (npm) dependencies, which are unpacked from webjars and therefore not covered by the CycloneDX Maven plugin ([#1620]). - trino: Add SBOMs for the web UI, both for the two npm projects behind it and for the pre-built JavaScript vendored into the source tree ([#1620]). - hive: Restore the `get_table` and `get_table_objects_by_name` Thrift methods that HIVE-26537 removed in Hive 4.0.1, for `4.2.0` ([#1636]). @@ -187,7 +187,7 @@ All notable changes to this project will be documented in this file. [#1721]: https://github.com/stackabletech/docker-images/pull/1721 [#1723]: https://github.com/stackabletech/docker-images/pull/1723 [#1725]: https://github.com/stackabletech/docker-images/pull/1725 -[#1726]: https://github.com/stackabletech/docker-images/pull/1726 +[#1727]: https://github.com/stackabletech/docker-images/pull/1727 ## [26.7.0] - 2026-07-21 From 3943fbaf8790c3a69981683aeb29b7a5bd780d6c Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:40:38 +0100 Subject: [PATCH 3/3] fix patch subject muddling --- ...-NIFI-15958-Added-progress-logging-for-repository-sca.patch | 3 +-- ...-NIFI-15958-Added-progress-logging-for-repository-sca.patch | 3 +-- ...-NIFI-15958-Added-progress-logging-for-repository-sca.patch | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch b/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch index c36f40765..aa8c6ea8a 100644 --- a/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch +++ b/nifi/stackable/patches/2.12.0/0007-NIFI-15958-Added-progress-logging-for-repository-sca.patch @@ -1,8 +1,7 @@ From fed2334144a3f2e7a3114ee67061ab6d9c1d18d1 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Fri, 11 Sep 2026 11:09:10 -0500 -Subject: =?UTF-8?q?NIFI-15958=20Added=20progress=20logging=20for=20reposit?= - =?UTF-8?q?ory=20scanning=0AThis=20closes=20#11671?= +Subject: NIFI-15958 Added progress logging for repository scanning --- .../provenance/store/WriteAheadStorePartition.java | 10 ++++++++++ diff --git a/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch b/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch index 19591208e..f1e464bbc 100644 --- a/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch +++ b/nifi/stackable/patches/2.6.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch @@ -1,8 +1,7 @@ From 4892c479a14454865e5748e20c39b814045f5fc3 Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Fri, 11 Sep 2026 11:09:10 -0500 -Subject: =?UTF-8?q?NIFI-15958=20Added=20progress=20logging=20for=20reposit?= - =?UTF-8?q?ory=20scanning=0AThis=20closes=20#11671?= +Subject: NIFI-15958 Added progress logging for repository scanning --- .../provenance/store/WriteAheadStorePartition.java | 10 ++++++++++ diff --git a/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch b/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch index 00c5784f0..75df7b640 100644 --- a/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch +++ b/nifi/stackable/patches/2.9.0/0010-NIFI-15958-Added-progress-logging-for-repository-sca.patch @@ -1,8 +1,7 @@ From 5f93946bccf8c77d9aacf548f32a44ea151aaefc Mon Sep 17 00:00:00 2001 From: exceptionfactory Date: Fri, 11 Sep 2026 11:09:10 -0500 -Subject: =?UTF-8?q?NIFI-15958=20Added=20progress=20logging=20for=20reposit?= - =?UTF-8?q?ory=20scanning=0AThis=20closes=20#11671?= +Subject: NIFI-15958 Added progress logging for repository scanning --- .../provenance/store/WriteAheadStorePartition.java | 10 ++++++++++