From 9b1cc94c9a0ed90075d8436cb4d7016f5433739f Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:44:44 +0200 Subject: [PATCH 01/13] chore: update ubi to version 10 --- stackable-base/Dockerfile | 18 ++++++++++++------ stackable-devel/Dockerfile | 16 +++++++++++----- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index 7de424ea3..9f781fa3a 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -35,13 +35,19 @@ cd /secret-operator cargo auditable --quiet build --release --package cert-tools && cargo cyclonedx --all --spec-version 1.5 --describe binaries EOF -# Find the latest version at https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5?container-tabs=gti +# Find the latest version at https://catalog.redhat.com +# Use the search to find the image you are interested in because navigating the catalog folders is impossible. +# # IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures # rather than just the "Image Digest" that references the image for the selected architecture. -# The website is broken, so you can use this to find it: -# curl https://catalog.redhat.com/en/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5 \ -# | grep -oE 'registry.redhat.io/ubi9/ubi-minimal@sha256:[a-z0-9]{64}' -FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c7d44146f826037f6873d99da479299b889473492d3c1ab8af86f08af04ec8a0 AS builder +# +# The website is awful/broken, so you can use one of the following tools to find it: +# +# docker buildx imagetools inspect registry.redhat.io/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' +# # or +# skopeo inspect --format '{{.Digest}}' docker://registry.redhat.io/ubi10/ubi-minimal +# +FROM registry.redhat.io/ubi10/ubi-minimal@sha256:8935d83eadb5c9f621c13f4dfab7e1bbd8d948317dc67157279911ede1013112 # intentionally unused ARG PRODUCT_VERSION @@ -158,7 +164,7 @@ chown ${STACKABLE_USER_UID}:0 /stackable/.curlrc # CVE-2023-37920: Remove "e-Tugra" root certificates # e-Tugra's root certificates were subject to an investigation prompted by reporting of security issues in their systems # Until they are removed by default from ca-certificates, we should remove them manually -EXPECTED_CERTS_PACKAGE="ca-certificates-2025.2.80_v9.0.305-91.el9.noarch" +EXPECTED_CERTS_PACKAGE="ca-certificates-2025.2.80_v9.0.305-102.el10_1.noarch" ACTUAL_CERTS_PACKAGE="$(rpm -qa ca-certificates)" if [ "$ACTUAL_CERTS_PACKAGE" != "$EXPECTED_CERTS_PACKAGE" ]; then echo "The ca-certificates package was updated to $ACTUAL_CERTS_PACKAGE. Please check if the e-Tugra root certificates are present. \ diff --git a/stackable-devel/Dockerfile b/stackable-devel/Dockerfile index cdec8f426..fd3c4a6b5 100644 --- a/stackable-devel/Dockerfile +++ b/stackable-devel/Dockerfile @@ -8,13 +8,19 @@ # Use `stackable-base` as a base for the final image stage instead # -# Find the latest version at https://catalog.redhat.com/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5?container-tabs=gti +# Find the latest version at https://catalog.redhat.com +# Use the search to find the image you are interested in because navigating the catalog folders is impossible. +# # IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures # rather than just the "Image Digest" that references the image for the selected architecture. -# The website is broken, so you can use this to find it: -# curl https://catalog.redhat.com/en/software/containers/ubi9/ubi-minimal/615bd9b4075b022acc111bf5 \ -# | grep -oE 'registry.redhat.io/ubi9/ubi-minimal@sha256:[a-z0-9]{64}' -FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c7d44146f826037f6873d99da479299b889473492d3c1ab8af86f08af04ec8a0 AS builder +# +# The website is awful/broken, so you can use one of the following tools to find it: +# +# docker buildx imagetools inspect registry.redhat.io/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' +# # or +# skopeo inspect --format '{{.Digest}}' docker://registry.redhat.io/ubi10/ubi-minimal +# +FROM registry.redhat.io/ubi10/ubi-minimal@sha256:8935d83eadb5c9f621c13f4dfab7e1bbd8d948317dc67157279911ede1013112 # intentionally unused ARG PRODUCT_VERSION From f569bc3bdbb0be2af693758b7366ff193350bf74 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:29:46 +0200 Subject: [PATCH 02/13] Squash merge chore/boost-from-source into feat/ubi10-hadoop --- .scripts/upload_new_boost_version.sh | 59 ++++++++++++++++++++++++++++ CHANGELOG.md | 3 ++ hadoop/Dockerfile | 16 ++++++-- hadoop/boost/Dockerfile | 40 +++++++++++++++++++ hadoop/boost/boil-config.toml | 5 +++ hadoop/hadoop/Dockerfile | 15 ++++--- hadoop/hadoop/boil-config.toml | 2 + 7 files changed, 132 insertions(+), 8 deletions(-) create mode 100755 .scripts/upload_new_boost_version.sh create mode 100644 hadoop/boost/Dockerfile create mode 100644 hadoop/boost/boil-config.toml diff --git a/.scripts/upload_new_boost_version.sh b/.scripts/upload_new_boost_version.sh new file mode 100755 index 000000000..200299691 --- /dev/null +++ b/.scripts/upload_new_boost_version.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# This script mirrors the boostorg/boost source bundle for the given version to Nexus. +# The boost source bundle is architecture independent. +# It contains it's own build system (b2) which is also built from source before building boost itself, so we don't need to worry about architecture specific builds. +# This artifact is used by the hadoop/boost local image. + + +set -euo pipefail + +VERSION=${1:?"Missing version number argument (arg 1)"} +NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"} + +read -r -s -p "Nexus Password: " NEXUS_PASSWORD +echo "" + +# https://stackoverflow.com/questions/4632028/how-to-create-a-temporary-directory +# Find the directory name of the script +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# the temp directory used, within $DIR +WORK_DIR=$(mktemp -d -p "$DIR") + +# check if tmp dir was created +if [[ ! "$WORK_DIR" || ! -d "$WORK_DIR" ]]; then + echo "Could not create temp dir" + exit 1 +fi + +# deletes the temp directory +function cleanup { + rm -rf "$WORK_DIR" +} + +# register the cleanup function to be called on the EXIT signal +trap cleanup EXIT + +cd "$WORK_DIR" || exit + +# boost does not currently publish signatures or SBOMs +BOOST_UNDERSCORE="$(echo "${VERSION}" | tr '.' '_')" +BOOST_TARBALL="boost_${BOOST_UNDERSCORE}.tar.bz2" +DOWNLOAD_URL="https://archives.boost.io/release/$VERSION/source/$BOOST_TARBALL" + +echo "Downloading boost" +if ! curl --fail -Ls -O "$DOWNLOAD_URL"; then + echo "Failed to download from $DOWNLOAD_URL" + exit 1 +fi + +FILE_NAME=$(basename "$DOWNLOAD_URL") + +echo "Uploading boost to Nexus" +if ! curl --fail -o /dev/null --progress-bar -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "$FILE_NAME" 'https://repo.stackable.tech/repository/packages/boost/'; then + echo "Failed to upload boost to Nexus" + exit 1 +fi + +echo "Successfully uploaded new version of boost ($VERSION) to Nexus" +echo "https://repo.stackable.tech/service/rest/repository/browse/packages/boost/" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa86a1e2..c87288d73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ All notable changes to this project will be documented in this file. ### Changed - hbase: Update `hbase-opa-authorizer` from `0.1.0` to `0.2.0` and then `0.3.0` ([#1446], [#1454]). +- hadoop: Build the boost library from source instead of installing the EPEL package ([#1475]). + This is a requirement for the adoption of UBI 10 as base image because the package is not available there. ### Fixed @@ -25,6 +27,7 @@ All notable changes to this project will be documented in this file. [#1463]: https://github.com/stackabletech/docker-images/pull/1463 [#1466]: https://github.com/stackabletech/docker-images/pull/1466 [#1474]: https://github.com/stackabletech/docker-images/pull/1474 +[#1475]: https://github.com/stackabletech/docker-images/pull/1475 ## [26.3.0] - 2026-03-16 diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 2757b4e8c..f5ebf2b14 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -90,6 +90,7 @@ LABEL \ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/*-src.tar.gz /stackable +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/boost /stackable/boost COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS_VERSION}.jar /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS_VERSION}.jar COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS_VERSION}-src.tar.gz /stackable @@ -105,9 +106,11 @@ microdnf update # tar is required for `kubectl cp` which can be used to copy the log files # or profiler flamegraph from the Pod # It is already installed in the base image but leaving here for documentation purposes +# libstdc++ is a runtime dependency for boost microdnf install \ fuse \ fuse-libs \ + libstdc++ \ tar microdnf clean all rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt @@ -132,8 +135,13 @@ chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar # Set correct permissions and ownerships -chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" -chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" +chown --recursive ${STACKABLE_USER_UID}:0 \ +/stackable/hadoop /stackable/jmx /stackable/async-profiler \ +"/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" \ +/stackable/boost +chmod --recursive g=u /stackable/jmx /stackable/async-profiler \ +"/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" \ +/stackable/boost # Workaround for https://issues.apache.org/jira/browse/HADOOP-12845 # The problem is that our stackable-devel image does contain the openssl-devel package @@ -162,7 +170,7 @@ EOF USER ${STACKABLE_USER_UID} ENV HOME=/stackable -ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server +ENV LD_LIBRARY_PATH=/stackable/boost/lib:/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server ENV PATH="${PATH}":/stackable/hadoop/bin ENV HADOOP_HOME=/stackable/hadoop ENV HADOOP_CONF_DIR=/stackable/config @@ -174,6 +182,8 @@ ENV ASYNC_PROFILER_HOME=/stackable/async-profiler # if HADOOP_YARN_HOME does not exist at all, so we set it here to a sensible default. ENV HADOOP_YARN_HOME=/stackable/hadoop ENV HADOOP_MAPRED_HOME=/stackable/hadoop +ENV BOOST_ROOT=/stackable/boost +ENV CPATH=/stackable/boost/include WORKDIR /stackable/hadoop CMD ["echo", "This image is not meant to be 'run' directly."] diff --git a/hadoop/boost/Dockerfile b/hadoop/boost/Dockerfile new file mode 100644 index 000000000..80f1b27dd --- /dev/null +++ b/hadoop/boost/Dockerfile @@ -0,0 +1,40 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# check=error=true + +# The boost library is a runtime dependency of the Hadoop native code +# but is not available in the RedHat UBI10 or EPEL10 repositories, so we need to build +# it ourselves. +# +# NOTE: We use a published source bundle instead of the patchable workflow +# because boost uses git submodules for it's build system +# and patchable doesn't support these. +# The source bundle contains everything needed to build b2 (the build system) +# and boost except for dependencies which are automatically discovered. +# +FROM local-image/stackable-base AS boost-builder + +ARG BOOST_VERSION + +WORKDIR /tmp + +RUN < Date: Mon, 15 Jun 2026 16:58:46 +0200 Subject: [PATCH 03/13] update image hash and try out ssl 3 fix for 3.5.0 --- ...lude-Engine-API-when-building-with-O.patch | 95 +++++++++++++++++++ stackable-base/Dockerfile | 6 +- stackable-devel/Dockerfile | 6 +- 3 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 hadoop/hadoop/stackable/patches/3.5.0/0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch diff --git a/hadoop/hadoop/stackable/patches/3.5.0/0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.5.0/0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch new file mode 100644 index 000000000..2a1d90134 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.5.0/0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -0,0 +1,95 @@ +From 417cf6efdf1950da52867f883d361435f488d811 Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Mon, 15 Jun 2026 15:59:39 +0200 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x. + +Source https://github.com/apache/hadoop/pull/8550/ +--- + .../hadoop/crypto/random/OpensslSecureRandom.c | 17 +++++++++++++++-- + 1 file changed, 15 insertions(+), 2 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b..047e1bb402 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); + static int (*dlsym_ENGINE_finish) (ENGINE *); + static int (*dlsym_ENGINE_free) (ENGINE *); ++#endif + static int (*dlsym_RAND_bytes) (unsigned char *, int); + static unsigned long (*dlsym_ERR_get_error) (void); + #endif +@@ -126,12 +128,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + openssl, "ENGINE_load_rdrand"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup"); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \ + openssl, "ENGINE_set_default"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); + LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); + #endif +@@ -239,7 +243,7 @@ static void windows_locking_callback(int mode, int type, char *file, int line) + } + #endif /* WINDOWS */ + +-#ifdef UNIX ++#if defined(UNIX) && OPENSSL_VERSION_NUMBER < 0x10100000L + static void pthreads_locking_callback(int mode, int type, char *file, int line); + static unsigned long pthreads_thread_id(void); + static pthread_mutex_t *lock_cs; +@@ -300,7 +304,7 @@ static unsigned long pthreads_thread_id(void) + return thread_id; + } + +-#endif /* UNIX */ ++#endif /* UNIX && OpenSSL < 1.1.0 */ + + /** + * If using an Intel chipset with RDRAND, the high-performance hardware +@@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) + */ + static ENGINE * openssl_rand_init(void) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -339,10 +344,14 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; ++#else ++ return NULL; ++#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -353,6 +362,10 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif ++#else ++ UNUSED(eng); ++ UNUSED(clean_locks); ++#endif + } + + static int openssl_rand_bytes(unsigned char *buf, int num) diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index 05a2e4268..2619fb42c 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -43,11 +43,11 @@ EOF # # The website is awful/broken, so you can use one of the following tools to find it: # -# docker buildx imagetools inspect registry.redhat.io/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' +# docker buildx imagetools inspect registry.access.redhat.com/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' # # or -# skopeo inspect --format '{{.Digest}}' docker://registry.redhat.io/ubi10/ubi-minimal +# skopeo inspect --format '{{.Digest}}' docker://registry.access.redhat.com/ubi10/ubi-minimal # -FROM registry.redhat.io/ubi10/ubi-minimal@sha256:8935d83eadb5c9f621c13f4dfab7e1bbd8d948317dc67157279911ede1013112 +FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:76c113359a458e3f04057762b5bd4a9837a6987520434dea158c728280116713 # intentionally unused ARG PRODUCT_VERSION diff --git a/stackable-devel/Dockerfile b/stackable-devel/Dockerfile index fd3c4a6b5..d7ec5d583 100644 --- a/stackable-devel/Dockerfile +++ b/stackable-devel/Dockerfile @@ -16,11 +16,11 @@ # # The website is awful/broken, so you can use one of the following tools to find it: # -# docker buildx imagetools inspect registry.redhat.io/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' +# docker buildx imagetools inspect registry.access.redhat.com/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' # # or -# skopeo inspect --format '{{.Digest}}' docker://registry.redhat.io/ubi10/ubi-minimal +# skopeo inspect --format '{{.Digest}}' docker://registry.access.redhat.com/ubi10/ubi-minimal # -FROM registry.redhat.io/ubi10/ubi-minimal@sha256:8935d83eadb5c9f621c13f4dfab7e1bbd8d948317dc67157279911ede1013112 +FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:76c113359a458e3f04057762b5bd4a9837a6987520434dea158c728280116713 # intentionally unused ARG PRODUCT_VERSION From 8da24644aae2a2e6724e34ebfdd8d40d9f7e0799 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:28:25 +0200 Subject: [PATCH 04/13] add openssl fix for ubi 10 to 3.4.x versions --- ...lude-Engine-API-when-building-with-O.patch | 95 +++++++++++++++++++ ...lude-Engine-API-when-building-with-O.patch | 95 +++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 hadoop/hadoop/stackable/patches/3.4.2/0010-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch create mode 100644 hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch diff --git a/hadoop/hadoop/stackable/patches/3.4.2/0010-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.4.2/0010-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch new file mode 100644 index 000000000..5f5f75573 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.2/0010-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -0,0 +1,95 @@ +From fb632e667004fd707a814cb6415c953a7478dd26 Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Mon, 15 Jun 2026 15:59:39 +0200 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x. + +Source https://github.com/apache/hadoop/pull/8550/ +--- + .../hadoop/crypto/random/OpensslSecureRandom.c | 17 +++++++++++++++-- + 1 file changed, 15 insertions(+), 2 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b..047e1bb402 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); + static int (*dlsym_ENGINE_finish) (ENGINE *); + static int (*dlsym_ENGINE_free) (ENGINE *); ++#endif + static int (*dlsym_RAND_bytes) (unsigned char *, int); + static unsigned long (*dlsym_ERR_get_error) (void); + #endif +@@ -126,12 +128,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + openssl, "ENGINE_load_rdrand"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup"); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \ + openssl, "ENGINE_set_default"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); + LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); + #endif +@@ -239,7 +243,7 @@ static void windows_locking_callback(int mode, int type, char *file, int line) + } + #endif /* WINDOWS */ + +-#ifdef UNIX ++#if defined(UNIX) && OPENSSL_VERSION_NUMBER < 0x10100000L + static void pthreads_locking_callback(int mode, int type, char *file, int line); + static unsigned long pthreads_thread_id(void); + static pthread_mutex_t *lock_cs; +@@ -300,7 +304,7 @@ static unsigned long pthreads_thread_id(void) + return thread_id; + } + +-#endif /* UNIX */ ++#endif /* UNIX && OpenSSL < 1.1.0 */ + + /** + * If using an Intel chipset with RDRAND, the high-performance hardware +@@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) + */ + static ENGINE * openssl_rand_init(void) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -339,10 +344,14 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; ++#else ++ return NULL; ++#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -353,6 +362,10 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif ++#else ++ UNUSED(eng); ++ UNUSED(clean_locks); ++#endif + } + + static int openssl_rand_bytes(unsigned char *buf, int num) diff --git a/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch new file mode 100644 index 000000000..b6e258bd9 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -0,0 +1,95 @@ +From e224e0e240a6223ce1fe706a18438cde1e811886 Mon Sep 17 00:00:00 2001 +From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> +Date: Mon, 15 Jun 2026 15:59:39 +0200 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x. + +Source https://github.com/apache/hadoop/pull/8550/ +--- + .../hadoop/crypto/random/OpensslSecureRandom.c | 17 +++++++++++++++-- + 1 file changed, 15 insertions(+), 2 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b..047e1bb402 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); + static int (*dlsym_ENGINE_finish) (ENGINE *); + static int (*dlsym_ENGINE_free) (ENGINE *); ++#endif + static int (*dlsym_RAND_bytes) (unsigned char *, int); + static unsigned long (*dlsym_ERR_get_error) (void); + #endif +@@ -126,12 +128,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + openssl, "ENGINE_load_rdrand"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup"); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \ + openssl, "ENGINE_set_default"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); + LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); + #endif +@@ -239,7 +243,7 @@ static void windows_locking_callback(int mode, int type, char *file, int line) + } + #endif /* WINDOWS */ + +-#ifdef UNIX ++#if defined(UNIX) && OPENSSL_VERSION_NUMBER < 0x10100000L + static void pthreads_locking_callback(int mode, int type, char *file, int line); + static unsigned long pthreads_thread_id(void); + static pthread_mutex_t *lock_cs; +@@ -300,7 +304,7 @@ static unsigned long pthreads_thread_id(void) + return thread_id; + } + +-#endif /* UNIX */ ++#endif /* UNIX && OpenSSL < 1.1.0 */ + + /** + * If using an Intel chipset with RDRAND, the high-performance hardware +@@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) + */ + static ENGINE * openssl_rand_init(void) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -339,10 +344,14 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; ++#else ++ return NULL; ++#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -353,6 +362,10 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif ++#else ++ UNUSED(eng); ++ UNUSED(clean_locks); ++#endif + } + + static int openssl_rand_bytes(unsigned char *buf, int num) From 2f247031116f1b75f268e7ef195c8217b1af87bd Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 16 Jun 2026 12:59:29 +0200 Subject: [PATCH 05/13] remove hadoop/boost --- hadoop/boost/Dockerfile | 40 ----------------------------------- hadoop/boost/boil-config.toml | 5 ----- 2 files changed, 45 deletions(-) delete mode 100644 hadoop/boost/Dockerfile delete mode 100644 hadoop/boost/boil-config.toml diff --git a/hadoop/boost/Dockerfile b/hadoop/boost/Dockerfile deleted file mode 100644 index 80f1b27dd..000000000 --- a/hadoop/boost/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 -# check=error=true - -# The boost library is a runtime dependency of the Hadoop native code -# but is not available in the RedHat UBI10 or EPEL10 repositories, so we need to build -# it ourselves. -# -# NOTE: We use a published source bundle instead of the patchable workflow -# because boost uses git submodules for it's build system -# and patchable doesn't support these. -# The source bundle contains everything needed to build b2 (the build system) -# and boost except for dependencies which are automatically discovered. -# -FROM local-image/stackable-base AS boost-builder - -ARG BOOST_VERSION - -WORKDIR /tmp - -RUN < Date: Tue, 16 Jun 2026 13:03:29 +0200 Subject: [PATCH 06/13] merge hadoop Dockerfiles from maim --- hadoop/Dockerfile | 16 +++------------- hadoop/hadoop/Dockerfile | 3 +++ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index f5ebf2b14..2757b4e8c 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -90,7 +90,6 @@ LABEL \ COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION} COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/*-src.tar.gz /stackable -COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/boost /stackable/boost COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS_VERSION}.jar /stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS_VERSION}.jar COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hdfs-utils-${HDFS_UTILS_VERSION}-src.tar.gz /stackable @@ -106,11 +105,9 @@ microdnf update # tar is required for `kubectl cp` which can be used to copy the log files # or profiler flamegraph from the Pod # It is already installed in the base image but leaving here for documentation purposes -# libstdc++ is a runtime dependency for boost microdnf install \ fuse \ fuse-libs \ - libstdc++ \ tar microdnf clean all rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt @@ -135,13 +132,8 @@ chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER_VERSION}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar # Set correct permissions and ownerships -chown --recursive ${STACKABLE_USER_UID}:0 \ -/stackable/hadoop /stackable/jmx /stackable/async-profiler \ -"/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" \ -/stackable/boost -chmod --recursive g=u /stackable/jmx /stackable/async-profiler \ -"/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" \ -/stackable/boost +chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER_VERSION}-${TARGETOS}-${ARCH}" +chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE_VERSION}" # Workaround for https://issues.apache.org/jira/browse/HADOOP-12845 # The problem is that our stackable-devel image does contain the openssl-devel package @@ -170,7 +162,7 @@ EOF USER ${STACKABLE_USER_UID} ENV HOME=/stackable -ENV LD_LIBRARY_PATH=/stackable/boost/lib:/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server +ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server ENV PATH="${PATH}":/stackable/hadoop/bin ENV HADOOP_HOME=/stackable/hadoop ENV HADOOP_CONF_DIR=/stackable/config @@ -182,8 +174,6 @@ ENV ASYNC_PROFILER_HOME=/stackable/async-profiler # if HADOOP_YARN_HOME does not exist at all, so we set it here to a sensible default. ENV HADOOP_YARN_HOME=/stackable/hadoop ENV HADOOP_MAPRED_HOME=/stackable/hadoop -ENV BOOST_ROOT=/stackable/boost -ENV CPATH=/stackable/boost/include WORKDIR /stackable/hadoop CMD ["echo", "This image is not meant to be 'run' directly."] diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile index fe6429eee..b8387410e 100644 --- a/hadoop/hadoop/Dockerfile +++ b/hadoop/hadoop/Dockerfile @@ -10,6 +10,7 @@ ARG PROTOBUF_VERSION ARG AWS_JAVA_SDK_BUNDLE_VERSION ARG AZURE_STORAGE_VERSION ARG AZURE_KEYVAULT_CORE_VERSION +ARG ANALYTICSACCELERATOR_S3_VERSION ARG STACKABLE_USER_UID WORKDIR /stackable @@ -166,6 +167,8 @@ else cp /stackable/hadoop/share/hadoop/common/lib/hadoop-aws-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ # Starting with Hadoop 3.4.3 the aws-java-sdk-bundle is not included any more. curl -o /stackable/hadoop-cloud-libraries/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE_VERSION}.jar" + # Starting with Hadoop 3.4.3 the S3A InputStreamType factory references analyticsaccelerator-s3 (software.amazon.s3.analyticsaccelerator.request.ObjectClient) during S3AFileSystem.initialize. + curl -o /stackable/hadoop-cloud-libraries/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar --fail "https://repo.stackable.tech/repository/packages/aws/analyticsaccelerator-s3-${ANALYTICSACCELERATOR_S3_VERSION}.jar" cp /stackable/hadoop/share/hadoop/common/lib/hadoop-azure-${NEW_VERSION}.jar /stackable/hadoop-cloud-libraries/ cp /stackable/hadoop/share/hadoop/common/lib/azure-storage-${AZURE_STORAGE_VERSION}.jar /stackable/hadoop-cloud-libraries/ cp /stackable/hadoop/share/hadoop/common/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE_VERSION}.jar /stackable/hadoop-cloud-libraries/ From c66329b5d35a18ceb184ee6733430f9249247247 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 16 Jun 2026 17:59:27 +0200 Subject: [PATCH 07/13] install gzip --- hadoop/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 2757b4e8c..faf918905 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -105,9 +105,11 @@ microdnf update # tar is required for `kubectl cp` which can be used to copy the log files # or profiler flamegraph from the Pod # It is already installed in the base image but leaving here for documentation purposes +# TODO: gzip is needed by rpm and it is not included in UBI 10. Probably needs to go higher up the image chain. microdnf install \ fuse \ fuse-libs \ + gzip \ tar microdnf clean all rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE_VERSION}\n" | sort > /stackable/package_manifest.txt From abbb5a230392eec4fdc7fdbef1832c13350836f5 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 29 Oct 2026 13:01:03 +0100 Subject: [PATCH 08/13] fix: Bump the UBI 10 base image and install gzip in java-base The hadoop final stage unpacks async-profiler with `curl | tar -xz`, which needs gzip. On UBI 9 every image built on java-base had it, because krb5-workstation pulls in pam, which pulls in cracklib, which requires gzip. On UBI 10 krb5-workstation no longer depends on pam, so nothing installs gzip. It goes into java-base so that the Java images keep what they had on UBI 9, and images that never had it do not gain it. --- hadoop/Dockerfile | 2 -- java-base/Dockerfile | 3 +++ stackable-base/Dockerfile | 4 ++-- stackable-devel/Dockerfile | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 1add8f8ca..a24dfa7a6 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -105,11 +105,9 @@ microdnf update # tar is required for `kubectl cp` which can be used to copy the log files # or profiler flamegraph from the Pod # It is already installed in the base image but leaving here for documentation purposes -# TODO: gzip is needed by rpm and it is not included in UBI 10. Probably needs to go higher up the image chain. microdnf install \ fuse \ fuse-libs \ - gzip \ tar microdnf clean all # List all rpm packages installed in the system diff --git a/java-base/Dockerfile b/java-base/Dockerfile index 4f5eaa54e..d0c80fb3a 100644 --- a/java-base/Dockerfile +++ b/java-base/Dockerfile @@ -40,6 +40,9 @@ RUN microdnf update && \ tzdata-java \ # Most of the Java tools needs at least "klist" and "kinit" for Kerberos integration krb5-workstation \ + # Needed by `tar -z`, e.g. when hadoop unpacks async-profiler. UBI 9 pulled it in through krb5-workstation, UBI 10 does not. + # Added 2026-10 as gzip-1.13-4.el10_2, which includes the fixes for CVE-2026-41991 and CVE-2026-41992 (RHSA-2026:61625). + gzip \ --nodocs && \ microdnf clean all diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index 1d71ad3c9..9a3c8849a 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -33,7 +33,7 @@ EOF # # docker buildx imagetools inspect registry.access.redhat.com/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' # -FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:76c113359a458e3f04057762b5bd4a9837a6987520434dea158c728280116713 AS builder +FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:ab67cd02caf8706e34a695a1432ebe965934f6128ec352a6c3f4048b43a4fb5e AS builder # intentionally unused ARG PRODUCT_VERSION @@ -150,7 +150,7 @@ chown ${STACKABLE_USER_UID}:0 /stackable/.curlrc # CVE-2023-37920: Remove "e-Tugra" root certificates # e-Tugra's root certificates were subject to an investigation prompted by reporting of security issues in their systems # Until they are removed by default from ca-certificates, we should remove them manually -EXPECTED_CERTS_PACKAGE="ca-certificates-2025.2.80_v9.0.305-102.el10_1.noarch" +EXPECTED_CERTS_PACKAGE="ca-certificates-2026.2.92_v9.0.319-100.3.el10_2.noarch" ACTUAL_CERTS_PACKAGE="$(rpm -qa ca-certificates)" if [ "$ACTUAL_CERTS_PACKAGE" != "$EXPECTED_CERTS_PACKAGE" ]; then echo "The ca-certificates package was updated to $ACTUAL_CERTS_PACKAGE. Please check if the e-Tugra root certificates are present. \ diff --git a/stackable-devel/Dockerfile b/stackable-devel/Dockerfile index dc3053a1e..c589b0eac 100644 --- a/stackable-devel/Dockerfile +++ b/stackable-devel/Dockerfile @@ -12,7 +12,7 @@ # # docker buildx imagetools inspect registry.access.redhat.com/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' # -FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:76c113359a458e3f04057762b5bd4a9837a6987520434dea158c728280116713 AS builder +FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:ab67cd02caf8706e34a695a1432ebe965934f6128ec352a6c3f4048b43a4fb5e AS builder # intentionally unused ARG PRODUCT_VERSION From 56ef78eac8db4f6c24477d9d1d935a987b997719 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 29 Oct 2026 13:06:40 +0100 Subject: [PATCH 09/13] fix(hadoop): Guard every OpenSSL ENGINE call in the native code apache/hadoop#8550 alone leaves OpensslCipher resolving ENGINE_by_id and ENGINE_free in initIDs. RHEL 10's headers define OPENSSL_NO_ENGINE, but its libcrypto still exports both symbols, so the lookup succeeds there by accident. Against a libcrypto without them, initIDs throws UnsatisfiedLinkError and the AES-CTR codec falls back to JCE with only a warning. The two follow-up patches guard the remaining ENGINE paths in OpensslSecureRandom and OpensslCipher, and fix the misspelt ENGINE_by_free on Windows. --- ...lude-Engine-API-when-building-with-O.patch | 42 ++----- ...ard-OpenSSL-ENGINE-code-paths-as-a-w.patch | 112 ++++++++++++++++++ ...ard-the-cipher-s-ENGINE-lookups-with.patch | 110 +++++++++++++++++ ...ude-Engine-API-when-building-with-O.patch} | 42 ++----- ...ard-OpenSSL-ENGINE-code-paths-as-a-w.patch | 112 ++++++++++++++++++ ...ard-the-cipher-s-ENGINE-lookups-with.patch | 110 +++++++++++++++++ 6 files changed, 462 insertions(+), 66 deletions(-) create mode 100644 hadoop/hadoop/stackable/patches/3.4.3/0010-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch create mode 100644 hadoop/hadoop/stackable/patches/3.4.3/0011-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch rename hadoop/hadoop/stackable/patches/3.5.0/{0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch => 0008-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch} (68%) create mode 100644 hadoop/hadoop/stackable/patches/3.5.0/0009-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch create mode 100644 hadoop/hadoop/stackable/patches/3.5.0/0010-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch diff --git a/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch index b6e258bd9..4eb406d54 100644 --- a/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch +++ b/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -1,15 +1,14 @@ -From e224e0e240a6223ce1fe706a18438cde1e811886 Mon Sep 17 00:00:00 2001 -From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Mon, 15 Jun 2026 15:59:39 +0200 -Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x. +From 62b70efc20e8a619b73f553ca78ef9b85b025240 Mon Sep 17 00:00:00 2001 +From: Rohit Kumar +Date: Mon, 15 Jun 2026 15:54:13 +0530 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x -Source https://github.com/apache/hadoop/pull/8550/ --- - .../hadoop/crypto/random/OpensslSecureRandom.c | 17 +++++++++++++++-- - 1 file changed, 15 insertions(+), 2 deletions(-) + .../org/apache/hadoop/crypto/random/OpensslSecureRandom.c | 8 ++++++++ + 1 file changed, 8 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c -index 3f141be05b..047e1bb402 100644 +index 3f141be05b54930e45b092671d02e33396cd4a98..d4351c1a5d70da0a73cfb64f2937b4bd04dcd541 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c @@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); @@ -41,24 +40,6 @@ index 3f141be05b..047e1bb402 100644 LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); #endif -@@ -239,7 +243,7 @@ static void windows_locking_callback(int mode, int type, char *file, int line) - } - #endif /* WINDOWS */ - --#ifdef UNIX -+#if defined(UNIX) && OPENSSL_VERSION_NUMBER < 0x10100000L - static void pthreads_locking_callback(int mode, int type, char *file, int line); - static unsigned long pthreads_thread_id(void); - static pthread_mutex_t *lock_cs; -@@ -300,7 +304,7 @@ static unsigned long pthreads_thread_id(void) - return thread_id; - } - --#endif /* UNIX */ -+#endif /* UNIX && OpenSSL < 1.1.0 */ - - /** - * If using an Intel chipset with RDRAND, the high-performance hardware @@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) */ static ENGINE * openssl_rand_init(void) @@ -67,12 +48,10 @@ index 3f141be05b..047e1bb402 100644 #if OPENSSL_VERSION_NUMBER < 0x10100000L locks_setup(); -@@ -339,10 +344,14 @@ static ENGINE * openssl_rand_init(void) +@@ -339,10 +344,12 @@ static ENGINE * openssl_rand_init(void) } return eng; -+#else -+ return NULL; +#endif } @@ -82,13 +61,10 @@ index 3f141be05b..047e1bb402 100644 if (NULL != eng) { dlsym_ENGINE_finish(eng); dlsym_ENGINE_free(eng); -@@ -353,6 +362,10 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) +@@ -353,6 +360,7 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) locks_cleanup(); } #endif -+#else -+ UNUSED(eng); -+ UNUSED(clean_locks); +#endif } diff --git a/hadoop/hadoop/stackable/patches/3.4.3/0010-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch b/hadoop/hadoop/stackable/patches/3.4.3/0010-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch new file mode 100644 index 000000000..7d92815cb --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.3/0010-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch @@ -0,0 +1,112 @@ +From b436664681bb663d08878ab38bba5e251d016eda Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Tue, 27 Oct 2026 16:12:48 +0100 +Subject: HADOOP-19872. Guard OpenSSL ENGINE code paths as a whole and update + RNG docs + +--- + .../crypto/random/OpensslSecureRandom.java | 9 ++++----- + .../hadoop/crypto/random/OpensslSecureRandom.c | 16 ++++++++++------ + .../random/org_apache_hadoop_crypto_random.h | 2 ++ + 3 files changed, 16 insertions(+), 11 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +index b9f569d0c90780fee218258c0d1c34aa276d16d7..2d30b57e9d55893efb18b1bf7e3ed1b8dfdf48ca 100644 +--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java ++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +@@ -32,11 +32,10 @@ import org.slf4j.LoggerFactory; + * This implementation is thread-safe. + *

+ * +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used and it's much faster than +- * {@link java.security.SecureRandom}. If RDRAND is unavailable, default +- * OpenSSL secure random generator will be used. It's still faster +- * and can generate strong random bytes. ++ * When built against OpenSSL older than 3.0 on a CPU with RDRAND, the ++ * hardware random number generator is used through OpenSSL's RDRAND engine. ++ * Otherwise, including every build against OpenSSL 3.0 or later, OpenSSL's ++ * default secure random generator is used. + *

+ * See https://wiki.openssl.org/index.php/Random_Numbers + * See http://en.wikipedia.org/wiki/RdRand +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index d4351c1a5d70da0a73cfb64f2937b4bd04dcd541..8fb88365f715ce95195c2596f21953177fab3219 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -90,8 +90,10 @@ static __dlsym_RAND_bytes dlsym_RAND_bytes; + static __dlsym_ERR_get_error dlsym_ERR_get_error; + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void); + static void openssl_rand_clean(ENGINE *eng, int clean_locks); ++#endif + static int openssl_rand_bytes(unsigned char *buf, int num); + + JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR +@@ -170,7 +172,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + env, openssl, "ERR_get_error"); + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + openssl_rand_init(); ++#endif + } + + JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B +@@ -307,12 +311,14 @@ static unsigned long pthreads_thread_id(void) + #endif /* UNIX */ + + /** +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used. ++ * Before OpenSSL 3.0, the RDRAND engine is made the default RAND method if ++ * the CPU supports it. OpenSSL 3.0 deprecates the ENGINE API and some ++ * distributions build without it, so from 3.0 on RAND_bytes uses OpenSSL's ++ * default DRBG. + */ ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -344,12 +350,10 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; +-#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -360,8 +364,8 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif +-#endif + } ++#endif + + static int openssl_rand_bytes(unsigned char *buf, int num) + { +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +index 1200718e69f301408e4aae92ef4e331f82a70fb4..8e2eb94e6149f4b26fb00cde7ead5dc609859051 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +@@ -33,7 +33,9 @@ + #define UNUSED(x) ((void)(x)) + + #include ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #include ++#endif + #include + #include + diff --git a/hadoop/hadoop/stackable/patches/3.4.3/0011-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch b/hadoop/hadoop/stackable/patches/3.4.3/0011-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch new file mode 100644 index 000000000..6348d394a --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.3/0011-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch @@ -0,0 +1,110 @@ +From c53b426246d05cefb425d1ddf1302508ea80f3a8 Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Thu, 29 Oct 2026 11:34:54 +0100 +Subject: HADOOP-19872. Guard the cipher's ENGINE lookups with + OPENSSL_NO_ENGINE + +OpensslCipher still resolved ENGINE_by_id and ENGINE_free in initIDs. +Against a libcrypto that exports neither, initIDs throws +UnsatisfiedLinkError and the AES-CTR codec falls back to JCE with only a +warning. It loads on RHEL 10 only because that libcrypto still exports +the ENGINE symbols although its headers define OPENSSL_NO_ENGINE. + +The guard is OPENSSL_NO_ENGINE rather than the OpenSSL version, because +OpenSSL 3 builds that keep ENGINE still honour +hadoop.security.openssl.engine.id. + +Also fix the Windows lookup of ENGINE_free, which asked for +"ENGINE_by_free". + +Generated-by: Claude (Anthropic) +--- + .../org/apache/hadoop/crypto/OpensslCipher.c | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index 33be4a394f467cbffefba13d74d3cd8da5c7bb78..0c000f39050886eb8c2eedba850db2e3dc944b95 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -101,9 +101,11 @@ static EVP_CIPHER * (*dlsym_EVP_aes_128_ctr)(void); + static EVP_CIPHER * (*dlsym_EVP_sm4_ctr)(void); + static int (*dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); ++#ifndef OPENSSL_NO_ENGINE + static ENGINE * (*dlsym_ENGINE_by_id)(const char *id); + static int (*dlsym_ENGINE_free)(ENGINE *); + #endif ++#endif + static void *openssl; + #endif + +@@ -146,14 +148,16 @@ static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr; + typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_sm4_ctr)(void); + typedef int (__cdecl *__dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); +-typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); +-typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + + static __dlsym_EVP_sm4_ctr dlsym_EVP_sm4_ctr; + static __dlsym_OPENSSL_init_crypto dlsym_OPENSSL_init_crypto; ++#ifndef OPENSSL_NO_ENGINE ++typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); ++typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + static __dlsym_ENGINE_by_id dlsym_ENGINE_by_id; + static __dlsym_ENGINE_free dlsym_ENGINE_free; + #endif ++#endif + static HMODULE openssl; + #endif + +@@ -244,12 +248,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(dlsym_OPENSSL_init_crypto, env, openssl, \ + "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, \ + "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, \ + "ENGINE_free"); + #endif + #endif ++#endif + + #ifdef WINDOWS + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_new, dlsym_EVP_CIPHER_CTX_new, \ +@@ -284,10 +290,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(__dlsym_OPENSSL_init_crypto, dlsym_OPENSSL_init_crypto, \ + env, openssl, "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_by_id, dlsym_ENGINE_by_id, \ + env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_free, dlsym_ENGINE_free, \ +- env, openssl, "ENGINE_by_free"); ++ env, openssl, "ENGINE_free"); ++#endif + #endif + #endif + +@@ -362,7 +370,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initEngine + { + ENGINE *e = NULL; + +-#if OPENSSL_VERSION_NUMBER >= 0x10101001L ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + if (engineId != NULL) { + const char *id = (*env)->GetStringUTFChars(env, engineId, NULL); + if (id != NULL) { +@@ -574,10 +582,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_clean + dlsym_EVP_CIPHER_CTX_free(context); + } + ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + ENGINE *e = LONG_TO_ENGINE(engine); + if (e) { + dlsym_ENGINE_free(e); + } ++#endif + } + + JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibraryName diff --git a/hadoop/hadoop/stackable/patches/3.5.0/0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.5.0/0008-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch similarity index 68% rename from hadoop/hadoop/stackable/patches/3.5.0/0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch rename to hadoop/hadoop/stackable/patches/3.5.0/0008-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch index 2a1d90134..136ec54b1 100644 --- a/hadoop/hadoop/stackable/patches/3.5.0/0007-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch +++ b/hadoop/hadoop/stackable/patches/3.5.0/0008-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -1,15 +1,14 @@ -From 417cf6efdf1950da52867f883d361435f488d811 Mon Sep 17 00:00:00 2001 -From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> -Date: Mon, 15 Jun 2026 15:59:39 +0200 -Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x. +From 897b1bc5a97f92cca6ad787364ffaf45d6dadee6 Mon Sep 17 00:00:00 2001 +From: Rohit Kumar +Date: Mon, 15 Jun 2026 15:54:13 +0530 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x -Source https://github.com/apache/hadoop/pull/8550/ --- - .../hadoop/crypto/random/OpensslSecureRandom.c | 17 +++++++++++++++-- - 1 file changed, 15 insertions(+), 2 deletions(-) + .../org/apache/hadoop/crypto/random/OpensslSecureRandom.c | 8 ++++++++ + 1 file changed, 8 insertions(+) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c -index 3f141be05b..047e1bb402 100644 +index 3f141be05b54930e45b092671d02e33396cd4a98..d4351c1a5d70da0a73cfb64f2937b4bd04dcd541 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c @@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); @@ -41,24 +40,6 @@ index 3f141be05b..047e1bb402 100644 LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); #endif -@@ -239,7 +243,7 @@ static void windows_locking_callback(int mode, int type, char *file, int line) - } - #endif /* WINDOWS */ - --#ifdef UNIX -+#if defined(UNIX) && OPENSSL_VERSION_NUMBER < 0x10100000L - static void pthreads_locking_callback(int mode, int type, char *file, int line); - static unsigned long pthreads_thread_id(void); - static pthread_mutex_t *lock_cs; -@@ -300,7 +304,7 @@ static unsigned long pthreads_thread_id(void) - return thread_id; - } - --#endif /* UNIX */ -+#endif /* UNIX && OpenSSL < 1.1.0 */ - - /** - * If using an Intel chipset with RDRAND, the high-performance hardware @@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) */ static ENGINE * openssl_rand_init(void) @@ -67,12 +48,10 @@ index 3f141be05b..047e1bb402 100644 #if OPENSSL_VERSION_NUMBER < 0x10100000L locks_setup(); -@@ -339,10 +344,14 @@ static ENGINE * openssl_rand_init(void) +@@ -339,10 +344,12 @@ static ENGINE * openssl_rand_init(void) } return eng; -+#else -+ return NULL; +#endif } @@ -82,13 +61,10 @@ index 3f141be05b..047e1bb402 100644 if (NULL != eng) { dlsym_ENGINE_finish(eng); dlsym_ENGINE_free(eng); -@@ -353,6 +362,10 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) +@@ -353,6 +360,7 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) locks_cleanup(); } #endif -+#else -+ UNUSED(eng); -+ UNUSED(clean_locks); +#endif } diff --git a/hadoop/hadoop/stackable/patches/3.5.0/0009-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch b/hadoop/hadoop/stackable/patches/3.5.0/0009-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch new file mode 100644 index 000000000..f1b992d8e --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.5.0/0009-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch @@ -0,0 +1,112 @@ +From 6099a11bc5ccd6f8cd2be4eccb3d3736e41d72db Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Tue, 27 Oct 2026 16:12:48 +0100 +Subject: HADOOP-19872. Guard OpenSSL ENGINE code paths as a whole and update + RNG docs + +--- + .../crypto/random/OpensslSecureRandom.java | 9 ++++----- + .../hadoop/crypto/random/OpensslSecureRandom.c | 16 ++++++++++------ + .../random/org_apache_hadoop_crypto_random.h | 2 ++ + 3 files changed, 16 insertions(+), 11 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +index b9f569d0c90780fee218258c0d1c34aa276d16d7..2d30b57e9d55893efb18b1bf7e3ed1b8dfdf48ca 100644 +--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java ++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +@@ -32,11 +32,10 @@ import org.slf4j.LoggerFactory; + * This implementation is thread-safe. + *

+ * +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used and it's much faster than +- * {@link java.security.SecureRandom}. If RDRAND is unavailable, default +- * OpenSSL secure random generator will be used. It's still faster +- * and can generate strong random bytes. ++ * When built against OpenSSL older than 3.0 on a CPU with RDRAND, the ++ * hardware random number generator is used through OpenSSL's RDRAND engine. ++ * Otherwise, including every build against OpenSSL 3.0 or later, OpenSSL's ++ * default secure random generator is used. + *

+ * See https://wiki.openssl.org/index.php/Random_Numbers + * See http://en.wikipedia.org/wiki/RdRand +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index d4351c1a5d70da0a73cfb64f2937b4bd04dcd541..8fb88365f715ce95195c2596f21953177fab3219 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -90,8 +90,10 @@ static __dlsym_RAND_bytes dlsym_RAND_bytes; + static __dlsym_ERR_get_error dlsym_ERR_get_error; + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void); + static void openssl_rand_clean(ENGINE *eng, int clean_locks); ++#endif + static int openssl_rand_bytes(unsigned char *buf, int num); + + JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR +@@ -170,7 +172,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + env, openssl, "ERR_get_error"); + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + openssl_rand_init(); ++#endif + } + + JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B +@@ -307,12 +311,14 @@ static unsigned long pthreads_thread_id(void) + #endif /* UNIX */ + + /** +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used. ++ * Before OpenSSL 3.0, the RDRAND engine is made the default RAND method if ++ * the CPU supports it. OpenSSL 3.0 deprecates the ENGINE API and some ++ * distributions build without it, so from 3.0 on RAND_bytes uses OpenSSL's ++ * default DRBG. + */ ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -344,12 +350,10 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; +-#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -360,8 +364,8 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif +-#endif + } ++#endif + + static int openssl_rand_bytes(unsigned char *buf, int num) + { +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +index 1200718e69f301408e4aae92ef4e331f82a70fb4..8e2eb94e6149f4b26fb00cde7ead5dc609859051 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +@@ -33,7 +33,9 @@ + #define UNUSED(x) ((void)(x)) + + #include ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #include ++#endif + #include + #include + diff --git a/hadoop/hadoop/stackable/patches/3.5.0/0010-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch b/hadoop/hadoop/stackable/patches/3.5.0/0010-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch new file mode 100644 index 000000000..d19306e97 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.5.0/0010-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch @@ -0,0 +1,110 @@ +From 8a81b0b20632b47bdbbf45d9a91cc05b4115d94f Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Thu, 29 Oct 2026 11:34:54 +0100 +Subject: HADOOP-19872. Guard the cipher's ENGINE lookups with + OPENSSL_NO_ENGINE + +OpensslCipher still resolved ENGINE_by_id and ENGINE_free in initIDs. +Against a libcrypto that exports neither, initIDs throws +UnsatisfiedLinkError and the AES-CTR codec falls back to JCE with only a +warning. It loads on RHEL 10 only because that libcrypto still exports +the ENGINE symbols although its headers define OPENSSL_NO_ENGINE. + +The guard is OPENSSL_NO_ENGINE rather than the OpenSSL version, because +OpenSSL 3 builds that keep ENGINE still honour +hadoop.security.openssl.engine.id. + +Also fix the Windows lookup of ENGINE_free, which asked for +"ENGINE_by_free". + +Generated-by: Claude (Anthropic) +--- + .../org/apache/hadoop/crypto/OpensslCipher.c | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index 33be4a394f467cbffefba13d74d3cd8da5c7bb78..0c000f39050886eb8c2eedba850db2e3dc944b95 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -101,9 +101,11 @@ static EVP_CIPHER * (*dlsym_EVP_aes_128_ctr)(void); + static EVP_CIPHER * (*dlsym_EVP_sm4_ctr)(void); + static int (*dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); ++#ifndef OPENSSL_NO_ENGINE + static ENGINE * (*dlsym_ENGINE_by_id)(const char *id); + static int (*dlsym_ENGINE_free)(ENGINE *); + #endif ++#endif + static void *openssl; + #endif + +@@ -146,14 +148,16 @@ static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr; + typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_sm4_ctr)(void); + typedef int (__cdecl *__dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); +-typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); +-typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + + static __dlsym_EVP_sm4_ctr dlsym_EVP_sm4_ctr; + static __dlsym_OPENSSL_init_crypto dlsym_OPENSSL_init_crypto; ++#ifndef OPENSSL_NO_ENGINE ++typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); ++typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + static __dlsym_ENGINE_by_id dlsym_ENGINE_by_id; + static __dlsym_ENGINE_free dlsym_ENGINE_free; + #endif ++#endif + static HMODULE openssl; + #endif + +@@ -244,12 +248,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(dlsym_OPENSSL_init_crypto, env, openssl, \ + "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, \ + "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, \ + "ENGINE_free"); + #endif + #endif ++#endif + + #ifdef WINDOWS + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_new, dlsym_EVP_CIPHER_CTX_new, \ +@@ -284,10 +290,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(__dlsym_OPENSSL_init_crypto, dlsym_OPENSSL_init_crypto, \ + env, openssl, "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_by_id, dlsym_ENGINE_by_id, \ + env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_free, dlsym_ENGINE_free, \ +- env, openssl, "ENGINE_by_free"); ++ env, openssl, "ENGINE_free"); ++#endif + #endif + #endif + +@@ -362,7 +370,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initEngine + { + ENGINE *e = NULL; + +-#if OPENSSL_VERSION_NUMBER >= 0x10101001L ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + if (engineId != NULL) { + const char *id = (*env)->GetStringUTFChars(env, engineId, NULL); + if (id != NULL) { +@@ -574,10 +582,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_clean + dlsym_EVP_CIPHER_CTX_free(context); + } + ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + ENGINE *e = LONG_TO_ENGINE(engine); + if (e) { + dlsym_ENGINE_free(e); + } ++#endif + } + + JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibraryName From 81f2a8c0c94e47d629eea28ae666d646ba3d5d2f Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 29 Oct 2026 14:59:18 +0100 Subject: [PATCH 10/13] fix(hadoop): Backport HADOOP-19551 to 3.3.6 for GCC 14 GCC 14, which UBI 10 ships, turns implicit-function-declaration into an error by default. OpensslSecureRandom.c defines locks_setup and locks_cleanup outside the version guard that declares the dlsym_CRYPTO_* lock functions they call, so hadoop-common's native build fails on UBI 10. Only OpenSSL before 1.1 calls them. 3.4.3 and 3.5.0 already contain HADOOP-19551, which restores the old default with -Wno-error=implicit-function-declaration. --- ...x-compilation-error-of-native-librar.patch | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 hadoop/hadoop/stackable/patches/3.3.6/0014-HADOOP-19551.-Fix-compilation-error-of-native-librar.patch diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0014-HADOOP-19551.-Fix-compilation-error-of-native-librar.patch b/hadoop/hadoop/stackable/patches/3.3.6/0014-HADOOP-19551.-Fix-compilation-error-of-native-librar.patch new file mode 100644 index 000000000..e0dbaa481 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0014-HADOOP-19551.-Fix-compilation-error-of-native-librar.patch @@ -0,0 +1,26 @@ +From 26af6e62c84f24ce7d81c5c27f5fcae0a4edfd2b Mon Sep 17 00:00:00 2001 +From: Masatake Iwasaki +Date: Wed, 7 May 2025 15:01:01 +0900 +Subject: HADOOP-19551. Fix compilation error of native libraries on newer GCC. + (#7644) + +Reviewed-by: Chris Nauroth +(cherry picked from commit 7ea3f51287c54ac563ef19d42c5bc102699ea025) +--- + hadoop-common-project/hadoop-common/HadoopCommon.cmake | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/HadoopCommon.cmake b/hadoop-common-project/hadoop-common/HadoopCommon.cmake +index 4de70ac3f78e2de5128e1ac48b9c1a20f1f4d18b..ade701675ad79915d29258e4be9c23ca02be4ffa 100644 +--- a/hadoop-common-project/hadoop-common/HadoopCommon.cmake ++++ b/hadoop-common-project/hadoop-common/HadoopCommon.cmake +@@ -145,6 +145,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Make GNU extensions available. + hadoop_add_compiler_flags("-D_GNU_SOURCE") + ++ # using old default behavior on GCC >= 14.0 ++ hadoop_add_compiler_flags("-Wno-error=implicit-function-declaration") ++ + # If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit. + if(JVM_ARCH_DATA_MODEL EQUAL 32) + # Force 32-bit code generation on amd64/x86_64, ppc64, sparc64 From f300df9ade5c33ffafcb05ed29ba08ad2af959ee Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 29 Oct 2026 15:04:04 +0100 Subject: [PATCH 11/13] fix(hadoop): Backport HDFS-17226 to 3.3.6 for GCC 14 The libstdc++ headers of GCC 14, which UBI 10 ships, no longer pull in transitively, so the libhdfspp public headers that use uint16_t, uint32_t and uint64_t without including it fail to compile on UBI 10. 3.4.3 and 3.5.0 already contain HDFS-17226, which adds the include. --- ...ding-native-libraries-fails-on-Fedor.patch | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 hadoop/hadoop/stackable/patches/3.3.6/0015-HDFS-17226.-Building-native-libraries-fails-on-Fedor.patch diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0015-HDFS-17226.-Building-native-libraries-fails-on-Fedor.patch b/hadoop/hadoop/stackable/patches/3.3.6/0015-HDFS-17226.-Building-native-libraries-fails-on-Fedor.patch new file mode 100644 index 000000000..6a1c42ce5 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0015-HDFS-17226.-Building-native-libraries-fails-on-Fedor.patch @@ -0,0 +1,61 @@ +From 4e739baadfd68b94abd45fe9e2df1c8bfa2bc880 Mon Sep 17 00:00:00 2001 +From: Kengo Seki +Date: Wed, 23 Apr 2025 00:12:22 +0900 +Subject: HDFS-17226. Building native libraries fails on Fedora 38. (#6191) + +(cherry picked from commit a47df85360bfebbb7dfd658630b7c66d982a49cf) +--- + .../src/main/native/libhdfspp/include/hdfspp/content_summary.h | 1 + + .../src/main/native/libhdfspp/include/hdfspp/fsinfo.h | 1 + + .../src/main/native/libhdfspp/include/hdfspp/statinfo.h | 1 + + .../src/main/native/libhdfspp/include/hdfspp/uri.h | 1 + + 4 files changed, 4 insertions(+) + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h +index 6762a65ca908b513a8fdea0a117a52835af126b7..d9809bab59291b6aec68b72dcba798606b5ad3ee 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h +@@ -19,6 +19,7 @@ + #define HDFSPP_CONTENT_SUMMARY_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h +index 103cf133bec221e9450c38fbc33eec035d73e767..156ed445181a331760df3e245201df1ec84f5da5 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h +@@ -19,6 +19,7 @@ + #define HDFSPP_FSINFO_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h +index 1da1f18a592bdcc6173443b6418694fdf8c1c361..5924d1db005870c5beb26a881db9a30702039e2d 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h +@@ -19,6 +19,7 @@ + #define HDFSPP_STATINFO_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h +index bc3d8b96d3f66e7b8904d19f69a9006441f8f647..a7f735afb5dbc1c1bc7c80621d5cebc38ea95927 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + #include + + namespace hdfs From c533ed195d4d1c4bf7671ce202bd2fcf3429406b Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 29 Oct 2026 14:53:41 +0100 Subject: [PATCH 12/13] fix(hadoop): Guard the OpenSSL ENGINE calls in 3.3.6 as well Hive 3.1.3 and 4.0.0 build against Hadoop 3.3.6, so 3.3.6 has to build on UBI 10 too. Its OpensslSecureRandom.c is identical to 3.4.3's and fails to compile against headers that define OPENSSL_NO_ENGINE in the same way. OpensslCipher.c in 3.3.6 never looks up an ENGINE, because engine selection and SM4 only arrived in 3.4, so the third patch applied to 3.4.3 and 3.5.0 has nothing to guard here. --- ...lude-Engine-API-when-building-with-O.patch | 71 +++++++++++ ...ard-OpenSSL-ENGINE-code-paths-as-a-w.patch | 112 ++++++++++++++++++ 2 files changed, 183 insertions(+) create mode 100644 hadoop/hadoop/stackable/patches/3.3.6/0016-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch create mode 100644 hadoop/hadoop/stackable/patches/3.3.6/0017-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0016-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.3.6/0016-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch new file mode 100644 index 000000000..412aaf24b --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0016-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -0,0 +1,71 @@ +From 463b8032e9e396cdbe1afab84153055644a4b2a3 Mon Sep 17 00:00:00 2001 +From: Rohit Kumar +Date: Mon, 15 Jun 2026 15:54:13 +0530 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x + +--- + .../org/apache/hadoop/crypto/random/OpensslSecureRandom.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b54930e45b092671d02e33396cd4a98..d4351c1a5d70da0a73cfb64f2937b4bd04dcd541 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); + static int (*dlsym_ENGINE_finish) (ENGINE *); + static int (*dlsym_ENGINE_free) (ENGINE *); ++#endif + static int (*dlsym_RAND_bytes) (unsigned char *, int); + static unsigned long (*dlsym_ERR_get_error) (void); + #endif +@@ -126,12 +128,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + openssl, "ENGINE_load_rdrand"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup"); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \ + openssl, "ENGINE_set_default"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); + LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); + #endif +@@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) + */ + static ENGINE * openssl_rand_init(void) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -339,10 +344,12 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; ++#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -353,6 +360,7 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif ++#endif + } + + static int openssl_rand_bytes(unsigned char *buf, int num) diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0017-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch b/hadoop/hadoop/stackable/patches/3.3.6/0017-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch new file mode 100644 index 000000000..6b984601c --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0017-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch @@ -0,0 +1,112 @@ +From a7337bec7474934a3fac0fc5b41e76123996b77c Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Tue, 27 Oct 2026 16:12:48 +0100 +Subject: HADOOP-19872. Guard OpenSSL ENGINE code paths as a whole and update + RNG docs + +--- + .../crypto/random/OpensslSecureRandom.java | 9 ++++----- + .../hadoop/crypto/random/OpensslSecureRandom.c | 16 ++++++++++------ + .../random/org_apache_hadoop_crypto_random.h | 2 ++ + 3 files changed, 16 insertions(+), 11 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +index a7a609ce440b61aabda0d856bcc16d6fef5e82a2..a25160a9a780246f8fab85dc70473af143057f69 100644 +--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java ++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +@@ -32,11 +32,10 @@ import org.slf4j.LoggerFactory; + * This implementation is thread-safe. + *

+ * +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used and it's much faster than +- * {@link java.security.SecureRandom}. If RDRAND is unavailable, default +- * OpenSSL secure random generator will be used. It's still faster +- * and can generate strong random bytes. ++ * When built against OpenSSL older than 3.0 on a CPU with RDRAND, the ++ * hardware random number generator is used through OpenSSL's RDRAND engine. ++ * Otherwise, including every build against OpenSSL 3.0 or later, OpenSSL's ++ * default secure random generator is used. + *

+ * See https://wiki.openssl.org/index.php/Random_Numbers + * See http://en.wikipedia.org/wiki/RdRand +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index d4351c1a5d70da0a73cfb64f2937b4bd04dcd541..8fb88365f715ce95195c2596f21953177fab3219 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -90,8 +90,10 @@ static __dlsym_RAND_bytes dlsym_RAND_bytes; + static __dlsym_ERR_get_error dlsym_ERR_get_error; + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void); + static void openssl_rand_clean(ENGINE *eng, int clean_locks); ++#endif + static int openssl_rand_bytes(unsigned char *buf, int num); + + JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR +@@ -170,7 +172,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + env, openssl, "ERR_get_error"); + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + openssl_rand_init(); ++#endif + } + + JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B +@@ -307,12 +311,14 @@ static unsigned long pthreads_thread_id(void) + #endif /* UNIX */ + + /** +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used. ++ * Before OpenSSL 3.0, the RDRAND engine is made the default RAND method if ++ * the CPU supports it. OpenSSL 3.0 deprecates the ENGINE API and some ++ * distributions build without it, so from 3.0 on RAND_bytes uses OpenSSL's ++ * default DRBG. + */ ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -344,12 +350,10 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; +-#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -360,8 +364,8 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif +-#endif + } ++#endif + + static int openssl_rand_bytes(unsigned char *buf, int num) + { +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +index 1200718e69f301408e4aae92ef4e331f82a70fb4..8e2eb94e6149f4b26fb00cde7ead5dc609859051 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +@@ -33,7 +33,9 @@ + #define UNUSED(x) ((void)(x)) + + #include ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #include ++#endif + #include + #include + From f6dac7f56bf7bdddbdfc16dedd5c43075604794f Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 29 Oct 2026 13:07:08 +0100 Subject: [PATCH 13/13] docs: Add a changelog entry for UBI 10 and the Hadoop OpenSSL 3 fix --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcdfc15c9..fbc111455 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,9 @@ All notable changes to this project will be documented in this file. ### Changed +- stackable-base, stackable-devel: Build on UBI 10 instead of UBI 9, which affects every image. java-base installs gzip, which UBI 9 pulled in through krb5-workstation and UBI 10 does not ([#1534]). +- hadoop: Build the native code against OpenSSL 3 without the ENGINE API in `3.3.6`, `3.4.3` and `3.5.0`, by applying apache/hadoop#8550 ([HADOOP-19872](https://issues.apache.org/jira/browse/HADOOP-19872)) and two patches of our own that guard the remaining ENGINE calls ([#1534]). +- hadoop: Backport [HADOOP-19551](https://issues.apache.org/jira/browse/HADOOP-19551) and [HDFS-17226](https://issues.apache.org/jira/browse/HDFS-17226) into `3.3.6`, so its native code compiles with GCC 14 ([#1534]). - krb5: Bump base image ([#1721]). - testing-tools: Bump various tools and Python dependencies ([#1720]). - ubi-rust-builders: Bump protoc to `36.2` ([#1717]). @@ -131,6 +134,7 @@ All notable changes to this project will be documented in this file. - opa: Remove 1.12.3 ([#1697]). - spark: Remove 3.5.8 and 4.1.1 ([#1710]). +[#1534]: https://github.com/stackabletech/docker-images/pull/1534 [#1593]: https://github.com/stackabletech/docker-images/pull/1593 [#1595]: https://github.com/stackabletech/docker-images/pull/1595 [#1596]: https://github.com/stackabletech/docker-images/pull/1596