diff --git a/CHANGELOG.md b/CHANGELOG.md index bcdfc15c9..fbc111455 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,9 @@ All notable changes to this project will be documented in this file. ### Changed +- stackable-base, stackable-devel: Build on UBI 10 instead of UBI 9, which affects every image. java-base installs gzip, which UBI 9 pulled in through krb5-workstation and UBI 10 does not ([#1534]). +- hadoop: Build the native code against OpenSSL 3 without the ENGINE API in `3.3.6`, `3.4.3` and `3.5.0`, by applying apache/hadoop#8550 ([HADOOP-19872](https://issues.apache.org/jira/browse/HADOOP-19872)) and two patches of our own that guard the remaining ENGINE calls ([#1534]). +- hadoop: Backport [HADOOP-19551](https://issues.apache.org/jira/browse/HADOOP-19551) and [HDFS-17226](https://issues.apache.org/jira/browse/HDFS-17226) into `3.3.6`, so its native code compiles with GCC 14 ([#1534]). - krb5: Bump base image ([#1721]). - testing-tools: Bump various tools and Python dependencies ([#1720]). - ubi-rust-builders: Bump protoc to `36.2` ([#1717]). @@ -131,6 +134,7 @@ All notable changes to this project will be documented in this file. - opa: Remove 1.12.3 ([#1697]). - spark: Remove 3.5.8 and 4.1.1 ([#1710]). +[#1534]: https://github.com/stackabletech/docker-images/pull/1534 [#1593]: https://github.com/stackabletech/docker-images/pull/1593 [#1595]: https://github.com/stackabletech/docker-images/pull/1595 [#1596]: https://github.com/stackabletech/docker-images/pull/1596 diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0014-HADOOP-19551.-Fix-compilation-error-of-native-librar.patch b/hadoop/hadoop/stackable/patches/3.3.6/0014-HADOOP-19551.-Fix-compilation-error-of-native-librar.patch new file mode 100644 index 000000000..e0dbaa481 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0014-HADOOP-19551.-Fix-compilation-error-of-native-librar.patch @@ -0,0 +1,26 @@ +From 26af6e62c84f24ce7d81c5c27f5fcae0a4edfd2b Mon Sep 17 00:00:00 2001 +From: Masatake Iwasaki +Date: Wed, 7 May 2025 15:01:01 +0900 +Subject: HADOOP-19551. Fix compilation error of native libraries on newer GCC. + (#7644) + +Reviewed-by: Chris Nauroth +(cherry picked from commit 7ea3f51287c54ac563ef19d42c5bc102699ea025) +--- + hadoop-common-project/hadoop-common/HadoopCommon.cmake | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/HadoopCommon.cmake b/hadoop-common-project/hadoop-common/HadoopCommon.cmake +index 4de70ac3f78e2de5128e1ac48b9c1a20f1f4d18b..ade701675ad79915d29258e4be9c23ca02be4ffa 100644 +--- a/hadoop-common-project/hadoop-common/HadoopCommon.cmake ++++ b/hadoop-common-project/hadoop-common/HadoopCommon.cmake +@@ -145,6 +145,9 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Make GNU extensions available. + hadoop_add_compiler_flags("-D_GNU_SOURCE") + ++ # using old default behavior on GCC >= 14.0 ++ hadoop_add_compiler_flags("-Wno-error=implicit-function-declaration") ++ + # If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit. + if(JVM_ARCH_DATA_MODEL EQUAL 32) + # Force 32-bit code generation on amd64/x86_64, ppc64, sparc64 diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0015-HDFS-17226.-Building-native-libraries-fails-on-Fedor.patch b/hadoop/hadoop/stackable/patches/3.3.6/0015-HDFS-17226.-Building-native-libraries-fails-on-Fedor.patch new file mode 100644 index 000000000..6a1c42ce5 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0015-HDFS-17226.-Building-native-libraries-fails-on-Fedor.patch @@ -0,0 +1,61 @@ +From 4e739baadfd68b94abd45fe9e2df1c8bfa2bc880 Mon Sep 17 00:00:00 2001 +From: Kengo Seki +Date: Wed, 23 Apr 2025 00:12:22 +0900 +Subject: HDFS-17226. Building native libraries fails on Fedora 38. (#6191) + +(cherry picked from commit a47df85360bfebbb7dfd658630b7c66d982a49cf) +--- + .../src/main/native/libhdfspp/include/hdfspp/content_summary.h | 1 + + .../src/main/native/libhdfspp/include/hdfspp/fsinfo.h | 1 + + .../src/main/native/libhdfspp/include/hdfspp/statinfo.h | 1 + + .../src/main/native/libhdfspp/include/hdfspp/uri.h | 1 + + 4 files changed, 4 insertions(+) + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h +index 6762a65ca908b513a8fdea0a117a52835af126b7..d9809bab59291b6aec68b72dcba798606b5ad3ee 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/content_summary.h +@@ -19,6 +19,7 @@ + #define HDFSPP_CONTENT_SUMMARY_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h +index 103cf133bec221e9450c38fbc33eec035d73e767..156ed445181a331760df3e245201df1ec84f5da5 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/fsinfo.h +@@ -19,6 +19,7 @@ + #define HDFSPP_FSINFO_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h +index 1da1f18a592bdcc6173443b6418694fdf8c1c361..5924d1db005870c5beb26a881db9a30702039e2d 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h +@@ -19,6 +19,7 @@ + #define HDFSPP_STATINFO_H_ + + #include ++#include + + namespace hdfs { + +diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h +index bc3d8b96d3f66e7b8904d19f69a9006441f8f647..a7f735afb5dbc1c1bc7c80621d5cebc38ea95927 100644 +--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h ++++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + #include + + namespace hdfs diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0016-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.3.6/0016-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch new file mode 100644 index 000000000..412aaf24b --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0016-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -0,0 +1,71 @@ +From 463b8032e9e396cdbe1afab84153055644a4b2a3 Mon Sep 17 00:00:00 2001 +From: Rohit Kumar +Date: Mon, 15 Jun 2026 15:54:13 +0530 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x + +--- + .../org/apache/hadoop/crypto/random/OpensslSecureRandom.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b54930e45b092671d02e33396cd4a98..d4351c1a5d70da0a73cfb64f2937b4bd04dcd541 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); + static int (*dlsym_ENGINE_finish) (ENGINE *); + static int (*dlsym_ENGINE_free) (ENGINE *); ++#endif + static int (*dlsym_RAND_bytes) (unsigned char *, int); + static unsigned long (*dlsym_ERR_get_error) (void); + #endif +@@ -126,12 +128,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + openssl, "ENGINE_load_rdrand"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup"); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \ + openssl, "ENGINE_set_default"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); + LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); + #endif +@@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) + */ + static ENGINE * openssl_rand_init(void) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -339,10 +344,12 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; ++#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -353,6 +360,7 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif ++#endif + } + + static int openssl_rand_bytes(unsigned char *buf, int num) diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0017-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch b/hadoop/hadoop/stackable/patches/3.3.6/0017-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch new file mode 100644 index 000000000..6b984601c --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0017-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch @@ -0,0 +1,112 @@ +From a7337bec7474934a3fac0fc5b41e76123996b77c Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Tue, 27 Oct 2026 16:12:48 +0100 +Subject: HADOOP-19872. Guard OpenSSL ENGINE code paths as a whole and update + RNG docs + +--- + .../crypto/random/OpensslSecureRandom.java | 9 ++++----- + .../hadoop/crypto/random/OpensslSecureRandom.c | 16 ++++++++++------ + .../random/org_apache_hadoop_crypto_random.h | 2 ++ + 3 files changed, 16 insertions(+), 11 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +index a7a609ce440b61aabda0d856bcc16d6fef5e82a2..a25160a9a780246f8fab85dc70473af143057f69 100644 +--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java ++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +@@ -32,11 +32,10 @@ import org.slf4j.LoggerFactory; + * This implementation is thread-safe. + *

+ * +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used and it's much faster than +- * {@link java.security.SecureRandom}. If RDRAND is unavailable, default +- * OpenSSL secure random generator will be used. It's still faster +- * and can generate strong random bytes. ++ * When built against OpenSSL older than 3.0 on a CPU with RDRAND, the ++ * hardware random number generator is used through OpenSSL's RDRAND engine. ++ * Otherwise, including every build against OpenSSL 3.0 or later, OpenSSL's ++ * default secure random generator is used. + *

+ * See https://wiki.openssl.org/index.php/Random_Numbers + * See http://en.wikipedia.org/wiki/RdRand +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index d4351c1a5d70da0a73cfb64f2937b4bd04dcd541..8fb88365f715ce95195c2596f21953177fab3219 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -90,8 +90,10 @@ static __dlsym_RAND_bytes dlsym_RAND_bytes; + static __dlsym_ERR_get_error dlsym_ERR_get_error; + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void); + static void openssl_rand_clean(ENGINE *eng, int clean_locks); ++#endif + static int openssl_rand_bytes(unsigned char *buf, int num); + + JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR +@@ -170,7 +172,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + env, openssl, "ERR_get_error"); + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + openssl_rand_init(); ++#endif + } + + JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B +@@ -307,12 +311,14 @@ static unsigned long pthreads_thread_id(void) + #endif /* UNIX */ + + /** +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used. ++ * Before OpenSSL 3.0, the RDRAND engine is made the default RAND method if ++ * the CPU supports it. OpenSSL 3.0 deprecates the ENGINE API and some ++ * distributions build without it, so from 3.0 on RAND_bytes uses OpenSSL's ++ * default DRBG. + */ ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -344,12 +350,10 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; +-#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -360,8 +364,8 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif +-#endif + } ++#endif + + static int openssl_rand_bytes(unsigned char *buf, int num) + { +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +index 1200718e69f301408e4aae92ef4e331f82a70fb4..8e2eb94e6149f4b26fb00cde7ead5dc609859051 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +@@ -33,7 +33,9 @@ + #define UNUSED(x) ((void)(x)) + + #include ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #include ++#endif + #include + #include + diff --git a/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch new file mode 100644 index 000000000..4eb406d54 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.3/0009-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -0,0 +1,71 @@ +From 62b70efc20e8a619b73f553ca78ef9b85b025240 Mon Sep 17 00:00:00 2001 +From: Rohit Kumar +Date: Mon, 15 Jun 2026 15:54:13 +0530 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x + +--- + .../org/apache/hadoop/crypto/random/OpensslSecureRandom.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b54930e45b092671d02e33396cd4a98..d4351c1a5d70da0a73cfb64f2937b4bd04dcd541 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); + static int (*dlsym_ENGINE_finish) (ENGINE *); + static int (*dlsym_ENGINE_free) (ENGINE *); ++#endif + static int (*dlsym_RAND_bytes) (unsigned char *, int); + static unsigned long (*dlsym_ERR_get_error) (void); + #endif +@@ -126,12 +128,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + openssl, "ENGINE_load_rdrand"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup"); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \ + openssl, "ENGINE_set_default"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); + LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); + #endif +@@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) + */ + static ENGINE * openssl_rand_init(void) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -339,10 +344,12 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; ++#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -353,6 +360,7 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif ++#endif + } + + static int openssl_rand_bytes(unsigned char *buf, int num) diff --git a/hadoop/hadoop/stackable/patches/3.4.3/0010-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch b/hadoop/hadoop/stackable/patches/3.4.3/0010-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch new file mode 100644 index 000000000..7d92815cb --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.3/0010-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch @@ -0,0 +1,112 @@ +From b436664681bb663d08878ab38bba5e251d016eda Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Tue, 27 Oct 2026 16:12:48 +0100 +Subject: HADOOP-19872. Guard OpenSSL ENGINE code paths as a whole and update + RNG docs + +--- + .../crypto/random/OpensslSecureRandom.java | 9 ++++----- + .../hadoop/crypto/random/OpensslSecureRandom.c | 16 ++++++++++------ + .../random/org_apache_hadoop_crypto_random.h | 2 ++ + 3 files changed, 16 insertions(+), 11 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +index b9f569d0c90780fee218258c0d1c34aa276d16d7..2d30b57e9d55893efb18b1bf7e3ed1b8dfdf48ca 100644 +--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java ++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +@@ -32,11 +32,10 @@ import org.slf4j.LoggerFactory; + * This implementation is thread-safe. + *

+ * +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used and it's much faster than +- * {@link java.security.SecureRandom}. If RDRAND is unavailable, default +- * OpenSSL secure random generator will be used. It's still faster +- * and can generate strong random bytes. ++ * When built against OpenSSL older than 3.0 on a CPU with RDRAND, the ++ * hardware random number generator is used through OpenSSL's RDRAND engine. ++ * Otherwise, including every build against OpenSSL 3.0 or later, OpenSSL's ++ * default secure random generator is used. + *

+ * See https://wiki.openssl.org/index.php/Random_Numbers + * See http://en.wikipedia.org/wiki/RdRand +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index d4351c1a5d70da0a73cfb64f2937b4bd04dcd541..8fb88365f715ce95195c2596f21953177fab3219 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -90,8 +90,10 @@ static __dlsym_RAND_bytes dlsym_RAND_bytes; + static __dlsym_ERR_get_error dlsym_ERR_get_error; + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void); + static void openssl_rand_clean(ENGINE *eng, int clean_locks); ++#endif + static int openssl_rand_bytes(unsigned char *buf, int num); + + JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR +@@ -170,7 +172,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + env, openssl, "ERR_get_error"); + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + openssl_rand_init(); ++#endif + } + + JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B +@@ -307,12 +311,14 @@ static unsigned long pthreads_thread_id(void) + #endif /* UNIX */ + + /** +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used. ++ * Before OpenSSL 3.0, the RDRAND engine is made the default RAND method if ++ * the CPU supports it. OpenSSL 3.0 deprecates the ENGINE API and some ++ * distributions build without it, so from 3.0 on RAND_bytes uses OpenSSL's ++ * default DRBG. + */ ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -344,12 +350,10 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; +-#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -360,8 +364,8 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif +-#endif + } ++#endif + + static int openssl_rand_bytes(unsigned char *buf, int num) + { +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +index 1200718e69f301408e4aae92ef4e331f82a70fb4..8e2eb94e6149f4b26fb00cde7ead5dc609859051 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +@@ -33,7 +33,9 @@ + #define UNUSED(x) ((void)(x)) + + #include ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #include ++#endif + #include + #include + diff --git a/hadoop/hadoop/stackable/patches/3.4.3/0011-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch b/hadoop/hadoop/stackable/patches/3.4.3/0011-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch new file mode 100644 index 000000000..6348d394a --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.3/0011-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch @@ -0,0 +1,110 @@ +From c53b426246d05cefb425d1ddf1302508ea80f3a8 Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Thu, 29 Oct 2026 11:34:54 +0100 +Subject: HADOOP-19872. Guard the cipher's ENGINE lookups with + OPENSSL_NO_ENGINE + +OpensslCipher still resolved ENGINE_by_id and ENGINE_free in initIDs. +Against a libcrypto that exports neither, initIDs throws +UnsatisfiedLinkError and the AES-CTR codec falls back to JCE with only a +warning. It loads on RHEL 10 only because that libcrypto still exports +the ENGINE symbols although its headers define OPENSSL_NO_ENGINE. + +The guard is OPENSSL_NO_ENGINE rather than the OpenSSL version, because +OpenSSL 3 builds that keep ENGINE still honour +hadoop.security.openssl.engine.id. + +Also fix the Windows lookup of ENGINE_free, which asked for +"ENGINE_by_free". + +Generated-by: Claude (Anthropic) +--- + .../org/apache/hadoop/crypto/OpensslCipher.c | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index 33be4a394f467cbffefba13d74d3cd8da5c7bb78..0c000f39050886eb8c2eedba850db2e3dc944b95 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -101,9 +101,11 @@ static EVP_CIPHER * (*dlsym_EVP_aes_128_ctr)(void); + static EVP_CIPHER * (*dlsym_EVP_sm4_ctr)(void); + static int (*dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); ++#ifndef OPENSSL_NO_ENGINE + static ENGINE * (*dlsym_ENGINE_by_id)(const char *id); + static int (*dlsym_ENGINE_free)(ENGINE *); + #endif ++#endif + static void *openssl; + #endif + +@@ -146,14 +148,16 @@ static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr; + typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_sm4_ctr)(void); + typedef int (__cdecl *__dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); +-typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); +-typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + + static __dlsym_EVP_sm4_ctr dlsym_EVP_sm4_ctr; + static __dlsym_OPENSSL_init_crypto dlsym_OPENSSL_init_crypto; ++#ifndef OPENSSL_NO_ENGINE ++typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); ++typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + static __dlsym_ENGINE_by_id dlsym_ENGINE_by_id; + static __dlsym_ENGINE_free dlsym_ENGINE_free; + #endif ++#endif + static HMODULE openssl; + #endif + +@@ -244,12 +248,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(dlsym_OPENSSL_init_crypto, env, openssl, \ + "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, \ + "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, \ + "ENGINE_free"); + #endif + #endif ++#endif + + #ifdef WINDOWS + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_new, dlsym_EVP_CIPHER_CTX_new, \ +@@ -284,10 +290,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(__dlsym_OPENSSL_init_crypto, dlsym_OPENSSL_init_crypto, \ + env, openssl, "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_by_id, dlsym_ENGINE_by_id, \ + env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_free, dlsym_ENGINE_free, \ +- env, openssl, "ENGINE_by_free"); ++ env, openssl, "ENGINE_free"); ++#endif + #endif + #endif + +@@ -362,7 +370,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initEngine + { + ENGINE *e = NULL; + +-#if OPENSSL_VERSION_NUMBER >= 0x10101001L ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + if (engineId != NULL) { + const char *id = (*env)->GetStringUTFChars(env, engineId, NULL); + if (id != NULL) { +@@ -574,10 +582,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_clean + dlsym_EVP_CIPHER_CTX_free(context); + } + ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + ENGINE *e = LONG_TO_ENGINE(engine); + if (e) { + dlsym_ENGINE_free(e); + } ++#endif + } + + JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibraryName diff --git a/hadoop/hadoop/stackable/patches/3.5.0/0008-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch b/hadoop/hadoop/stackable/patches/3.5.0/0008-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch new file mode 100644 index 000000000..136ec54b1 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.5.0/0008-HADOOP-19872-Exclude-Engine-API-when-building-with-O.patch @@ -0,0 +1,71 @@ +From 897b1bc5a97f92cca6ad787364ffaf45d6dadee6 Mon Sep 17 00:00:00 2001 +From: Rohit Kumar +Date: Mon, 15 Jun 2026 15:54:13 +0530 +Subject: HADOOP-19872: Exclude Engine API when building with OpenSSL 3.x + +--- + .../org/apache/hadoop/crypto/random/OpensslSecureRandom.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index 3f141be05b54930e45b092671d02e33396cd4a98..d4351c1a5d70da0a73cfb64f2937b4bd04dcd541 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -49,11 +49,13 @@ static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)()); + static void (*dlsym_ENGINE_load_rdrand) (void); + static void (*dlsym_ENGINE_cleanup) (void); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * (*dlsym_ENGINE_by_id) (const char *); + static int (*dlsym_ENGINE_init) (ENGINE *); + static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int); + static int (*dlsym_ENGINE_finish) (ENGINE *); + static int (*dlsym_ENGINE_free) (ENGINE *); ++#endif + static int (*dlsym_RAND_bytes) (unsigned char *, int); + static unsigned long (*dlsym_ERR_get_error) (void); + #endif +@@ -126,12 +128,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + openssl, "ENGINE_load_rdrand"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup"); + #endif ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \ + openssl, "ENGINE_set_default"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free"); ++#endif + LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes"); + LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error"); + #endif +@@ -308,6 +312,7 @@ static unsigned long pthreads_thread_id(void) + */ + static ENGINE * openssl_rand_init(void) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -339,10 +344,12 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; ++#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -353,6 +360,7 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif ++#endif + } + + static int openssl_rand_bytes(unsigned char *buf, int num) diff --git a/hadoop/hadoop/stackable/patches/3.5.0/0009-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch b/hadoop/hadoop/stackable/patches/3.5.0/0009-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch new file mode 100644 index 000000000..f1b992d8e --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.5.0/0009-HADOOP-19872.-Guard-OpenSSL-ENGINE-code-paths-as-a-w.patch @@ -0,0 +1,112 @@ +From 6099a11bc5ccd6f8cd2be4eccb3d3736e41d72db Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Tue, 27 Oct 2026 16:12:48 +0100 +Subject: HADOOP-19872. Guard OpenSSL ENGINE code paths as a whole and update + RNG docs + +--- + .../crypto/random/OpensslSecureRandom.java | 9 ++++----- + .../hadoop/crypto/random/OpensslSecureRandom.c | 16 ++++++++++------ + .../random/org_apache_hadoop_crypto_random.h | 2 ++ + 3 files changed, 16 insertions(+), 11 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +index b9f569d0c90780fee218258c0d1c34aa276d16d7..2d30b57e9d55893efb18b1bf7e3ed1b8dfdf48ca 100644 +--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java ++++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/random/OpensslSecureRandom.java +@@ -32,11 +32,10 @@ import org.slf4j.LoggerFactory; + * This implementation is thread-safe. + *

+ * +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used and it's much faster than +- * {@link java.security.SecureRandom}. If RDRAND is unavailable, default +- * OpenSSL secure random generator will be used. It's still faster +- * and can generate strong random bytes. ++ * When built against OpenSSL older than 3.0 on a CPU with RDRAND, the ++ * hardware random number generator is used through OpenSSL's RDRAND engine. ++ * Otherwise, including every build against OpenSSL 3.0 or later, OpenSSL's ++ * default secure random generator is used. + *

+ * See https://wiki.openssl.org/index.php/Random_Numbers + * See http://en.wikipedia.org/wiki/RdRand +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +index d4351c1a5d70da0a73cfb64f2937b4bd04dcd541..8fb88365f715ce95195c2596f21953177fab3219 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c +@@ -90,8 +90,10 @@ static __dlsym_RAND_bytes dlsym_RAND_bytes; + static __dlsym_ERR_get_error dlsym_ERR_get_error; + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void); + static void openssl_rand_clean(ENGINE *eng, int clean_locks); ++#endif + static int openssl_rand_bytes(unsigned char *buf, int num); + + JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_initSR +@@ -170,7 +172,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_ + env, openssl, "ERR_get_error"); + #endif + ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + openssl_rand_init(); ++#endif + } + + JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_nextRandBytes___3B +@@ -307,12 +311,14 @@ static unsigned long pthreads_thread_id(void) + #endif /* UNIX */ + + /** +- * If using an Intel chipset with RDRAND, the high-performance hardware +- * random number generator will be used. ++ * Before OpenSSL 3.0, the RDRAND engine is made the default RAND method if ++ * the CPU supports it. OpenSSL 3.0 deprecates the ENGINE API and some ++ * distributions build without it, so from 3.0 on RAND_bytes uses OpenSSL's ++ * default DRBG. + */ ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + static ENGINE * openssl_rand_init(void) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + #if OPENSSL_VERSION_NUMBER < 0x10100000L + locks_setup(); + +@@ -344,12 +350,10 @@ static ENGINE * openssl_rand_init(void) + } + + return eng; +-#endif + } + + static void openssl_rand_clean(ENGINE *eng, int clean_locks) + { +-#if OPENSSL_VERSION_NUMBER < 0x30000000L + if (NULL != eng) { + dlsym_ENGINE_finish(eng); + dlsym_ENGINE_free(eng); +@@ -360,8 +364,8 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks) + locks_cleanup(); + } + #endif +-#endif + } ++#endif + + static int openssl_rand_bytes(unsigned char *buf, int num) + { +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +index 1200718e69f301408e4aae92ef4e331f82a70fb4..8e2eb94e6149f4b26fb00cde7ead5dc609859051 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/org_apache_hadoop_crypto_random.h +@@ -33,7 +33,9 @@ + #define UNUSED(x) ((void)(x)) + + #include ++#if OPENSSL_VERSION_NUMBER < 0x30000000L + #include ++#endif + #include + #include + diff --git a/hadoop/hadoop/stackable/patches/3.5.0/0010-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch b/hadoop/hadoop/stackable/patches/3.5.0/0010-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch new file mode 100644 index 000000000..d19306e97 --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.5.0/0010-HADOOP-19872.-Guard-the-cipher-s-ENGINE-lookups-with.patch @@ -0,0 +1,110 @@ +From 8a81b0b20632b47bdbbf45d9a91cc05b4115d94f Mon Sep 17 00:00:00 2001 +From: Lars Francke +Date: Thu, 29 Oct 2026 11:34:54 +0100 +Subject: HADOOP-19872. Guard the cipher's ENGINE lookups with + OPENSSL_NO_ENGINE + +OpensslCipher still resolved ENGINE_by_id and ENGINE_free in initIDs. +Against a libcrypto that exports neither, initIDs throws +UnsatisfiedLinkError and the AES-CTR codec falls back to JCE with only a +warning. It loads on RHEL 10 only because that libcrypto still exports +the ENGINE symbols although its headers define OPENSSL_NO_ENGINE. + +The guard is OPENSSL_NO_ENGINE rather than the OpenSSL version, because +OpenSSL 3 builds that keep ENGINE still honour +hadoop.security.openssl.engine.id. + +Also fix the Windows lookup of ENGINE_free, which asked for +"ENGINE_by_free". + +Generated-by: Claude (Anthropic) +--- + .../org/apache/hadoop/crypto/OpensslCipher.c | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index 33be4a394f467cbffefba13d74d3cd8da5c7bb78..0c000f39050886eb8c2eedba850db2e3dc944b95 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -101,9 +101,11 @@ static EVP_CIPHER * (*dlsym_EVP_aes_128_ctr)(void); + static EVP_CIPHER * (*dlsym_EVP_sm4_ctr)(void); + static int (*dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); ++#ifndef OPENSSL_NO_ENGINE + static ENGINE * (*dlsym_ENGINE_by_id)(const char *id); + static int (*dlsym_ENGINE_free)(ENGINE *); + #endif ++#endif + static void *openssl; + #endif + +@@ -146,14 +148,16 @@ static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr; + typedef EVP_CIPHER * (__cdecl *__dlsym_EVP_sm4_ctr)(void); + typedef int (__cdecl *__dlsym_OPENSSL_init_crypto)(uint64_t opts, \ + const OPENSSL_INIT_SETTINGS *settings); +-typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); +-typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + + static __dlsym_EVP_sm4_ctr dlsym_EVP_sm4_ctr; + static __dlsym_OPENSSL_init_crypto dlsym_OPENSSL_init_crypto; ++#ifndef OPENSSL_NO_ENGINE ++typedef ENGINE * (__cdecl *__dlsym_ENGINE_by_id)(const char *id); ++typedef int (__cdecl *__dlsym_ENGINE_free)(ENGINE *e); + static __dlsym_ENGINE_by_id dlsym_ENGINE_by_id; + static __dlsym_ENGINE_free dlsym_ENGINE_free; + #endif ++#endif + static HMODULE openssl; + #endif + +@@ -244,12 +248,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(dlsym_OPENSSL_init_crypto, env, openssl, \ + "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, \ + "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, \ + "ENGINE_free"); + #endif + #endif ++#endif + + #ifdef WINDOWS + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_new, dlsym_EVP_CIPHER_CTX_new, \ +@@ -284,10 +290,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + #if OPENSSL_VERSION_NUMBER >= 0x10101001L + LOAD_DYNAMIC_SYMBOL(__dlsym_OPENSSL_init_crypto, dlsym_OPENSSL_init_crypto, \ + env, openssl, "OPENSSL_init_crypto"); ++#ifndef OPENSSL_NO_ENGINE + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_by_id, dlsym_ENGINE_by_id, \ + env, openssl, "ENGINE_by_id"); + LOAD_DYNAMIC_SYMBOL(__dlsym_ENGINE_free, dlsym_ENGINE_free, \ +- env, openssl, "ENGINE_by_free"); ++ env, openssl, "ENGINE_free"); ++#endif + #endif + #endif + +@@ -362,7 +370,7 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initEngine + { + ENGINE *e = NULL; + +-#if OPENSSL_VERSION_NUMBER >= 0x10101001L ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + if (engineId != NULL) { + const char *id = (*env)->GetStringUTFChars(env, engineId, NULL); + if (id != NULL) { +@@ -574,10 +582,12 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_clean + dlsym_EVP_CIPHER_CTX_free(context); + } + ++#if OPENSSL_VERSION_NUMBER >= 0x10101001L && !defined(OPENSSL_NO_ENGINE) + ENGINE *e = LONG_TO_ENGINE(engine); + if (e) { + dlsym_ENGINE_free(e); + } ++#endif + } + + JNIEXPORT jstring JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_getLibraryName diff --git a/java-base/Dockerfile b/java-base/Dockerfile index 4f5eaa54e..d0c80fb3a 100644 --- a/java-base/Dockerfile +++ b/java-base/Dockerfile @@ -40,6 +40,9 @@ RUN microdnf update && \ tzdata-java \ # Most of the Java tools needs at least "klist" and "kinit" for Kerberos integration krb5-workstation \ + # Needed by `tar -z`, e.g. when hadoop unpacks async-profiler. UBI 9 pulled it in through krb5-workstation, UBI 10 does not. + # Added 2026-10 as gzip-1.13-4.el10_2, which includes the fixes for CVE-2026-41991 and CVE-2026-41992 (RHSA-2026:61625). + gzip \ --nodocs && \ microdnf clean all diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index a9ff42ff5..9a3c8849a 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -31,9 +31,9 @@ EOF # Find the latest version: # -# docker buildx imagetools inspect registry.access.redhat.com/ubi9/ubi-minimal --format '{{json .Manifest.Digest}}' +# docker buildx imagetools inspect registry.access.redhat.com/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' # -FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:5ed244b62bbf4095080144d9d35eb8fcd3d39a9801f94aadd63b9d10978a01ae AS builder +FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:ab67cd02caf8706e34a695a1432ebe965934f6128ec352a6c3f4048b43a4fb5e AS builder # intentionally unused ARG PRODUCT_VERSION @@ -150,7 +150,7 @@ chown ${STACKABLE_USER_UID}:0 /stackable/.curlrc # CVE-2023-37920: Remove "e-Tugra" root certificates # e-Tugra's root certificates were subject to an investigation prompted by reporting of security issues in their systems # Until they are removed by default from ca-certificates, we should remove them manually -EXPECTED_CERTS_PACKAGE="ca-certificates-2026.2.92_v9.0.319-90.3.el9_8.noarch" +EXPECTED_CERTS_PACKAGE="ca-certificates-2026.2.92_v9.0.319-100.3.el10_2.noarch" ACTUAL_CERTS_PACKAGE="$(rpm -qa ca-certificates)" if [ "$ACTUAL_CERTS_PACKAGE" != "$EXPECTED_CERTS_PACKAGE" ]; then echo "The ca-certificates package was updated to $ACTUAL_CERTS_PACKAGE. Please check if the e-Tugra root certificates are present. \ diff --git a/stackable-devel/Dockerfile b/stackable-devel/Dockerfile index 52e49f3cf..c589b0eac 100644 --- a/stackable-devel/Dockerfile +++ b/stackable-devel/Dockerfile @@ -10,9 +10,9 @@ # Find the latest version: # -# docker buildx imagetools inspect registry.access.redhat.com/ubi9/ubi-minimal --format '{{json .Manifest.Digest}}' +# docker buildx imagetools inspect registry.access.redhat.com/ubi10/ubi-minimal --format '{{json .Manifest.Digest}}' # -FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:5ed244b62bbf4095080144d9d35eb8fcd3d39a9801f94aadd63b9d10978a01ae AS builder +FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:ab67cd02caf8706e34a695a1432ebe965934f6128ec352a6c3f4048b43a4fb5e AS builder # intentionally unused ARG PRODUCT_VERSION