mod_http3: run the QUIC transport on ngtcp2

.github/workflows/linux.yml

@@ -180,7 +180,8 @@ jobs:
               PYTEST_ARGS=modules/http2
           # -------------------------------------------------------------------------
           # mod_http3 needs OpenSSL >= 3.5 and libnghttp3, which Ubuntu
-          # 26.04 has. Its curl has no HTTP/3, so one is built from source.
+          # 26.04 has, and ngtcp2 >= 1.22.1, which it has not. Its curl
+          # has no HTTP/3. ngtcp2 and curl are built from source.
           # Strict mode turns a skip for a missing dependency into a failure.
           - name: mod_http3
             os: ubuntu-26.04
@@ -451,6 +452,8 @@ jobs:
     env:
       NOTEST_CFLAGS: ${{ matrix.notest-cflags }}
       CONFIG: ${{ matrix.config }}
+      # Built from source for a job with --enable-http3.
+      NGTCP2_VERSION: 1.25.0
     name: ${{ matrix.name }}
     steps:
     # JOBID names the artifacts uploaded on failure; see "Artifacts" in

CMakeLists.txt

@@ -116,6 +116,8 @@ SET(NGHTTP2_INCLUDE_DIR   "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Direct
 SET(NGHTTP2_LIBRARIES     ${default_nghttp2_libraries}   CACHE STRING "NGHTTP2 libraries to link with")
 SET(NGHTTP3_INCLUDE_DIR   "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Directory with NGHTTP3 include files within nghttp3 subdirectory")
 SET(NGHTTP3_LIBRARIES     ${default_nghttp3_libraries}   CACHE STRING "NGHTTP3 libraries to link with")
+SET(NGTCP2_INCLUDE_DIR    "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Directory with NGTCP2 include files within ngtcp2 subdirectory")
+SET(NGTCP2_LIBRARIES      "${CMAKE_INSTALL_PREFIX}/lib/ngtcp2.lib" "${CMAKE_INSTALL_PREFIX}/lib/ngtcp2_crypto_ossl.lib" CACHE STRING "NGTCP2 libraries to link with")
 SET(PCRE_CFLAGS           "${default_pcre_cflags}"       CACHE STRING "PCRE flags for util_pcre.c compilation")
 SET(PCRE_INCLUDE_DIR      "${CMAKE_INSTALL_PREFIX}/include" CACHE STRING "Directory with PCRE include files")
 SET(PCRE_LIBRARIES        ${default_pcre_libraries}      CACHE STRING "PCRE libraries to link with")
@@ -244,6 +246,18 @@ ELSE()
   SET(NGHTTP3_FOUND FALSE)
 ENDIF()
 
+# See if ngtcp2 exists in a configured or defaulted location
+SET(NGTCP2_FOUND TRUE)
+IF(EXISTS "${NGTCP2_INCLUDE_DIR}/ngtcp2/ngtcp2_crypto_ossl.h")
+  FOREACH(onelib ${NGTCP2_LIBRARIES})
+    IF(NOT EXISTS ${onelib})
+      SET(NGTCP2_FOUND FALSE)
+    ENDIF()
+  ENDFOREACH()
+ELSE()
+  SET(NGTCP2_FOUND FALSE)
+ENDIF()
+
 # See if we have Brotli
 SET(BROTLI_FOUND TRUE)
 IF(EXISTS "${BROTLI_INCLUDE_DIR}/brotli/encode.h")
@@ -288,6 +302,7 @@ MESSAGE(STATUS "LIBXML2_FOUND ............ : ${LIBXML2_FOUND}")
 MESSAGE(STATUS "LUA51_FOUND .............. : ${LUA51_FOUND}")
 MESSAGE(STATUS "NGHTTP2_FOUND ............ : ${NGHTTP2_FOUND}")
 MESSAGE(STATUS "NGHTTP3_FOUND ............ : ${NGHTTP3_FOUND}")
+MESSAGE(STATUS "NGTCP2_FOUND ............. : ${NGTCP2_FOUND}")
 MESSAGE(STATUS "OPENSSL_FOUND ............ : ${OPENSSL_FOUND}")
 MESSAGE(STATUS "ZLIB_FOUND ............... : ${ZLIB_FOUND}")
 MESSAGE(STATUS "BROTLI_FOUND ............. : ${BROTLI_FOUND}")
@@ -575,20 +590,20 @@ SET(mod_http2_extra_sources
   modules/http2/h2_util.c            modules/http2/h2_workers.c
   modules/http2/h2_ws.c
 )
-SET(mod_http3_requires               NGHTTP3_FOUND)
-SET(mod_http3_extra_includes         ${NGHTTP3_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/modules/http3/include ${OPENSSL_INCLUDE_DIR})
-SET(mod_http3_extra_libs             ${NGHTTP3_LIBRARIES} ${OPENSSL_LIBRARIES})
+SET(mod_http3_requires               NGHTTP3_FOUND NGTCP2_FOUND)
+SET(mod_http3_extra_includes         ${NGHTTP3_INCLUDE_DIR} ${NGTCP2_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/modules/http3/include ${OPENSSL_INCLUDE_DIR})
+SET(mod_http3_extra_libs             ${NGHTTP3_LIBRARIES} ${NGTCP2_LIBRARIES} ${OPENSSL_LIBRARIES})
 SET(mod_http3_main_source            modules/http3/mod_http3.c)
 SET(mod_http3_extra_sources
   modules/http3/h3_callbacks.c     modules/http3/h3_config.c
   modules/http3/h3_filter.c        modules/http3/h3_hooks.c
   modules/http3/h3_io.c            modules/http3/h3_request.c
   modules/http3/h3_server.c        modules/http3/h3_session.c
   modules/http3/h3_socket.c        modules/http3/h3_socket_os.c
+  modules/http3/h3_socket_bpf.c
   modules/http3/h3_stream.c        modules/http3/h3_threads.c
   modules/http3/quic/h3q.c         modules/http3/quic/h3q_conn.c
   modules/http3/quic/h3q_stream.c
-  modules/http3/quic/detail/h3q_addr.c
   modules/http3/quic/detail/h3q_tls.c
 )
 SET(mod_ldap_extra_defines           LDAP_DECLARE_EXPORT)

changes-entries/h3_flow_window_autotune.txt

@@ -0,0 +1,5 @@
+  *) mod_http3: Added H3MaxWindow (default 6 MiB). Flow-control windows for
+     client uploads now grow when a client fills them within a few round
+     trips, up to this cap, so uploads on long, fast paths are no longer held
+     to a fixed 1 MiB window.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_memory_cid_map.txt

@@ -0,0 +1,11 @@
+  *) mod_http3: Less memory per connection. The QUIC connection keeps its
+     connection IDs in 16 slots of its own, not in an APR pool (8 KB at
+     least), and makes the CONNECTION_CLOSE buffer only when it closes. The
+     session and stream allocators keep at most MaxMemFree, as in mod_http2,
+     so an idle connection gives back its peak memory.
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: A client can no longer put an Initial DCID that is a CID of
+     another connection into the CID map. Before, the map could keep a key
+     that pointed into the pool of a freed connection.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_ngtcp2_early_data.txt

@@ -0,0 +1,14 @@
+  *) mod_http3: Added H3EarlyData (default off). A resumed client may send its
+     first request as 0-RTT data and gets the response one round trip sooner.
+     Only safe methods run before the handshake completes; other methods wait
+     for it, because 0-RTT data can be replayed (RFC 8470).
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: Moved the QUIC transport from the OpenSSL QUIC server to
+     ngtcp2, with OpenSSL as the TLS backend through its QUIC TLS API. OpenSSL
+     still drops server-side 0-RTT packets. ngtcp2 >= 1.22.1 is a new build
+     dependency (--with-ngtcp2). Stream data is no longer copied: nghttp3
+     releases a buffer when the peer acknowledges it. H3IdleTimeout now always
+     closes with NO_ERROR; the QUIC idle timer runs 2 seconds longer and
+     closes silently.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_parent_socket.txt

@@ -0,0 +1,5 @@
+  *) mod_http3: On Linux the parent binds the QUIC socket while it still runs
+     as root, so H3Port 443 works without CAP_NET_BIND_SERVICE. The socket
+     stays bound across graceful restarts and the children inherit it; one
+     child at a time serves it.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_pktinfo_ecn_token_key.txt

@@ -0,0 +1,21 @@
+  *) mod_http3: Linux: on a [::] socket, a reply now goes out from the address
+     the client sent to (IP_PKTINFO, IPV6_PKTINFO). Before, the kernel picked
+     the source, and on a host with more than one address a client could get
+     the reply from another address and drop it.
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: Linux: ECN. The module reads the ECN mark of each packet and
+     gives it to ngtcp2, which reports it back to the client in ACK_ECN
+     frames, and it puts the mark that ngtcp2 asks for on each packet it
+     sends.
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: With ListenCoresBucketsRatio and more than one QUIC socket,
+     the server sends disable_active_migration, so clients do not move a
+     connection to a new address that another child can get.
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: The parent makes the key for Retry and stateless reset tokens
+     and keeps it across graceful restarts. A Retry token from a child is now
+     good in every other child and in the next generation.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_reuseport_buckets.txt

@@ -0,0 +1,6 @@
+  *) mod_http3: On Linux, with ListenCoresBucketsRatio, the parent binds one
+     SO_REUSEPORT QUIC socket per listener bucket and one child serves each,
+     so more than one child serves HTTP/3. The sockets stay bound when
+     children start and stop, so the kernel keeps each peer on the same
+     socket.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_udp_gso_gro.txt

@@ -0,0 +1,5 @@
+  *) mod_http3: Linux: QUIC packets to one peer now go out in one sendmsg with
+     UDP_SEGMENT (GSO), and reads use UDP_GRO. Without kernel support the
+     module sends one packet at a time. A stream read now sends the new
+     flow-control window at once, so a blocked client does not wait.
+     [Alexander Gerasimov <codeguard gmail.com>]

modules/http3/config2.m4

@@ -31,12 +31,12 @@ h3_server.lo dnl
 h3_session.lo dnl
 h3_socket.lo dnl
 h3_socket_os.lo dnl
+h3_socket_bpf.lo dnl
 h3_stream.lo dnl
 h3_threads.lo dnl
 h3q.lo dnl
 h3q_conn.lo dnl
 h3q_stream.lo dnl
-h3q_addr.lo dnl
 h3q_tls.lo dnl
 "
 
@@ -159,11 +159,57 @@ AC_DEFUN([APACHE_CHECK_NGHTTP3],[
 ])
 
 
+dnl
+dnl APACHE_CHECK_NGTCP2
+dnl
+dnl Configure for ngtcp2 and its OpenSSL crypto helper, giving preference
+dnl to "--with-ngtcp2=<path>" if it was specified.
+dnl
+AC_DEFUN([APACHE_CHECK_NGTCP2],[
+  AC_CACHE_CHECK([for ngtcp2], [ac_cv_ngtcp2], [
+    ac_cv_ngtcp2=no
+    ap_ngtcp2_base=""
+    AC_ARG_WITH(ngtcp2, APACHE_HELP_STRING(--with-ngtcp2=PATH, ngtcp2 installation directory), [
+      if test "x$withval" != "xyes" -a "x$withval" != "x"; then
+        ap_ngtcp2_base="`cd $withval ; pwd`"
+      fi
+    ])
+
+    saved_CPPFLAGS="$CPPFLAGS"
+    saved_PKG_CONFIG_PATH="$PKG_CONFIG_PATH"
+    if test "x$ap_ngtcp2_base" != "x"; then
+      PKG_CONFIG_PATH="${ap_ngtcp2_base}/lib/pkgconfig:${ap_ngtcp2_base}/lib64/pkgconfig${PKG_CONFIG_PATH+:}${PKG_CONFIG_PATH}"
+      export PKG_CONFIG_PATH
+    fi
+    if test -n "$PKGCONFIG" && $PKGCONFIG --exists libngtcp2 libngtcp2_crypto_ossl; then
+      ap_ngtcp2_cflags="`$PKGCONFIG --cflags libngtcp2 libngtcp2_crypto_ossl`"
+      ap_ngtcp2_libs="`$PKGCONFIG --libs libngtcp2 libngtcp2_crypto_ossl`"
+    elif test "x$ap_ngtcp2_base" != "x"; then
+      ap_ngtcp2_cflags="-I$ap_ngtcp2_base/include"
+      ap_ngtcp2_libs="-L$ap_ngtcp2_base/lib -lngtcp2_crypto_ossl -lngtcp2"
+    else
+      ap_ngtcp2_cflags=""
+      ap_ngtcp2_libs="-lngtcp2_crypto_ossl -lngtcp2"
+    fi
+    PKG_CONFIG_PATH="$saved_PKG_CONFIG_PATH"
+
+    CPPFLAGS="$CPPFLAGS $ap_ngtcp2_cflags"
+    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <ngtcp2/ngtcp2.h>]],[[
+#if NGTCP2_VERSION_NUM < 0x011601
+#error "ngtcp2 1.22.1 or later is required"
+#endif]])],
+      [ac_cv_ngtcp2=yes
+       APR_ADDTO(MOD_CFLAGS, [$ap_ngtcp2_cflags])
+       APR_ADDTO(MOD_LDFLAGS, [$ap_ngtcp2_libs])])
+    CPPFLAGS="$saved_CPPFLAGS"
+  ])
+])
+
 dnl # hook module into the Autoconf mechanism (--enable-http3)
 APACHE_MODULE(http3, [HTTP/3 protocol handling in addition to HTTP protocol
-handling. Implemented by mod_http3. This module requires a libnghttp3
-installation and OpenSSL with QUIC support.
-See --with-nghttp3 on how to manage non-standard locations. This module
+handling. Implemented by mod_http3. This module requires libnghttp3,
+libngtcp2 with its OpenSSL crypto helper, and OpenSSL with the QUIC TLS API.
+See --with-nghttp3 and --with-ngtcp2 on how to manage non-standard locations. This module
 is usually linked shared and requires loading. ], $http3_objs, , no, [
     APACHE_CHECK_OPENSSL
     if test "$ac_cv_openssl" = "yes" ; then
@@ -174,7 +220,8 @@ is usually linked shared and requires loading. ], $http3_objs, , no, [
 
     if test "$enable_http3" != "no" ; then
         APACHE_CHECK_NGHTTP3
-        if test "$ac_cv_nghttp3" = "yes" ; then
+        APACHE_CHECK_NGTCP2
+        if test "$ac_cv_nghttp3" = "yes" -a "$ac_cv_ngtcp2" = "yes" ; then
             APR_ADDTO(MOD_CFLAGS, [-I\$(top_srcdir)/$modpath_current/include])
             if test "x$enable_http3" = "xshared"; then
                # The only symbol which needs to be exported is the module

modules/http3/h3_config.c

@@ -34,6 +34,7 @@
 #include "h3_config.h"
 #include "h3_os.h"
 #include "h3_request.h"
+#include "h3_server.h"
 #include "mod_http3.h"
 #include "quic/detail/h3q_tls.h"
 
@@ -72,7 +73,9 @@ void* h3_merge_server_config(apr_pool_t* p, void* base_conf, void* new_conf)
     merged->h3_handshake_timeout = new->h3_handshake_timeout ? new->h3_handshake_timeout : base->h3_handshake_timeout;
     merged->h3_idle_timeout = new->h3_idle_timeout ? new->h3_idle_timeout : base->h3_idle_timeout;
     merged->h3_socket_buffer_size = new->h3_socket_buffer_size ? new->h3_socket_buffer_size : base->h3_socket_buffer_size;
+    merged->h3_max_window = new->h3_max_window ? new->h3_max_window : base->h3_max_window;
     merged->h3_session_tickets = new->h3_session_tickets != H3_FLAG_UNSET ? new->h3_session_tickets : base->h3_session_tickets;
+    merged->h3_early_data = new->h3_early_data != H3_FLAG_UNSET ? new->h3_early_data : base->h3_early_data;
     merged->h3_stream_timeout = new->h3_stream_timeout ? new->h3_stream_timeout : base->h3_stream_timeout;
     merged->h3_max_stream_errors = new->h3_max_stream_errors ? new->h3_max_stream_errors : base->h3_max_stream_errors;
     merged->h3_qpack_capacity_set = new->h3_qpack_capacity_set ? new->h3_qpack_capacity_set : base->h3_qpack_capacity_set;
@@ -197,6 +200,19 @@ static const char* set_h3_stream_buffer_size(cmd_parms* cmd, void* dummy H3_UNUS
     return NULL;
 }
 
+static const char* set_h3_max_window(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg)
+{
+    apr_uint64_t val = 0;
+    if (apr_cstr_atoui64(&val, arg) != APR_SUCCESS || val < H3_MAX_WINDOW_MIN || val > H3_MAX_WINDOW_MAX)
+    {
+        return apr_psprintf(cmd->pool, "H3MaxWindow: '%s' must be a byte count from %lu to %lu", arg, (unsigned long)H3_MAX_WINDOW_MIN, (unsigned long)H3_MAX_WINDOW_MAX);
+    }
+    h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module);
+    CHECK(conf);
+    conf->h3_max_window = (apr_size_t)val;
+    return NULL;
+}
+
 static const char* set_h3_socket_buffer_size(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg)
 {
     if (!arg || !*arg)
@@ -519,6 +535,14 @@ static const char* set_h3_session_tickets(cmd_parms* cmd, void* dummy H3_UNUSED,
     return NULL;
 }
 
+static const char* set_h3_early_data(cmd_parms* cmd, void* dummy H3_UNUSED, int flag)
+{
+    h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module);
+    CHECK(conf);
+    conf->h3_early_data = flag ? H3_FLAG_ON : H3_FLAG_OFF;
+    return NULL;
+}
+
 static const char* set_h3_alt_svc(cmd_parms* cmd, void* dummy H3_UNUSED, int flag)
 {
     h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module);
@@ -677,6 +701,10 @@ int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp,
             {
                 vc->h3_socket_buffer_size = H3_SOCKET_BUFFER_SIZE_DEFAULT;
             }
+            if (vc->h3_max_window == 0)
+            {
+                vc->h3_max_window = H3_MAX_WINDOW_DEFAULT;
+            }
             if (vc->h3_max_stream_errors == 0)
             {
                 vc->h3_max_stream_errors = H3_MAX_STREAM_ERRORS_DEFAULT;
@@ -726,6 +754,10 @@ int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp,
             {
                 vc->h3_session_tickets = H3_FLAG_ON;
             }
+            if (vc->h3_early_data == H3_FLAG_UNSET)
+            {
+                vc->h3_early_data = H3_FLAG_OFF;
+            }
             if (vc->h3_alt_svc_max_age == 0)
             {
                 vc->h3_alt_svc_max_age = H3_ALT_SVC_MAX_AGE_DEFAULT;
@@ -756,6 +788,7 @@ int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp,
     SSL_CTX_set_cert_cb(conf->ssl_ctx, h3_sni_select_cert, sni);
 
     h3_request_init();
+    h3_server_post_config(s);
     return OK;
 }
 
@@ -784,7 +817,9 @@ const command_rec h3_cmds[] = {
     AP_INIT_TAKE1("H3MaxResponseBodySize", set_h3_max_response_body_size, NULL, RSRC_CONF, "Maximum HTTP/3 response body size in bytes; an explicit limit enables bounded whole-response buffering (default: unlimited streaming)"),
     AP_INIT_FLAG("H3AddressValidation", set_h3_address_validation, NULL, RSRC_CONF, "Whether to validate client addresses with a QUIC Retry packet before accepting a connection (default: on)"),
     AP_INIT_TAKE1("H3SocketBufferSize", set_h3_socket_buffer_size, NULL, RSRC_CONF, "Bytes requested for the QUIC socket send and receive buffers; the OS may grant less (default: 2097152)"),
+    AP_INIT_TAKE1("H3MaxWindow", set_h3_max_window, NULL, RSRC_CONF, "Largest flow-control window, per connection and per stream, that autotuning may grow a client upload window to (default: 6291456)"),
     AP_INIT_FLAG("H3SessionTickets", set_h3_session_tickets, NULL, RSRC_CONF, "Whether to issue TLS session tickets so returning clients can resume instead of running a full handshake (default: on)"),
+    AP_INIT_FLAG("H3EarlyData", set_h3_early_data, NULL, RSRC_CONF, "Whether to accept 0-RTT request data on resumed connections; unsafe methods wait for the handshake (default: off)"),
     AP_INIT_TAKE1("H3StreamTimeout", set_h3_stream_timeout, NULL, RSRC_CONF, "Seconds a response may make no progress before the stream is aborted (default: the server Timeout)"),
     AP_INIT_TAKE1("H3MaxStreamErrors", set_h3_max_stream_errors, NULL, RSRC_CONF, "Stream errors one connection may cause before it is closed with H3_EXCESSIVE_LOAD (default: 8)"),
     AP_INIT_TAKE1("H3QpackTableCapacity", set_h3_qpack_table_capacity, NULL, RSRC_CONF, "Bytes of QPACK dynamic table a client may use for request headers; 0 disables it (default: 4096)"),