mod_http3: bind the QUIC socket in the parent, one SO_REUSEPORT socket per listener bucket

changes-entries/h3_mod_ssl_certificate.txt

@@ -0,0 +1,24 @@
+  *) mod_http3: Load the HTTP/3 certificate and key in post_config, while
+     httpd still runs privileged, so a root-only key no longer fails in the
+     unprivileged child and both mod_ssl and mod_http3 read the same files.
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: A host with "h3" in Protocols now serves HTTP/3 with the
+     certificate mod_ssl resolved for it (SSLCertificateFile and mod_md
+     alike), the way mod_http2 rides mod_ssl. H3CertificatePath and
+     H3CertificateKeyPath are removed; drop them from existing configurations.
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: Select the HTTP/3 certificate by SNI, so each virtual host on
+     a shared port serves its own certificate instead of the first host's.
+     Names are matched as httpd matches virtual hosts, wildcard ServerAlias
+     included. A name that matches no host gets the certificate of the first
+     virtual host defined, as with mod_ssl.
+     [Alexander Gerasimov <codeguard gmail.com>]
+
+  *) mod_http3: Give every virtual host its own mod_http3 configuration
+     (AP_MODULE_FLAG_ALWAYS_MERGE). A host without H3 directives used to share
+     the main server's, so per-host settings such as H3AltSvc and
+     H3AltSvcMaxAge were silently ignored and hosts could not carry their own
+     certificate.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_parent_socket.txt

@@ -0,0 +1,5 @@
+  *) mod_http3: On Linux the parent binds the QUIC socket while it still runs
+     as root, so H3Port 443 works without CAP_NET_BIND_SERVICE. The socket
+     stays bound across graceful restarts and the children inherit it; one
+     child at a time serves it.
+     [Alexander Gerasimov <codeguard gmail.com>]

changes-entries/h3_reuseport_buckets.txt

@@ -0,0 +1,6 @@
+  *) mod_http3: On Linux, with ListenCoresBucketsRatio, the parent binds one
+     SO_REUSEPORT QUIC socket per listener bucket and one child serves each,
+     so more than one child serves HTTP/3. The sockets stay bound when
+     children start and stop, so the kernel keeps each peer on the same
+     socket.
+     [Alexander Gerasimov <codeguard gmail.com>]

modules/http3/h3_config.c

@@ -21,6 +21,7 @@
 #include <http_core.h>
 #include <http_log.h>
 #include <http_main.h>
+#include <http_protocol.h>
 
 #include <apr_cstr.h>
 #include <apr_pools.h>
@@ -33,7 +34,9 @@
 #include "h3_config.h"
 #include "h3_os.h"
 #include "h3_request.h"
+#include "h3_server.h"
 #include "mod_http3.h"
+#include "quic/detail/h3q_tls.h"
 
 apr_port_t get_server_port(const server_rec* s)
 {
@@ -58,8 +61,6 @@ void* h3_merge_server_config(apr_pool_t* p, void* base_conf, void* new_conf)
     h3_server_conf* base = (h3_server_conf*)base_conf;
     h3_server_conf* new = (h3_server_conf*)new_conf;
 
-    merged->h3_cert_path = new->h3_cert_path ? new->h3_cert_path : base->h3_cert_path;
-    merged->h3_key_path = new->h3_key_path ? new->h3_key_path : base->h3_key_path;
     merged->h3_port = new->h3_port ? new->h3_port : base->h3_port;
     merged->h3_max_concurrent_streams = new->h3_max_concurrent_streams ? new->h3_max_concurrent_streams : base->h3_max_concurrent_streams;
     merged->h3_max_connections = new->h3_max_connections ? new->h3_max_connections : base->h3_max_connections;
@@ -93,28 +94,6 @@ static int mpm_query(int code)
     return ap_mpm_query(code, &value) == APR_SUCCESS ? value : -1;
 }
 
-static const char* set_string(cmd_parms* cmd, const char* arg, const char* field)
-{
-    h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module);
-    if (!conf)
-    {
-        ap_log_error(APLOG_MARK, APLOG_ERR, 0, cmd->server, "mod_http3: server config missing in directive");
-        return "mod_http3: internal error: no server config";
-    }
-    *(const char**)((char*)conf + (apr_size_t)field) = apr_pstrdup(cmd->pool, arg);
-    return NULL;
-}
-
-static const char* set_h3_cert_path(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg)
-{
-    return set_string(cmd, arg, (const char*)offsetof(h3_server_conf, h3_cert_path));
-}
-
-static const char* set_h3_key_path(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg)
-{
-    return set_string(cmd, arg, (const char*)offsetof(h3_server_conf, h3_key_path));
-}
-
 static const char* set_h3_port(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg)
 {
     if (!arg || !*arg)
@@ -583,22 +562,101 @@ static const char* set_h3_alt_svc_max_age(cmd_parms* cmd, void* dummy H3_UNUSED,
     return NULL;
 }
 
-int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, server_rec* s)
+static apr_status_t ssl_ctx_cleanup(void* data)
+{
+    SSL_CTX_free(data);
+    return APR_SUCCESS;
+}
+
+int h3_ssl_add_cert_files(server_rec* s, apr_pool_t* p, apr_array_header_t* cert_files, apr_array_header_t* key_files)
 {
+    CHECK(s && p && cert_files && key_files, return DECLINED;);
+    h3_server_conf* conf = ap_get_module_config(s->module_config, &http3_module);
+    if (!conf || ap_state_query(AP_SQ_MAIN_STATE) == AP_SQ_MS_CREATE_PRE_CONFIG || !ap_is_allowed_protocol(NULL, NULL, s, "h3") || cert_files->nelts == 0)
+    {
+        return DECLINED;
+    }
+    /* Still privileged here, so a root-only key loads the way it does for mod_ssl. */
+    char err[H3Q_ERRLEN] = {0};
+    conf->ssl_ctx = h3q_tls_ctx_create((const char* const*)cert_files->elts, (size_t)cert_files->nelts, (const char* const*)key_files->elts, (size_t)key_files->nelts, conf->h3_session_tickets != H3_FLAG_OFF, err, sizeof(err));
+    if (!conf->ssl_ctx)
+    {
+        ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: %s", err);
+        return HTTP_INTERNAL_SERVER_ERROR;
+    }
+    apr_pool_cleanup_register(p, conf->ssl_ctx, ssl_ctx_cleanup, apr_pool_cleanup_null);
+    ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "mod_http3: serving HTTP/3 with mod_ssl certificate %s", APR_ARRAY_IDX(cert_files, 0, const char*));
+    return DECLINED;
+}
+
+/** One HTTP/3 host name and the TLS context that carries its certificate. */
+typedef struct
+{
+    const char* name;
+    SSL_CTX* ctx;
+} h3_sni_host;
+
+/// cert_cb: serve each host its own certificate over one listener, selected by SNI.
+/// SSL_set_SSL_CTX does not switch the certificate of a QUIC connection; applying
+/// the matched host's certificate, key and chain to the connection does.
+static int h3_sni_select_cert(SSL* ssl, void* arg)
+{
+    const apr_array_header_t* hosts = arg;
+    const char* sni = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name);
+    for (int i = 0; sni && i < hosts->nelts; i++)
+    {
+        const h3_sni_host* h = &APR_ARRAY_IDX(hosts, i, h3_sni_host);
+        /* httpd's ServerAlias matcher: wildcards for wild_names, exact otherwise. */
+        if (ap_strcasecmp_match(sni, h->name) == 0)
+        {
+            STACK_OF(X509)* chain = NULL;
+            SSL_CTX_get0_chain_certs(h->ctx, &chain);
+            return SSL_use_certificate(ssl, SSL_CTX_get0_certificate(h->ctx)) == 1 && SSL_use_PrivateKey(ssl, SSL_CTX_get0_privatekey(h->ctx)) == 1 && (!chain || SSL_set1_chain(ssl, chain) == 1);
+        }
+    }
+    return 1; /* no match: the listener's own certificate */
+}
+
+int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, server_rec* s)
+{
+    CHECK(p);
     CHECK(ptemp);
     CHECK(s);
     h3_server_conf* conf = NULL;
+    server_rec* owner = NULL;
 
     if (ap_state_query(AP_SQ_MAIN_STATE) == AP_SQ_MS_CREATE_PRE_CONFIG)
     {
         return OK;
     }
 
+    /* Names of every HTTP/3 host, so the listener can pick a certificate by SNI. */
+    apr_array_header_t* sni = apr_array_make(p, 4, sizeof(h3_sni_host));
+
     for (server_rec* vs = s; vs; vs = vs->next)
     {
         h3_server_conf* vc = ap_get_module_config(vs->module_config, &http3_module);
-        if (vc->h3_cert_path && vc->h3_key_path)
+        if (vc->ssl_ctx)
         {
+            if (vs->server_hostname)
+            {
+                h3_sni_host* e = apr_array_push(sni);
+                e->name = vs->server_hostname;
+                e->ctx = vc->ssl_ctx;
+            }
+            /* ServerAlias: httpd keeps exact names in names and wildcards in wild_names. */
+            for (int i = 0; vs->names && i < vs->names->nelts; i++)
+            {
+                h3_sni_host* e = apr_array_push(sni);
+                e->name = APR_ARRAY_IDX(vs->names, i, const char*);
+                e->ctx = vc->ssl_ctx;
+            }
+            for (int i = 0; vs->wild_names && i < vs->wild_names->nelts; i++)
+            {
+                h3_sni_host* e = apr_array_push(sni);
+                e->name = APR_ARRAY_IDX(vs->wild_names, i, const char*);
+                e->ctx = vc->ssl_ctx;
+            }
             vc->host_port = get_server_port(vs);
             if (vc->h3_port == 0)
             {
@@ -681,34 +739,25 @@ int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool
             {
                 vc->h3_idle_timeout = H3_IDLE_TIMEOUT_DEFAULT;
             }
+            /* httpd prepends virtual hosts, so the last match is the first one defined:
+             * it owns the listener, as it is mod_ssl's default; the rest still advertise it. */
             conf = vc;
-            break;
+            owner = vs;
         }
     }
 
-    CHECK(conf && conf->h3_cert_path && conf->h3_key_path, return HTTP_INTERNAL_SERVER_ERROR;);
-
-    /* Validate cert and key files are readable */
-    apr_file_t* f = NULL;
-    if (apr_file_open(&f, conf->h3_cert_path, APR_READ, APR_OS_DEFAULT, ptemp) != APR_SUCCESS)
+    if (!conf)
     {
-        ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: H3CertificatePath not readable: %s", conf->h3_cert_path);
+        ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: no host serves HTTP/3: add h3 to Protocols on a host with SSLEngine on");
         return HTTP_INTERNAL_SERVER_ERROR;
     }
-    apr_file_close(f);
-    f = NULL;
+    ap_log_error(APLOG_MARK, APLOG_INFO, 0, owner, "h3_post_config: pid=%d h3_port=%d mpm=%s threaded=%d forked=%d max_threads=%d", h3_getpid(), (int)conf->h3_port, ap_show_mpm(), mpm_query(AP_MPMQ_IS_THREADED), mpm_query(AP_MPMQ_IS_FORKED), mpm_query(AP_MPMQ_MAX_THREADS));
 
-    if (apr_file_open(&f, conf->h3_key_path, APR_READ, APR_OS_DEFAULT, ptemp) != APR_SUCCESS)
-    {
-        ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: H3CertificateKeyPath not readable: %s", conf->h3_key_path);
-        return HTTP_INTERNAL_SERVER_ERROR;
-    }
-    apr_file_close(f);
+    /* The listener serves conf's certificate by default and swaps by SNI. */
+    SSL_CTX_set_cert_cb(conf->ssl_ctx, h3_sni_select_cert, sni);
 
     h3_request_init();
-
-    ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "h3_post_config: pid=%d cert=%s key=%s h3_port=%d mpm=%s threaded=%d forked=%d max_threads=%d", h3_getpid(), conf->h3_cert_path, conf->h3_key_path, (int)conf->h3_port, ap_show_mpm(), mpm_query(AP_MPMQ_IS_THREADED), mpm_query(AP_MPMQ_IS_FORKED),
-                 mpm_query(AP_MPMQ_MAX_THREADS));
+    h3_server_post_config(s);
     return OK;
 }
 
@@ -725,8 +774,6 @@ void* h3_merge_dir_config(apr_pool_t* p H3_UNUSED, void* base, void* add H3_UNUS
 }
 
 const command_rec h3_cmds[] = {
-    AP_INIT_TAKE1("H3CertificatePath", set_h3_cert_path, NULL, RSRC_CONF, "Path to the SSL certificate file for HTTP/3"),
-    AP_INIT_TAKE1("H3CertificateKeyPath", set_h3_key_path, NULL, RSRC_CONF, "Path to the SSL certificate key file for HTTP/3"),
     AP_INIT_TAKE1("H3Port", set_h3_port, NULL, RSRC_CONF, "UDP port to listen on for QUIC/HTTP-3 (default: same as main server)"),
     AP_INIT_TAKE1("H3MaxConcurrentStreams", set_h3_max_concurrent_streams, NULL, RSRC_CONF, "Maximum number of concurrent HTTP/3 streams per connection (default: 100)"),
     AP_INIT_TAKE1("H3MaxConnections", set_h3_max_connections, NULL, RSRC_CONF, "Maximum concurrent QUIC/HTTP/3 connections per child process (default: 256)"),

modules/http3/h3_hooks.c

@@ -82,7 +82,7 @@ int h3_hook_fixups(request_rec* r)
 
     h3_server_conf* conf = ap_get_module_config(r->server->module_config, &http3_module);
 
-    if (!conf || !conf->h3_cert_path || !conf->h3_key_path || conf->h3_port == 0)
+    if (!conf || !conf->ssl_ctx || conf->h3_port == 0)
     {
         return DECLINED;
     }

modules/http3/h3_io.c

@@ -131,10 +131,8 @@ apr_status_t h3_io_listen_start(apr_pool_t* pchild, server_rec* s, h3_server_con
 
     char qerr[H3Q_ERRLEN] = {0};
     h3q_config qcfg = {
-        .cert_path = conf->h3_cert_path,
-        .key_path = conf->h3_key_path,
+        .ssl_ctx = conf->ssl_ctx,
         .address_validation = (conf->h3_address_validation != H3_FLAG_OFF),
-        .session_tickets = (conf->h3_session_tickets != H3_FLAG_OFF),
     };
     io->qengine = h3q_engine_create(&qcfg, udp_fd, qerr, sizeof(qerr));
     if (!io->qengine)