diff --git a/changes-entries/h3_mod_ssl_certificate.txt b/changes-entries/h3_mod_ssl_certificate.txt new file mode 100644 index 00000000000..ab44463450c --- /dev/null +++ b/changes-entries/h3_mod_ssl_certificate.txt @@ -0,0 +1,24 @@ + *) mod_http3: Load the HTTP/3 certificate and key in post_config, while + httpd still runs privileged, so a root-only key no longer fails in the + unprivileged child and both mod_ssl and mod_http3 read the same files. + [Alexander Gerasimov ] + + *) mod_http3: A host with "h3" in Protocols now serves HTTP/3 with the + certificate mod_ssl resolved for it (SSLCertificateFile and mod_md + alike), the way mod_http2 rides mod_ssl. H3CertificatePath and + H3CertificateKeyPath are removed; drop them from existing configurations. + [Alexander Gerasimov ] + + *) mod_http3: Select the HTTP/3 certificate by SNI, so each virtual host on + a shared port serves its own certificate instead of the first host's. + Names are matched as httpd matches virtual hosts, wildcard ServerAlias + included. A name that matches no host gets the certificate of the first + virtual host defined, as with mod_ssl. + [Alexander Gerasimov ] + + *) mod_http3: Give every virtual host its own mod_http3 configuration + (AP_MODULE_FLAG_ALWAYS_MERGE). A host without H3 directives used to share + the main server's, so per-host settings such as H3AltSvc and + H3AltSvcMaxAge were silently ignored and hosts could not carry their own + certificate. + [Alexander Gerasimov ] diff --git a/changes-entries/h3_parent_socket.txt b/changes-entries/h3_parent_socket.txt new file mode 100644 index 00000000000..664b0aad378 --- /dev/null +++ b/changes-entries/h3_parent_socket.txt @@ -0,0 +1,5 @@ + *) mod_http3: On Linux the parent binds the QUIC socket while it still runs + as root, so H3Port 443 works without CAP_NET_BIND_SERVICE. The socket + stays bound across graceful restarts and the children inherit it; one + child at a time serves it. + [Alexander Gerasimov ] diff --git a/changes-entries/h3_reuseport_buckets.txt b/changes-entries/h3_reuseport_buckets.txt new file mode 100644 index 00000000000..6943718875a --- /dev/null +++ b/changes-entries/h3_reuseport_buckets.txt @@ -0,0 +1,6 @@ + *) mod_http3: On Linux, with ListenCoresBucketsRatio, the parent binds one + SO_REUSEPORT QUIC socket per listener bucket and one child serves each, + so more than one child serves HTTP/3. The sockets stay bound when + children start and stop, so the kernel keeps each peer on the same + socket. + [Alexander Gerasimov ] diff --git a/modules/http3/h3_config.c b/modules/http3/h3_config.c index cde39bd8e31..76e1d3e1dcd 100644 --- a/modules/http3/h3_config.c +++ b/modules/http3/h3_config.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -33,7 +34,9 @@ #include "h3_config.h" #include "h3_os.h" #include "h3_request.h" +#include "h3_server.h" #include "mod_http3.h" +#include "quic/detail/h3q_tls.h" apr_port_t get_server_port(const server_rec* s) { @@ -58,8 +61,6 @@ void* h3_merge_server_config(apr_pool_t* p, void* base_conf, void* new_conf) h3_server_conf* base = (h3_server_conf*)base_conf; h3_server_conf* new = (h3_server_conf*)new_conf; - merged->h3_cert_path = new->h3_cert_path ? new->h3_cert_path : base->h3_cert_path; - merged->h3_key_path = new->h3_key_path ? new->h3_key_path : base->h3_key_path; merged->h3_port = new->h3_port ? new->h3_port : base->h3_port; merged->h3_max_concurrent_streams = new->h3_max_concurrent_streams ? new->h3_max_concurrent_streams : base->h3_max_concurrent_streams; merged->h3_max_connections = new->h3_max_connections ? new->h3_max_connections : base->h3_max_connections; @@ -93,28 +94,6 @@ static int mpm_query(int code) return ap_mpm_query(code, &value) == APR_SUCCESS ? value : -1; } -static const char* set_string(cmd_parms* cmd, const char* arg, const char* field) -{ - h3_server_conf* conf = ap_get_module_config(cmd->server->module_config, &http3_module); - if (!conf) - { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, cmd->server, "mod_http3: server config missing in directive"); - return "mod_http3: internal error: no server config"; - } - *(const char**)((char*)conf + (apr_size_t)field) = apr_pstrdup(cmd->pool, arg); - return NULL; -} - -static const char* set_h3_cert_path(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg) -{ - return set_string(cmd, arg, (const char*)offsetof(h3_server_conf, h3_cert_path)); -} - -static const char* set_h3_key_path(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg) -{ - return set_string(cmd, arg, (const char*)offsetof(h3_server_conf, h3_key_path)); -} - static const char* set_h3_port(cmd_parms* cmd, void* dummy H3_UNUSED, const char* arg) { if (!arg || !*arg) @@ -583,22 +562,101 @@ static const char* set_h3_alt_svc_max_age(cmd_parms* cmd, void* dummy H3_UNUSED, return NULL; } -int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, server_rec* s) +static apr_status_t ssl_ctx_cleanup(void* data) +{ + SSL_CTX_free(data); + return APR_SUCCESS; +} + +int h3_ssl_add_cert_files(server_rec* s, apr_pool_t* p, apr_array_header_t* cert_files, apr_array_header_t* key_files) { + CHECK(s && p && cert_files && key_files, return DECLINED;); + h3_server_conf* conf = ap_get_module_config(s->module_config, &http3_module); + if (!conf || ap_state_query(AP_SQ_MAIN_STATE) == AP_SQ_MS_CREATE_PRE_CONFIG || !ap_is_allowed_protocol(NULL, NULL, s, "h3") || cert_files->nelts == 0) + { + return DECLINED; + } + /* Still privileged here, so a root-only key loads the way it does for mod_ssl. */ + char err[H3Q_ERRLEN] = {0}; + conf->ssl_ctx = h3q_tls_ctx_create((const char* const*)cert_files->elts, (size_t)cert_files->nelts, (const char* const*)key_files->elts, (size_t)key_files->nelts, conf->h3_session_tickets != H3_FLAG_OFF, err, sizeof(err)); + if (!conf->ssl_ctx) + { + ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: %s", err); + return HTTP_INTERNAL_SERVER_ERROR; + } + apr_pool_cleanup_register(p, conf->ssl_ctx, ssl_ctx_cleanup, apr_pool_cleanup_null); + ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "mod_http3: serving HTTP/3 with mod_ssl certificate %s", APR_ARRAY_IDX(cert_files, 0, const char*)); + return DECLINED; +} + +/** One HTTP/3 host name and the TLS context that carries its certificate. */ +typedef struct +{ + const char* name; + SSL_CTX* ctx; +} h3_sni_host; + +/// cert_cb: serve each host its own certificate over one listener, selected by SNI. +/// SSL_set_SSL_CTX does not switch the certificate of a QUIC connection; applying +/// the matched host's certificate, key and chain to the connection does. +static int h3_sni_select_cert(SSL* ssl, void* arg) +{ + const apr_array_header_t* hosts = arg; + const char* sni = SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); + for (int i = 0; sni && i < hosts->nelts; i++) + { + const h3_sni_host* h = &APR_ARRAY_IDX(hosts, i, h3_sni_host); + /* httpd's ServerAlias matcher: wildcards for wild_names, exact otherwise. */ + if (ap_strcasecmp_match(sni, h->name) == 0) + { + STACK_OF(X509)* chain = NULL; + SSL_CTX_get0_chain_certs(h->ctx, &chain); + return SSL_use_certificate(ssl, SSL_CTX_get0_certificate(h->ctx)) == 1 && SSL_use_PrivateKey(ssl, SSL_CTX_get0_privatekey(h->ctx)) == 1 && (!chain || SSL_set1_chain(ssl, chain) == 1); + } + } + return 1; /* no match: the listener's own certificate */ +} + +int h3_post_config(apr_pool_t* p, apr_pool_t* plog H3_UNUSED, apr_pool_t* ptemp, server_rec* s) +{ + CHECK(p); CHECK(ptemp); CHECK(s); h3_server_conf* conf = NULL; + server_rec* owner = NULL; if (ap_state_query(AP_SQ_MAIN_STATE) == AP_SQ_MS_CREATE_PRE_CONFIG) { return OK; } + /* Names of every HTTP/3 host, so the listener can pick a certificate by SNI. */ + apr_array_header_t* sni = apr_array_make(p, 4, sizeof(h3_sni_host)); + for (server_rec* vs = s; vs; vs = vs->next) { h3_server_conf* vc = ap_get_module_config(vs->module_config, &http3_module); - if (vc->h3_cert_path && vc->h3_key_path) + if (vc->ssl_ctx) { + if (vs->server_hostname) + { + h3_sni_host* e = apr_array_push(sni); + e->name = vs->server_hostname; + e->ctx = vc->ssl_ctx; + } + /* ServerAlias: httpd keeps exact names in names and wildcards in wild_names. */ + for (int i = 0; vs->names && i < vs->names->nelts; i++) + { + h3_sni_host* e = apr_array_push(sni); + e->name = APR_ARRAY_IDX(vs->names, i, const char*); + e->ctx = vc->ssl_ctx; + } + for (int i = 0; vs->wild_names && i < vs->wild_names->nelts; i++) + { + h3_sni_host* e = apr_array_push(sni); + e->name = APR_ARRAY_IDX(vs->wild_names, i, const char*); + e->ctx = vc->ssl_ctx; + } vc->host_port = get_server_port(vs); if (vc->h3_port == 0) { @@ -681,34 +739,25 @@ int h3_post_config(apr_pool_t* p H3_UNUSED, apr_pool_t* plog H3_UNUSED, apr_pool { vc->h3_idle_timeout = H3_IDLE_TIMEOUT_DEFAULT; } + /* httpd prepends virtual hosts, so the last match is the first one defined: + * it owns the listener, as it is mod_ssl's default; the rest still advertise it. */ conf = vc; - break; + owner = vs; } } - CHECK(conf && conf->h3_cert_path && conf->h3_key_path, return HTTP_INTERNAL_SERVER_ERROR;); - - /* Validate cert and key files are readable */ - apr_file_t* f = NULL; - if (apr_file_open(&f, conf->h3_cert_path, APR_READ, APR_OS_DEFAULT, ptemp) != APR_SUCCESS) + if (!conf) { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: H3CertificatePath not readable: %s", conf->h3_cert_path); + ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: no host serves HTTP/3: add h3 to Protocols on a host with SSLEngine on"); return HTTP_INTERNAL_SERVER_ERROR; } - apr_file_close(f); - f = NULL; + ap_log_error(APLOG_MARK, APLOG_INFO, 0, owner, "h3_post_config: pid=%d h3_port=%d mpm=%s threaded=%d forked=%d max_threads=%d", h3_getpid(), (int)conf->h3_port, ap_show_mpm(), mpm_query(AP_MPMQ_IS_THREADED), mpm_query(AP_MPMQ_IS_FORKED), mpm_query(AP_MPMQ_MAX_THREADS)); - if (apr_file_open(&f, conf->h3_key_path, APR_READ, APR_OS_DEFAULT, ptemp) != APR_SUCCESS) - { - ap_log_error(APLOG_MARK, APLOG_ERR, 0, s, "mod_http3: H3CertificateKeyPath not readable: %s", conf->h3_key_path); - return HTTP_INTERNAL_SERVER_ERROR; - } - apr_file_close(f); + /* The listener serves conf's certificate by default and swaps by SNI. */ + SSL_CTX_set_cert_cb(conf->ssl_ctx, h3_sni_select_cert, sni); h3_request_init(); - - ap_log_error(APLOG_MARK, APLOG_INFO, 0, s, "h3_post_config: pid=%d cert=%s key=%s h3_port=%d mpm=%s threaded=%d forked=%d max_threads=%d", h3_getpid(), conf->h3_cert_path, conf->h3_key_path, (int)conf->h3_port, ap_show_mpm(), mpm_query(AP_MPMQ_IS_THREADED), mpm_query(AP_MPMQ_IS_FORKED), - mpm_query(AP_MPMQ_MAX_THREADS)); + h3_server_post_config(s); return OK; } @@ -725,8 +774,6 @@ void* h3_merge_dir_config(apr_pool_t* p H3_UNUSED, void* base, void* add H3_UNUS } const command_rec h3_cmds[] = { - AP_INIT_TAKE1("H3CertificatePath", set_h3_cert_path, NULL, RSRC_CONF, "Path to the SSL certificate file for HTTP/3"), - AP_INIT_TAKE1("H3CertificateKeyPath", set_h3_key_path, NULL, RSRC_CONF, "Path to the SSL certificate key file for HTTP/3"), AP_INIT_TAKE1("H3Port", set_h3_port, NULL, RSRC_CONF, "UDP port to listen on for QUIC/HTTP-3 (default: same as main server)"), AP_INIT_TAKE1("H3MaxConcurrentStreams", set_h3_max_concurrent_streams, NULL, RSRC_CONF, "Maximum number of concurrent HTTP/3 streams per connection (default: 100)"), AP_INIT_TAKE1("H3MaxConnections", set_h3_max_connections, NULL, RSRC_CONF, "Maximum concurrent QUIC/HTTP/3 connections per child process (default: 256)"), diff --git a/modules/http3/h3_hooks.c b/modules/http3/h3_hooks.c index 87d99c977c0..709bd505ed4 100644 --- a/modules/http3/h3_hooks.c +++ b/modules/http3/h3_hooks.c @@ -82,7 +82,7 @@ int h3_hook_fixups(request_rec* r) h3_server_conf* conf = ap_get_module_config(r->server->module_config, &http3_module); - if (!conf || !conf->h3_cert_path || !conf->h3_key_path || conf->h3_port == 0) + if (!conf || !conf->ssl_ctx || conf->h3_port == 0) { return DECLINED; } diff --git a/modules/http3/h3_io.c b/modules/http3/h3_io.c index 22f618b98ec..66ad2abac40 100644 --- a/modules/http3/h3_io.c +++ b/modules/http3/h3_io.c @@ -131,10 +131,8 @@ apr_status_t h3_io_listen_start(apr_pool_t* pchild, server_rec* s, h3_server_con char qerr[H3Q_ERRLEN] = {0}; h3q_config qcfg = { - .cert_path = conf->h3_cert_path, - .key_path = conf->h3_key_path, + .ssl_ctx = conf->ssl_ctx, .address_validation = (conf->h3_address_validation != H3_FLAG_OFF), - .session_tickets = (conf->h3_session_tickets != H3_FLAG_OFF), }; io->qengine = h3q_engine_create(&qcfg, udp_fd, qerr, sizeof(qerr)); if (!io->qengine) diff --git a/modules/http3/h3_server.c b/modules/http3/h3_server.c index 9a7ef5fb121..1ac3ed07689 100644 --- a/modules/http3/h3_server.c +++ b/modules/http3/h3_server.c @@ -16,8 +16,11 @@ #include +#include +#include #include #include +#include #include #include @@ -32,6 +35,14 @@ #include "h3_socket.h" #include "mod_http3.h" +#ifdef __linux__ + #include + #include + #include + #include + #include +#endif + static volatile int child_stopping = 0; static apr_thread_t* port_acquire_thread = NULL; @@ -43,13 +54,98 @@ struct port_acquire_args h3_server_conf* conf; }; +#ifdef __linux__ +/* The parent binds the QUIC sockets while still privileged and keeps them across restarts. + * One SO_REUSEPORT socket per httpd listener bucket (ListenCoresBucketsRatio), so the kernel hash stays stable when children come and go. */ +typedef struct +{ + apr_pool_t* pool; + int* fds; + int n; + apr_port_t port; +} h3_parent_socket; + +/* The same count as httpd for ListenCoresBucketsRatio: online cores / ratio. Some MPMs count their buckets after post_config. + * ponytail: top-level directive only, not one inside . */ +static int listen_buckets(void) +{ + for (ap_directive_t* d = ap_conftree; d; d = d->next) + { + if (ap_have_so_reuseport > 0 && !ap_cstr_casecmp(d->directive, "ListenCoresBucketsRatio")) + { + long ratio = atol(d->args); + long n = ratio > 0 ? sysconf(_SC_NPROCESSORS_ONLN) / ratio : 1; + int limit = 1; + ap_mpm_query(AP_MPMQ_HARD_LIMIT_DAEMONS, &limit); /* A bucket with no child loses its peers. */ + return n < 1 ? 1 : n > limit ? limit : (int)n; + } + } + return 1; +} + +static h3_parent_socket* parent_socket(server_rec* s) +{ + void* ps = NULL; + apr_pool_userdata_get(&ps, "mod_http3.parent_socket", s->process->pool); + return ps; +} + +/* The children inherit them; the child that binds the abstract name of a bucket owns its socket, and exit releases it. */ +static int owner_lock = -1; + +static int own_port(apr_port_t port, int bucket) +{ + struct sockaddr_un a = {.sun_family = AF_UNIX}; + int n = snprintf(a.sun_path + 1, sizeof(a.sun_path) - 1, "mod_http3.%d.%d", (int)port, bucket); + if (owner_lock < 0) + { + owner_lock = socket(AF_UNIX, SOCK_DGRAM | SOCK_CLOEXEC, 0); + } + return owner_lock >= 0 && bind(owner_lock, (struct sockaddr*)&a, (socklen_t)(offsetof(struct sockaddr_un, sun_path) + 1 + (size_t)n)) == 0; +} +#endif + +/* Get this child the QUIC socket, or APR_EAGAIN while another child has it. */ +static apr_status_t acquire_port(apr_pool_t* pchild, server_rec* s, h3_server_conf* conf, int* fd) +{ +#ifdef __linux__ + h3_parent_socket* ps = parent_socket(s); + if (ps && ps->n && ps->port == conf->h3_port) + { + for (int i = 0; i < ps->n; i++) + { + if (own_port(ps->port, i)) + { + *fd = ps->fds[i]; + return APR_SUCCESS; + } + } + return APR_EAGAIN; + } +#endif + return h3_socket_open(conf->h3_port, conf->h3_socket_buffer_size, pchild, 0, fd); +} + +/* Give the port back when this child cannot serve it after all. */ +static void release_port(int fd) +{ + h3_socket_close(fd); +#ifdef __linux__ + if (owner_lock >= 0) + { + close(owner_lock); + owner_lock = -1; + } +#endif +} + static void* APR_THREAD_FUNC port_acquire_thread_fn(apr_thread_t* thread H3_UNUSED, void* data) { struct port_acquire_args* args = data; while (!child_stopping) { int udp_fd = -1; - apr_status_t rv = h3_socket_open(args->conf->h3_port, args->conf->h3_socket_buffer_size, args->pchild, &udp_fd); + apr_status_t rv = acquire_port(args->pchild, args->vhost, args->conf, &udp_fd); if (rv == APR_EAGAIN) { apr_sleep(apr_time_from_msec(H3_PORT_ACQUIRE_RETRY_MS)); @@ -62,13 +158,13 @@ static void* APR_THREAD_FUNC port_acquire_thread_fn(apr_thread_t* thread H3_UNUS } if (child_stopping) { - h3_socket_close(udp_fd); + release_port(udp_fd); break; } if (h3_io_listen_start(args->pchild, args->vhost, args->conf, udp_fd) != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ERR, 0, args->vhost, "port acquirer: h3_io_listen_start failed"); - h3_socket_close(udp_fd); + release_port(udp_fd); break; } ap_log_error(APLOG_MARK, APLOG_INFO, 0, args->vhost, "port acquirer: pid=%d acquired QUIC port after retrying", h3_getpid()); @@ -84,15 +180,18 @@ static h3_server_conf* find_h3_server(server_rec* s, server_rec** out_server) while (current) { h3_server_conf* tmp = ap_get_module_config(current->module_config, &http3_module); - if (tmp && tmp->h3_cert_path && tmp->h3_key_path && !conf) + /* httpd prepends virtual hosts: the last match is the first one defined, mod_ssl's default. */ + if (tmp && tmp->ssl_ctx) { conf = tmp; - conf->host_port = get_server_port(current); *out_server = current; - break; } current = current->next; } + if (conf) + { + conf->host_port = get_server_port(*out_server); + } return conf; } @@ -107,7 +206,7 @@ void h3_child_init(apr_pool_t* pchild, server_rec* s) } int udp_fd = -1; - apr_status_t rv = h3_socket_open(conf->h3_port, conf->h3_socket_buffer_size, pchild, &udp_fd); + apr_status_t rv = acquire_port(pchild, vhost, conf, &udp_fd); if (rv == APR_EAGAIN) { ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, vhost, "h3_child_init: pid=%d port already owned, will keep retrying in background", h3_getpid()); @@ -131,8 +230,57 @@ void h3_child_init(apr_pool_t* pchild, server_rec* s) if (h3_io_listen_start(pchild, vhost, conf, udp_fd) != APR_SUCCESS) { ap_log_error(APLOG_MARK, APLOG_ERR, 0, vhost, "h3_child_init: h3_io_listen_start failed"); - h3_socket_close(udp_fd); + release_port(udp_fd); + } +} + +void h3_server_post_config(server_rec* s) +{ +#ifdef __linux__ + h3_parent_socket* ps = parent_socket(s); + if (!ps) + { + ps = apr_pcalloc(s->process->pool, sizeof(*ps)); + apr_pool_userdata_set(ps, "mod_http3.parent_socket", apr_pool_cleanup_null, s->process->pool); + } + server_rec* vhost = NULL; + h3_server_conf* conf = find_h3_server(s, &vhost); + int n = listen_buckets(); + /* ponytail: a new port or bucket count binds again. An old single socket without SO_REUSEPORT blocks this until its children stop. */ + if (ps->n && (!conf || conf->h3_port != ps->port || n != ps->n)) + { + apr_pool_destroy(ps->pool); /* closes the old sockets */ + ps->pool = NULL; + ps->n = 0; + } + if (!conf || ps->n) + { + return; + } + int ok = apr_pool_create(&ps->pool, s->process->pool) == APR_SUCCESS; + if (ok) + { + ps->fds = apr_palloc(ps->pool, sizeof(int) * (size_t)n); } + for (int i = 0; ok && i < n; i++) + { + ok = h3_socket_open(conf->h3_port, conf->h3_socket_buffer_size, ps->pool, n > 1, &ps->fds[i]) == APR_SUCCESS; + } + if (!ok) + { + ap_log_error(APLOG_MARK, APLOG_WARNING, 0, vhost, "mod_http3: binding UDP port %d before the privilege drop failed; the children bind it themselves", (int)conf->h3_port); + if (ps->pool) + { + apr_pool_destroy(ps->pool); + } + ps->pool = NULL; + return; + } + ps->n = n; + ps->port = conf->h3_port; +#else + (void)s; +#endif } void h3_c1_child_stopping(apr_pool_t* p H3_UNUSED, int graceful) diff --git a/modules/http3/h3_socket.c b/modules/http3/h3_socket.c index 0a8b2ddae68..b894d8b0944 100644 --- a/modules/http3/h3_socket.c +++ b/modules/http3/h3_socket.c @@ -55,7 +55,7 @@ static void tune_buffers(apr_socket_t* sock, apr_size_t want, apr_port_t port, a } } -apr_status_t h3_socket_open(apr_port_t port, apr_size_t buffer_size, apr_pool_t* pool, int* out_fd) +apr_status_t h3_socket_open(apr_port_t port, apr_size_t buffer_size, apr_pool_t* pool, int reuse_port, int* out_fd) { CHECK(pool); CHECK(out_fd); @@ -68,6 +68,16 @@ apr_status_t h3_socket_open(apr_port_t port, apr_size_t buffer_size, apr_pool_t* } apr_socket_opt_set(sock, APR_IPV6_V6ONLY, 0); tune_buffers(sock, buffer_size, port, pool); +#ifdef SO_REUSEPORT + apr_os_sock_t reuse_sock = -1; + int on = 1; + if (reuse_port && apr_os_sock_get(&reuse_sock, sock) == APR_SUCCESS) + { + (void)setsockopt(reuse_sock, SOL_SOCKET, SO_REUSEPORT, (const char*)&on, sizeof(on)); + } +#else + (void)reuse_port; +#endif apr_sockaddr_t* addr = NULL; rv = apr_sockaddr_info_get(&addr, NULL, APR_INET6, port, 0, pool); if (rv != APR_SUCCESS) diff --git a/modules/http3/include/h3_config.h b/modules/http3/include/h3_config.h index b22f5b3ba0f..ef8a56cbc8e 100644 --- a/modules/http3/include/h3_config.h +++ b/modules/http3/include/h3_config.h @@ -36,8 +36,9 @@ struct h3_server_conf { apr_port_t host_port; - const char* h3_cert_path; - const char* h3_key_path; + /** QUIC TLS context, built from mod_ssl's certificate when the host lists h3 in Protocols. */ + struct ssl_ctx_st* ssl_ctx; + apr_port_t h3_port; apr_uint32_t h3_max_concurrent_streams; apr_uint32_t h3_max_connections; @@ -81,8 +82,7 @@ void* h3_create_server_config(apr_pool_t* p, server_rec* s); /** * ap_merge_server_config callback: produce a child vhost config that - * inherits each unset field from the parent. cert/key/h3_port use the - * new value if non-NULL/non-zero, else the base. + * inherits each unset field from the parent. * @param p Pool for the merged config. * @param base_conf Parent h3_server_conf. * @param new_conf Child h3_server_conf. @@ -111,16 +111,27 @@ void* h3_create_dir_config(apr_pool_t* p, char* dir); void* h3_merge_dir_config(apr_pool_t* p, void* base, void* add); /** - * ap_post_config hook: resolve cert/key/h3_port for the listening vhost - * and log the resolved values. No-op in AP_SQ_MS_CREATE_PRE_CONFIG - * (pre-config phase). Returns OK if a fully-configured vhost is found, - * HTTP_INTERNAL_SERVER_ERROR otherwise. - * @param p Config pool (unused). + * ap_ssl_add_cert_files hook: mod_ssl runs it for every SSLEngine vhost with + * the certificate and key files it is about to load (SSLCertificateFile plus + * anything mod_md added). When the vhost lists h3 in Protocols, builds its + * QUIC TLS context from them, here, before the server drops privileges. + * @param s The vhost being configured. + * @param p Config pool; owns the context. + * @param cert_files Certificate chain files, const char* elements. + * @param key_files Private key files, const char* elements. + * @return DECLINED, or HTTP_INTERNAL_SERVER_ERROR if the files do not load. + */ +int h3_ssl_add_cert_files(server_rec* s, apr_pool_t* p, apr_array_header_t* cert_files, apr_array_header_t* key_files); + +/** + * ap_post_config hook: fill in defaults on every vhost that serves HTTP/3; + * the first one owns the listener, and an SNI callback swaps in each other + * host's certificate by name. No-op in AP_SQ_MS_CREATE_PRE_CONFIG. + * @param p Config pool; owns the SNI host table. * @param plog Log pool (unused). * @param ptemp Temp pool (unused). * @param s The first server_rec in the configuration. - * @return OK, or HTTP_INTERNAL_SERVER_ERROR if no vhost has both - * H3CertificatePath and H3CertificateKeyPath set. + * @return OK, or HTTP_INTERNAL_SERVER_ERROR if no vhost serves HTTP/3. */ int h3_post_config(apr_pool_t* p, apr_pool_t* plog, apr_pool_t* ptemp, server_rec* s); diff --git a/modules/http3/include/h3_server.h b/modules/http3/include/h3_server.h index b18172e8066..2eb747a039c 100644 --- a/modules/http3/include/h3_server.h +++ b/modules/http3/include/h3_server.h @@ -33,6 +33,13 @@ */ void h3_child_init(apr_pool_t* pchild, server_rec* s); +/** + * Bind the QUIC socket in the parent while it is still privileged (Linux), and + * keep it across graceful restarts for the children to inherit. + * @param s Main server. + */ +void h3_server_post_config(server_rec* s); + /** * Child-stopping hook: stop retrying for the QUIC port, then either drain the * live connections (graceful) or tear the listener down at once (immediate). diff --git a/modules/http3/include/h3_socket.h b/modules/http3/include/h3_socket.h index 50a9ad6630c..91728ce823a 100644 --- a/modules/http3/include/h3_socket.h +++ b/modules/http3/include/h3_socket.h @@ -24,20 +24,21 @@ /** * Open a non-blocking IPv6 dual-stack UDP socket and bind it to the given port. - * Sets SO_REUSEADDR; on non-Windows also tries SO_REUSEPORT so multiple - * children can share the port. + * With @p reuse_port, sets SO_REUSEPORT (where the OS has it), so several + * sockets can share the port and the kernel spreads the peers over them. * Also asks for @p buffer_size on the send and receive buffers, which the OS * may cap; a refused or capped buffer is logged, never fatal. * @param port Port to bind; 0 lets the OS pick. * @param buffer_size Bytes requested for SO_RCVBUF and SO_SNDBUF; 0 keeps the * OS default. * @param pool Pool used for the underlying apr_socket_t lifetime. + * @param reuse_port Nonzero to set SO_REUSEPORT. * @param out_fd Out parameter: the resulting OS-level fd (suitable for * SSL_set_fd on OpenSSL QUIC). * @return APR_SUCCESS, APR_EAGAIN if the port is already bound, or another * APR error code. */ -apr_status_t h3_socket_open(apr_port_t port, apr_size_t buffer_size, apr_pool_t* pool, int* out_fd); +apr_status_t h3_socket_open(apr_port_t port, apr_size_t buffer_size, apr_pool_t* pool, int reuse_port, int* out_fd); /** * Close a UDP socket previously returned by h3_socket_open. diff --git a/modules/http3/include/quic/detail/h3q_tls.h b/modules/http3/include/quic/detail/h3q_tls.h index 0621442995a..956448f257c 100644 --- a/modules/http3/include/quic/detail/h3q_tls.h +++ b/modules/http3/include/quic/detail/h3q_tls.h @@ -24,15 +24,21 @@ #include "quic/h3q.h" /** - * Build the TLS context the listener serves from: certificate and key from - * @p cfg, "h3" as the only ALPN protocol, and a key log when SSLKEYLOGFILE is - * set. - * @param cfg Configuration supplying the certificate and key paths. - * @param err Buffer receiving the reason on failure; may be NULL. - * @param errlen Capacity of @p err. + * Build the TLS context the listener serves from: the certificate chain files + * with their keys (one per key type, as mod_ssl allows; a missing key file + * means the key sits in the chain file), "h3" as the only ALPN protocol, and + * a key log when SSLKEYLOGFILE is set. Safe before the server forks and drops + * privileges; children inherit the loaded keys. + * @param cert_files Certificate chain files; at least one. + * @param ncerts Number of entries in @p cert_files. + * @param key_files Private key files, matched by index to @p cert_files. + * @param nkeys Number of entries in @p key_files; may be fewer. + * @param session_tickets Non-zero to issue TLS 1.3 session tickets. + * @param err Buffer receiving the reason on failure; may be NULL. + * @param errlen Capacity of @p err. * @return New context, or NULL on failure. */ -SSL_CTX* h3q_tls_ctx_create(const h3q_config* cfg, char* err, size_t errlen); +SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const char* const* key_files, size_t nkeys, int session_tickets, char* err, size_t errlen); /** * Record a message in a caller-supplied error buffer, appending the OpenSSL diff --git a/modules/http3/include/quic/h3q.h b/modules/http3/include/quic/h3q.h index 88c90f0ead3..1398b310ecb 100644 --- a/modules/http3/include/quic/h3q.h +++ b/modules/http3/include/quic/h3q.h @@ -32,16 +32,15 @@ typedef struct h3q_stream h3q_stream; * a per-connection setting, applied by h3q_conn_prepare(). */ typedef struct h3q_config { - const char* cert_path; - const char* key_path; + /** TLS context from h3q_tls_ctx_create(); the engine takes its own reference. */ + struct ssl_ctx_st* ssl_ctx; unsigned address_validation : 1; - unsigned session_tickets : 1; } h3q_config; /** * Build the QUIC listener on @p udp_fd, together with the filter BIO that * recovers peer addresses from OpenSSL's accept queue. - * @param cfg Certificate, key and address validation. + * @param cfg TLS context and address validation. * @param udp_fd Pre-opened non-blocking UDP socket bound to the listen port, * borrowed for the engine's lifetime. * @param err Buffer receiving the reason on failure; may be NULL. diff --git a/modules/http3/mod_http3.c b/modules/http3/mod_http3.c index fb8db48ccb7..a5d71f7ed0a 100644 --- a/modules/http3/mod_http3.c +++ b/modules/http3/mod_http3.c @@ -40,7 +40,11 @@ static void register_hooks(apr_pool_t* p H3_UNUSED) { ap_hook_handler(h3_status_handler, NULL, NULL, APR_HOOK_MIDDLE); - ap_hook_post_config(h3_post_config, NULL, NULL, APR_HOOK_MIDDLE); + /* mod_ssl hands out the certificate files it resolved before it loads them; + * post_config runs after it so those files are known by then. */ + static const char* const after_ssl[] = {"mod_ssl.c", NULL}; + ap_hook_ssl_add_cert_files(h3_ssl_add_cert_files, NULL, NULL, APR_HOOK_LAST); + ap_hook_post_config(h3_post_config, after_ssl, NULL, APR_HOOK_MIDDLE); ap_hook_create_request(h3_hook_http_create_request, NULL, NULL, APR_HOOK_REALLY_FIRST); ap_hook_pre_read_request(h3_hook_pre_read_request, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_post_read_request(h3_hook_post_read_request, NULL, NULL, APR_HOOK_REALLY_FIRST); @@ -72,5 +76,5 @@ HTTP3_PUBLIC module http3_module = { h3_merge_server_config, /* merge per-server config structures */ h3_cmds, /* command apr_table_t */ register_hooks, /* register hooks */ - AP_MODULE_FLAG_NONE /* flags */ + AP_MODULE_FLAG_ALWAYS_MERGE /* every vhost gets its own config, H3 directives or not */ }; diff --git a/modules/http3/quic/detail/h3q_tls.c b/modules/http3/quic/detail/h3q_tls.c index 1ed6125b0d2..32d8a46d59f 100644 --- a/modules/http3/quic/detail/h3q_tls.c +++ b/modules/http3/quic/detail/h3q_tls.c @@ -77,9 +77,10 @@ static void h3q_tls_keylog_cb(const SSL* ssl, const char* line) } } -SSL_CTX* h3q_tls_ctx_create(const h3q_config* cfg, char* err, size_t errlen) +SSL_CTX* h3q_tls_ctx_create(const char* const* cert_files, size_t ncerts, const char* const* key_files, size_t nkeys, int session_tickets, char* err, size_t errlen) { - CHECK(cfg); + CHECK(cert_files && ncerts > 0); + CHECK(key_files || nkeys == 0); SSL_CTX* ssl_ctx = SSL_CTX_new(OSSL_QUIC_server_method()); if (!ssl_ctx) @@ -92,17 +93,21 @@ SSL_CTX* h3q_tls_ctx_create(const h3q_config* cfg, char* err, size_t errlen) SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_3_VERSION); SSL_CTX_set_max_proto_version(ssl_ctx, TLS1_3_VERSION); - if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cfg->cert_path) <= 0 || SSL_CTX_use_PrivateKey_file(ssl_ctx, cfg->key_path, SSL_FILETYPE_PEM) <= 0) + for (size_t i = 0; i < ncerts; i++) { - h3q_tls_error(err, errlen, "loading the certificate or private key failed"); - SSL_CTX_free(ssl_ctx); - return NULL; + const char* key_file = i < nkeys ? key_files[i] : cert_files[i]; + if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_files[i]) <= 0 || SSL_CTX_use_PrivateKey_file(ssl_ctx, key_file, SSL_FILETYPE_PEM) <= 0) + { + h3q_tls_error(err, errlen, "loading certificate %s with key %s failed", cert_files[i], key_file); + SSL_CTX_free(ssl_ctx); + return NULL; + } } static const unsigned char sid_ctx[] = "mod_http3"; SSL_CTX_set_session_id_context(ssl_ctx, sid_ctx, sizeof(sid_ctx) - 1); - if (!cfg->session_tickets) + if (!session_tickets) { /* TLS 1.3 resumption travels in tickets, so issuing none turns it off. */ SSL_CTX_set_num_tickets(ssl_ctx, 0); diff --git a/modules/http3/quic/h3q.c b/modules/http3/quic/h3q.c index 10f88b4a842..879a6058135 100644 --- a/modules/http3/quic/h3q.c +++ b/modules/http3/quic/h3q.c @@ -44,12 +44,13 @@ h3q_engine* h3q_engine_create(const h3q_config* cfg, int udp_fd, char* err, size } engine->peer_addr_ex_index = -1; - engine->ssl_ctx = h3q_tls_ctx_create(cfg, err, errlen); - if (!engine->ssl_ctx) + if (!cfg->ssl_ctx || !SSL_CTX_up_ref(cfg->ssl_ctx)) { + h3q_tls_error(err, errlen, "no TLS context to serve from"); h3q_engine_destroy(engine); return NULL; } + engine->ssl_ctx = cfg->ssl_ctx; BIO_METHOD* bm = BIO_meth_new(BIO_TYPE_FILTER | BIO_get_new_index(), "h3q_peer_addr"); if (!bm)