diff --git a/CHANGELOG.md b/CHANGELOG.md index 591c01a6..59af50ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#812]). - Add support for 2.6.7 and 3.0.0 (experimental) ([#815]). - Masters and region servers now have a default affinity to the OPA Pods when OPA authorization is configured ([#816]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#832]). ### Changed @@ -56,6 +58,10 @@ - BREAKING: The rest-server listener PVC template now carries the recommended labels without the version label, so that the labels stay stable across upgrades. Existing rest-server StatefulSets must be deleted once before the new operator can reconcile them ([#799]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY`, which containerdebug stopped reading in its version 0.2.0 when it + moved to `stackable-telemetry`; the directory is now set via `FILE_LOG_DIRECTORY`. The file is + rotated hourly and at most two files are kept, because rotation is opt-in ([#832]). [#776]: https://github.com/stackabletech/hbase-operator/pull/776 [#782]: https://github.com/stackabletech/hbase-operator/pull/782 @@ -76,6 +82,7 @@ [#825]: https://github.com/stackabletech/hbase-operator/pull/825 [#827]: https://github.com/stackabletech/hbase-operator/pull/827 [#829]: https://github.com/stackabletech/hbase-operator/pull/829 +[#832]: https://github.com/stackabletech/hbase-operator/pull/832 ## [26.7.0] - 2026-07-21 diff --git a/Cargo.nix b/Cargo.nix index 5fe38b3b..01ad8f17 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -10524,6 +10524,11 @@ rec { name = "rstest"; packageId = "rstest"; } + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "crds" "webhook" "test-support" ]; + } { name = "yaml_serde"; packageId = "yaml_serde"; @@ -10733,7 +10738,7 @@ rec { "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; }; - resolvedDefaultFeatures = [ "crds" "default" "webhook" ]; + resolvedDefaultFeatures = [ "crds" "default" "test-support" "webhook" ]; }; "stackable-operator-derive" = rec { crateName = "stackable-operator-derive"; diff --git a/docs/modules/hbase/pages/usage-guide/logging.adoc b/docs/modules/hbase/pages/usage-guide/logging.adoc index 2a07afdb..0a910119 100644 --- a/docs/modules/hbase/pages/usage-guide/logging.adoc +++ b/docs/modules/hbase/pages/usage-guide/logging.adoc @@ -22,4 +22,12 @@ spec: enableVectorAgent: true ---- +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `hbase` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/hbase` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index 1fac5bc3..7f7e48f9 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -31,4 +31,5 @@ built.workspace = true [dev-dependencies] rstest.workspace = true +stackable-operator = { workspace = true, features = ["test-support"] } yaml_serde.workspace = true diff --git a/rust/operator-binary/src/controller/build/container.rs b/rust/operator-binary/src/controller/build/container.rs index c746ff66..86dbbbae 100644 --- a/rust/operator-binary/src/controller/build/container.rs +++ b/rust/operator-binary/src/controller/build/container.rs @@ -35,7 +35,10 @@ use crate::{ properties::product_logging::MAX_HBASE_LOG_FILES_SIZE, role_group::{ListenerPlacement, RoleGroupInputs}, }, - crd::{CONFIG_DIR_NAME, HbaseRole, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME}, + crd::{ + CONFIG_DIR_NAME, CONTAINERDEBUG_LOG_DIR, HbaseRole, LISTENER_VOLUME_DIR, + LISTENER_VOLUME_NAME, + }, }; constant!(pub(crate) HBASE_CONTAINER_NAME: ContainerName = "hbase"); @@ -56,7 +59,9 @@ constant!(HADOOP_CONF_DIR_ENV: EnvVarName = "HADOOP_CONF_DIR"); constant!(pub(crate) REGION_MOVER_OPTS_ENV: EnvVarName = "REGION_MOVER_OPTS"); constant!(pub(crate) RUN_REGION_MOVER_ENV: EnvVarName = "RUN_REGION_MOVER"); constant!(STACKABLE_LOG_DIR_ENV: EnvVarName = "STACKABLE_LOG_DIR"); -constant!(CONTAINERDEBUG_LOG_DIRECTORY_ENV: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); +constant!(FILE_LOG_DIRECTORY_ENV: EnvVarName = "FILE_LOG_DIRECTORY"); +constant!(FILE_LOG_MAX_FILES_ENV: EnvVarName = "FILE_LOG_MAX_FILES"); +constant!(FILE_LOG_ROTATION_PERIOD_ENV: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); // These constants are hard coded in hbase-entrypoint.sh // You need to change them there too. @@ -145,11 +150,14 @@ pub(crate) fn containers_and_volumes(inputs: &RoleGroupInputs) -> Result Result, + // Only the HBase log files are budgeted. The `containerdebug` files in this + // volume are rotated by time, not by size, so they have no budget; raising + // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_HBASE_LOG_FILES_SIZE], )), @@ -303,6 +314,8 @@ mod tests { let _ = *REGION_MOVER_OPTS_ENV; let _ = *RUN_REGION_MOVER_ENV; let _ = *STACKABLE_LOG_DIR_ENV; - let _ = *CONTAINERDEBUG_LOG_DIRECTORY_ENV; + let _ = *FILE_LOG_DIRECTORY_ENV; + let _ = *FILE_LOG_MAX_FILES_ENV; + let _ = *FILE_LOG_ROTATION_PERIOD_ENV; } } diff --git a/rust/operator-binary/src/controller/build/properties/hbase_env.rs b/rust/operator-binary/src/controller/build/properties/hbase_env.rs index 2dffcef9..af222050 100644 --- a/rust/operator-binary/src/controller/build/properties/hbase_env.rs +++ b/rust/operator-binary/src/controller/build/properties/hbase_env.rs @@ -112,7 +112,7 @@ mod tests { false, "-Xtest".to_string(), BTreeMap::from([( - EnvVarName::from_str("CUSTOM_VAR").expect("valid variable name"), + EnvVarName::from_str_unsafe("CUSTOM_VAR"), "custom_value".to_string(), )]), ) diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs index 1716c319..42983be1 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs +++ b/rust/operator-binary/src/controller/build/properties/product_logging/mod.rs @@ -61,7 +61,9 @@ mod tests { assert!(!content.is_empty()); // A kept source must be present ... assert!(content.contains("files_log4j2")); + assert!(content.contains("files_tracing_rs")); + assert!(content.contains("*.tracing-rs.json")); // ... while a product-specific source we don't emit must not. - assert!(!content.contains("files_tracing_rs")); + assert!(!content.contains("files_py")); } } diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index 40a79236..0eeab314 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -102,6 +102,285 @@ tests: "timestamp": t'2025-10-02T09:27:28.582Z' } + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk\n\ndisk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test unparsable tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{ unparsable' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "errors": [ + .errors[0] + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{ unparsable", + "timestamp": .timestamp, + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "JSON not parsable: ")) + - name: Test tracing-rs log which is not a JSON object + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: 'false' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "false", + "timestamp": .timestamp, + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with invalid fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "errors": [ + .errors[0], + "Logger/target not found.", + "Level \"INVALID\" unknown, using \"INFO\" instead.", + "Fields are not an object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}", + "timestamp": .timestamp, + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: ")) + - name: Test tracing-rs log without a timestamp + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk", + "timestamp": .timestamp, + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The current time is used and not the timestamp which the file + # source attached to the event. The latter reaches the transform + # as a string, whereas now() yields a timestamp. + assert!(is_timestamp(.timestamp)) + - name: Test tracing-rs log without a message but with other fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log with a message which is not a string + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay\nmessage=42", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Test tracing-rs log without any fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}' + pod: hbase-regionserver-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "hbase", + "container": "containerdebug", + "errors": [ + "Message not found." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "hbase-regionserver-default-0", + "role": "regionserver", + "roleGroup": "default" + } + assert_eq!(expected_log_event, .) - name: Test Vector internal logs inputs: diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index 2fd3d923..b9e446d3 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -25,6 +25,12 @@ sources: - ${LOG_DIR}/*/*.log4j2.xml line_delimiter: "\r\n" + # Reads the logs of the containerdebug process + files_tracing_rs: + type: file + include: + - ${LOG_DIR}/*/*.tracing-rs.json + transforms: processed_files_stdout: inputs: @@ -174,11 +180,92 @@ transforms: } # Extends the processed files with the fields "container" and "file" + # Transforms the tracing-rs JSON logs written by the containerdebug process + processed_files_tracing_rs: + inputs: + - files_tracing_rs + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%+") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.target) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger/target not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], upcase(level)) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = upcase(level) + } + + fields, err = object(event.fields) + if err == null { + message, err = string(fields.message) + if err == null { + # The message is prepended to the other fields, so remove it from + # them. + del(fields.message) + } else { + # Keep an unusable message among the other fields instead of + # discarding its content. + message = "" + } + + other_fields = encode_key_value(fields, field_delimiter: "\n") + .message = join!(compact([message, other_fields]), "\n\n") + } else { + .errors = push(.errors, "Fields are not an object.") + # Fall back to the raw line, so that its content is not lost. + .message = raw_message + } + + if is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + } + extended_logs_files: inputs: - processed_files_stdout - processed_files_stderr - processed_files_log4j2 + - processed_files_tracing_rs type: remap source: | del(.source_type) diff --git a/rust/operator-binary/src/controller/build/resource/statefulset.rs b/rust/operator-binary/src/controller/build/resource/statefulset.rs index 9c6a73f8..0d9ee1c4 100644 --- a/rust/operator-binary/src/controller/build/resource/statefulset.rs +++ b/rust/operator-binary/src/controller/build/resource/statefulset.rs @@ -124,7 +124,7 @@ mod tests { /// `envOverrides` are applied after every operator-set environment variable, so users can /// override any of them (previously the operator's value silently won for the variables set - /// after the merge, and `CONTAINERDEBUG_LOG_DIRECTORY` was appended separately, producing a + /// after the merge, and `FILE_LOG_DIRECTORY` was appended separately, producing a /// duplicate entry when overridden). #[test] fn env_overrides_take_precedence_over_operator_set_env_vars() { @@ -146,7 +146,7 @@ spec: masters: envOverrides: RUN_REGION_MOVER: "overridden" - CONTAINERDEBUG_LOG_DIRECTORY: /debug-override + FILE_LOG_DIRECTORY: /debug-override roleGroups: default: replicas: 1 @@ -187,9 +187,7 @@ spec: .env .expect("the hbase container has env vars") .into_iter() - .filter(|env_var| { - ["RUN_REGION_MOVER", "CONTAINERDEBUG_LOG_DIRECTORY"].contains(&&*env_var.name) - }) + .filter(|env_var| ["RUN_REGION_MOVER", "FILE_LOG_DIRECTORY"].contains(&&*env_var.name)) .map(|env_var| (env_var.name, env_var.value.unwrap_or_default())) .collect(); @@ -198,7 +196,7 @@ spec: env, [ ( - "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), + "FILE_LOG_DIRECTORY".to_string(), "/debug-override".to_string() ), ("RUN_REGION_MOVER".to_string(), "overridden".to_string()), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index efa0f286..16dde311 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,7 @@ use core::fmt; use std::{ops::Deref, str::FromStr}; +use const_format::concatcp; use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; use stackable_operator::{ @@ -29,6 +30,7 @@ use stackable_operator::{ status::condition::{ClusterCondition, HasStatusCondition}, v2::{ config_overrides::KeyValueConfigOverrides, + product_logging::framework::STACKABLE_LOG_DIR, role_utils::{JavaCommonConfig, Role}, types::{ common::Port, @@ -53,6 +55,11 @@ pub const HBASE_OPERATOR_NAME: &str = "hbase.stackable.tech"; // You need to change it there too. pub const CONFIG_DIR_NAME: &str = "/stackable/conf"; +/// Directory which the `containerdebug` process writes its log files to. It must be exactly one +/// level below the log directory, because the Vector agent derives the container name from that +/// path segment. +pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); + pub const TLS_STORE_DIR: &str = "/stackable/tls"; constant!(pub TLS_STORE_VOLUME_NAME: VolumeName = "tls"); pub const TLS_STORE_PASSWORD: &str = "changeit"; diff --git a/tests/templates/kuttl/logging/hbase-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/hbase-vector-aggregator-values.yaml.j2 index 098fb0fb..ad245000 100644 --- a/tests/templates/kuttl/logging/hbase-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/hbase-vector-aggregator-values.yaml.j2 @@ -22,83 +22,139 @@ customConfig: transforms: validEvents: type: filter - inputs: [vector] + inputs: + - vector condition: is_null(.errors) filteredAutomaticLogConfigMasterHbase: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-master-automatic-log-config-0" && .container == "hbase" + filteredAutomaticLogConfigMasterContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hbase-master-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigMasterVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-master-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigMasterHbase: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-master-custom-log-config-0" && .container == "hbase" + filteredCustomLogConfigMasterContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hbase-master-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigMasterVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-master-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigRegionserverHbase: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-regionserver-automatic-log-config-0" && .container == "hbase" + filteredAutomaticLogConfigRegionserverContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hbase-regionserver-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigRegionserverVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-regionserver-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigRegionserverHbase: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-regionserver-custom-log-config-0" && .container == "hbase" + filteredCustomLogConfigRegionserverContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hbase-regionserver-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigRegionserverVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-regionserver-custom-log-config-0" && .container == "vector" filteredAutomaticLogConfigRestserverHbase: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-restserver-automatic-log-config-0" && .container == "hbase" + filteredAutomaticLogConfigRestserverContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hbase-restserver-automatic-log-config-0" && + .container == "containerdebug" filteredAutomaticLogConfigRestserverVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-restserver-automatic-log-config-0" && .container == "vector" filteredCustomLogConfigRestserverHbase: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-restserver-custom-log-config-0" && .container == "hbase" + filteredCustomLogConfigRestserverContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + .pod == "test-hbase-restserver-custom-log-config-0" && + .container == "containerdebug" filteredCustomLogConfigRestserverVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- .pod == "test-hbase-restserver-custom-log-config-0" && .container == "vector" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -106,11 +162,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: diff --git a/tests/templates/kuttl/smoke/30-assert.yaml.j2 b/tests/templates/kuttl/smoke/30-assert.yaml.j2 index 5038ff06..76c67f7e 100644 --- a/tests/templates/kuttl/smoke/30-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-assert.yaml.j2 @@ -259,8 +259,12 @@ spec: - pipefail - -c env: - - name: CONTAINERDEBUG_LOG_DIRECTORY + - name: FILE_LOG_DIRECTORY value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: HADOOP_CONF_DIR value: /stackable/conf - name: HBASE_CONF_DIR @@ -494,8 +498,12 @@ spec: - pipefail - -c env: - - name: CONTAINERDEBUG_LOG_DIRECTORY + - name: FILE_LOG_DIRECTORY value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: HADOOP_CONF_DIR value: /stackable/conf - name: HBASE_CONF_DIR @@ -725,8 +733,12 @@ spec: - pipefail - -c env: - - name: CONTAINERDEBUG_LOG_DIRECTORY + - name: FILE_LOG_DIRECTORY value: /stackable/log/containerdebug + - name: FILE_LOG_MAX_FILES + value: "2" + - name: FILE_LOG_ROTATION_PERIOD + value: hourly - name: HADOOP_CONF_DIR value: /stackable/conf - name: HBASE_CONF_DIR