diff --git a/CHANGELOG.md b/CHANGELOG.md index ca394090..4d837202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +- Add support for enabling secure mode with Kerberos ([#334]). - Generate OLM bundle for Release 23.4.0 ([#350]). - Missing CRD defaults for `status.conditions` field ([#354]). @@ -16,6 +17,7 @@ All notable changes to this project will be documented in this file. - Use testing-tools 0.2.0 ([#351]) - Run as root group ([#353]). +[#334]: https://github.com/stackabletech/hdfs-operator/pull/334 [#349]: https://github.com/stackabletech/hdfs-operator/pull/349 [#350]: https://github.com/stackabletech/hdfs-operator/pull/350 [#351]: https://github.com/stackabletech/hdfs-operator/pull/351 diff --git a/Cargo.lock b/Cargo.lock index c5ffa0de..cac6673c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,9 +16,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +checksum = "6342bd4f5a1205d7f41e94a41a901f5647c938cdfa96036338e8533c9d6c2450" dependencies = [ "anstyle", "anstyle-parse", @@ -73,9 +73,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" [[package]] name = "byteorder" @@ -1112,9 +1112,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "libgit2-sys" @@ -1166,9 +1166,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c" +checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" [[package]] name = "lock_api" @@ -1269,9 +1269,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.50" +version = "0.10.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e30d8bc91859781f0a943411186324d580f2bbeb71b452fe91ae344806af3f1" +checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" dependencies = [ "bitflags", "cfg-if", @@ -1295,9 +1295,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.85" +version = "0.9.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3d193fb1488ad46ffe3aaabc912cc931d02ee8518fe2959aea8ef52718b0c0" +checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" dependencies = [ "cc", "libc", @@ -1571,13 +1571,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", ] [[package]] @@ -1586,7 +1586,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -1595,6 +1595,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + [[package]] name = "rstest" version = "0.17.0" @@ -1632,9 +1638,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.13" +version = "0.37.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" +checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f" dependencies = [ "bitflags", "errno", @@ -2221,13 +2227,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -2279,9 +2285,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index 7892dc72..fe5a275d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,5 +4,6 @@ members = [ "rust/crd", "rust/operator", "rust/operator-binary" ] -#[patch."https://github.com/stackabletech/operator-rs.git"] -#stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } +# [patch."https://github.com/stackabletech/operator-rs.git"] +# stackable-operator = { path = "/home/sbernauer/stackabletech/operator-rs" } +# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } diff --git a/deploy/helm/hdfs-operator/crds/crds.yaml b/deploy/helm/hdfs-operator/crds/crds.yaml index 09c1a7c3..ba38b861 100644 --- a/deploy/helm/hdfs-operator/crds/crds.yaml +++ b/deploy/helm/hdfs-operator/crds/crds.yaml @@ -26,6 +26,26 @@ spec: properties: clusterConfig: properties: + authentication: + description: Configuration to set up a cluster secured using Kerberos. + nullable: true + properties: + kerberos: + description: Kerberos configuration + properties: + secretClass: + description: Name of the SecretClass providing the keytab for the HDFS services. + type: string + required: + - secretClass + type: object + tlsSecretClass: + default: tls + description: Name of the SecretClass providing the tls certificates for the WebUIs. + type: string + required: + - kerberos + type: object autoFormatFs: nullable: true type: boolean diff --git a/docs/modules/hdfs/images/hdfs_webui_kerberos.png b/docs/modules/hdfs/images/hdfs_webui_kerberos.png new file mode 100644 index 00000000..9bffbf9d Binary files /dev/null and b/docs/modules/hdfs/images/hdfs_webui_kerberos.png differ diff --git a/docs/modules/hdfs/pages/usage-guide/security.adoc b/docs/modules/hdfs/pages/usage-guide/security.adoc new file mode 100644 index 00000000..8926c6e9 --- /dev/null +++ b/docs/modules/hdfs/pages/usage-guide/security.adoc @@ -0,0 +1,78 @@ += Security + +== Authentication +Currently the only supported authentication mechanism is Kerberos, which is disabled by default. +For Kerberos to work a Kerberos KDC is needed, which the users needs to provide. +The xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation] states which kind of Kerberos servers are supported and how they can be configured. + +IMPORTANT: Kerberos is supported staring from HDFS version 3.3.x + +=== 1. Prepare Kerberos server +To configure HDFS to use Kerberos you first need to collect information about your Kerberos server, e.g. hostname and port. +Additionally you need a service-user, which the secret-operator uses to create create principals for the HDFS services. + +=== 2. Create Kerberos SecretClass +Afterwards you need to enter all the needed information into a SecretClass, as described in xref:home:secret-operator:secretclass.adoc#backend-kerberoskeytab[secret-operator documentation]. +The following guide assumes you have named your SecretClass `kerberos-hdfs`. + +=== 3. Configure HDFS to use SecretClass +The last step is to configure your HdfsCluster to use the newly created SecretClass. + +[source,yaml] +---- +spec: + clusterConfig: + authentication: + tlsSecretClass: tls # Optional, defaults to "tls" + kerberos: + secretClass: kerberos-hdfs # Put your SecretClass name in here +---- + +The `kerberos.secretClass` is used to give HDFS the possibility to request keytabs from the secret-operator. + +The `tlsSecretClass` is needed to request TLS certificates, used e.g. for the Web UIs. + + +=== 4. Verify that Kerberos is used +Use `stackablectl services list --all-namespaces` to get the endpoints where the HDFS namenodes are reachable. +Open the link (note that the namenode is now using https). +You should see a Web UI similar to the following: + +image:hdfs_webui_kerberos.png[] + +The important part is + +> Security is on. + +You can also shell into the namenode and try to access the file system: +`kubectl exec -it hdfs-namenode-default-0 -c namenode -- bash -c 'kdestroy && bin/hdfs dfs -ls /'` + +You should get the error message `org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS]`. + +=== 5. Access HDFS +In case you want to access your HDFS it is recommended to start up a client Pod that connects to HDFS, rather than shelling into the namenode. +We have an https://github.com/stackabletech/hdfs-operator/blob/main/tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2[integration test] for this exact purpose, where you can see how to connect and get a valid keytab. + +== Authorization +We currently don't support authorization yet. +In the future support will be added by writing an opa-authorizer to match our general xref:home:concepts:opa.adoc[] mechanisms. + +In the meantime a very basic level of authorization can be reached by using `configOverrides` to set the `hadoop.user.group.static.mapping.overrides` property. +In thew following example the `dr.who=;nn=;nm=;jn=;` part is needed for HDFS internal operations and the user `testuser` is granted admin permissions. + +[source,yaml] +---- +spec: + nameNodes: + configOverrides: &configOverrides + core-site.xml: + hadoop.user.group.static.mapping.overrides: "dr.who=;nn=;nm=;jn=;testuser=supergroup;" + dataNodes: + configOverrides: *configOverrides + journalNodes: + configOverrides: *configOverrides +---- + +== Wire encryption +In case kerberos is enabled, `Privacy` mode is used for best security. +Wire encryption without kerberos as well as https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SecureMode.html#Data_confidentiality[other wire encryption modes] are *not* supported. diff --git a/docs/modules/hdfs/partials/nav.adoc b/docs/modules/hdfs/partials/nav.adoc index 266f4f44..4db08468 100644 --- a/docs/modules/hdfs/partials/nav.adoc +++ b/docs/modules/hdfs/partials/nav.adoc @@ -7,6 +7,7 @@ * xref:hdfs:usage-guide/index.adoc[] ** xref:hdfs:usage-guide/resources.adoc[] ** xref:hdfs:usage-guide/logging-log-aggregation.adoc[] +** xref:hdfs:usage-guide/security.adoc[] ** xref:hdfs:usage-guide/monitoring.adoc[] ** xref:hdfs:usage-guide/scaling.adoc[] ** xref:hdfs:usage-guide/configuration-environment-overrides.adoc[] diff --git a/rust/crd/src/constants.rs b/rust/crd/src/constants.rs index e186b05c..55a4463b 100644 --- a/rust/crd/src/constants.rs +++ b/rust/crd/src/constants.rs @@ -12,6 +12,9 @@ pub const LABEL_STS_POD_NAME: &str = "statefulset.kubernetes.io/pod-name"; pub const HDFS_SITE_XML: &str = "hdfs-site.xml"; pub const CORE_SITE_XML: &str = "core-site.xml"; +pub const HADOOP_POLICY_XML: &str = "hadoop-policy.xml"; +pub const SSL_SERVER_XML: &str = "ssl-server.xml"; +pub const SSL_CLIENT_XML: &str = "ssl-client.xml"; pub const LOG4J_PROPERTIES: &str = "log4j.properties"; pub const SERVICE_PORT_NAME_RPC: &str = "rpc"; @@ -23,10 +26,12 @@ pub const SERVICE_PORT_NAME_METRICS: &str = "metrics"; pub const DEFAULT_NAME_NODE_METRICS_PORT: u16 = 8183; pub const DEFAULT_NAME_NODE_HTTP_PORT: u16 = 9870; +pub const DEFAULT_NAME_NODE_HTTPS_PORT: u16 = 9871; pub const DEFAULT_NAME_NODE_RPC_PORT: u16 = 8020; pub const DEFAULT_DATA_NODE_METRICS_PORT: u16 = 8082; pub const DEFAULT_DATA_NODE_HTTP_PORT: u16 = 9864; +pub const DEFAULT_DATA_NODE_HTTPS_PORT: u16 = 9865; pub const DEFAULT_DATA_NODE_DATA_PORT: u16 = 9866; pub const DEFAULT_DATA_NODE_IPC_PORT: u16 = 9867; @@ -40,6 +45,7 @@ pub const DFS_NAMENODE_NAME_DIR: &str = "dfs.namenode.name.dir"; pub const DFS_NAMENODE_SHARED_EDITS_DIR: &str = "dfs.namenode.shared.edits.dir"; pub const DFS_NAMENODE_RPC_ADDRESS: &str = "dfs.namenode.rpc-address"; pub const DFS_NAMENODE_HTTP_ADDRESS: &str = "dfs.namenode.http-address"; +pub const DFS_NAMENODE_HTTPS_ADDRESS: &str = "dfs.namenode.https-address"; pub const DFS_DATANODE_DATA_DIR: &str = "dfs.datanode.data.dir"; pub const DFS_JOURNALNODE_EDITS_DIR: &str = "dfs.journalnode.edits.dir"; pub const DFS_JOURNALNODE_RPC_ADDRESS: &str = "dfs.journalnode.rpc-address"; diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index 7bf71326..f4f315dc 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -1,9 +1,11 @@ pub mod affinity; pub mod constants; +pub mod security; pub mod storage; use affinity::get_affinity; use constants::*; +use security::{AuthenticationConfig, KerberosConfig}; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; use stackable_operator::{ @@ -75,6 +77,7 @@ pub struct HdfsClusterSpec { pub data_nodes: Option>, #[serde(default, skip_serializing_if = "Option::is_none")] pub journal_nodes: Option>, + // Cluster wide configuration pub cluster_config: HdfsClusterConfig, /// Cluster operations like pause reconciliation or cluster stop. #[serde(default)] @@ -103,6 +106,8 @@ pub struct HdfsClusterConfig { /// * external-unstable: Use a NodePort service #[serde(default)] pub listener_class: CurrentlySupportedListenerClasses, + /// Configuration to set up a cluster secured using Kerberos. + pub authentication: Option, } // TODO: Temporary solution until listener-operator is finished @@ -217,62 +222,6 @@ impl HdfsRole { } } - /// Returns required port name and port number tuples depending on the role. - pub fn ports(&self) -> Vec<(String, u16)> { - match self { - HdfsRole::NameNode => vec![ - ( - String::from(SERVICE_PORT_NAME_METRICS), - DEFAULT_NAME_NODE_METRICS_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_HTTP), - DEFAULT_NAME_NODE_HTTP_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_RPC), - DEFAULT_NAME_NODE_RPC_PORT, - ), - ], - HdfsRole::DataNode => vec![ - ( - String::from(SERVICE_PORT_NAME_METRICS), - DEFAULT_DATA_NODE_METRICS_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_DATA), - DEFAULT_DATA_NODE_DATA_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_HTTP), - DEFAULT_DATA_NODE_HTTP_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_IPC), - DEFAULT_DATA_NODE_IPC_PORT, - ), - ], - HdfsRole::JournalNode => vec![ - ( - String::from(SERVICE_PORT_NAME_METRICS), - DEFAULT_JOURNAL_NODE_METRICS_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_HTTP), - DEFAULT_JOURNAL_NODE_HTTP_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_HTTPS), - DEFAULT_JOURNAL_NODE_HTTPS_PORT, - ), - ( - String::from(SERVICE_PORT_NAME_RPC), - DEFAULT_JOURNAL_NODE_RPC_PORT, - ), - ], - } - } - /// Merge the [Name|Data|Journal]NodeConfigFragment defaults, role and role group settings. /// The priority is: default < role config < role_group config pub fn merged_config( @@ -403,7 +352,7 @@ impl HdfsRole { } /// Name of the Hadoop process HADOOP_OPTS. - pub fn hadoop_opts(&self) -> &'static str { + pub fn hadoop_opts_env_var_for_role(&self) -> &'static str { match self { HdfsRole::NameNode => "HDFS_NAMENODE_OPTS", HdfsRole::DataNode => "HDFS_DATANODE_OPTS", @@ -411,6 +360,14 @@ impl HdfsRole { } } + pub fn kerberos_service_name(&self) -> &'static str { + match self { + HdfsRole::NameNode => "nn", + HdfsRole::DataNode => "dn", + HdfsRole::JournalNode => "jn", + } + } + /// Return replicas for a certain rolegroup. pub fn role_group_replicas(&self, hdfs: &HdfsCluster, role_group: &str) -> i32 { match self { @@ -537,7 +494,11 @@ impl HdfsCluster { namespace: ns.clone(), role_group_service_name: rolegroup_ref.object_name(), pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), - ports: role.ports().iter().map(|(n, p)| (n.clone(), *p)).collect(), + ports: self + .ports(role) + .iter() + .map(|(n, p)| (n.clone(), *p)) + .collect(), }) }) .collect()) @@ -612,6 +573,9 @@ impl HdfsCluster { let pnk = vec![ PropertyNameKind::File(HDFS_SITE_XML.to_string()), PropertyNameKind::File(CORE_SITE_XML.to_string()), + PropertyNameKind::File(HADOOP_POLICY_XML.to_string()), + PropertyNameKind::File(SSL_SERVER_XML.to_string()), + PropertyNameKind::File(SSL_CLIENT_XML.to_string()), PropertyNameKind::Env, ]; @@ -650,6 +614,107 @@ impl HdfsCluster { Ok(result) } + + pub fn authentication_config(&self) -> Option<&AuthenticationConfig> { + self.spec.cluster_config.authentication.as_ref() + } + + pub fn has_kerberos_enabled(&self) -> bool { + self.kerberos_config().is_some() + } + + pub fn kerberos_config(&self) -> Option<&KerberosConfig> { + self.spec + .cluster_config + .authentication + .as_ref() + .map(|s| &s.kerberos) + } + + pub fn has_https_enabled(&self) -> bool { + self.https_secret_class().is_some() + } + + pub fn https_secret_class(&self) -> Option<&str> { + self.spec + .cluster_config + .authentication + .as_ref() + .map(|k| k.tls_secret_class.as_str()) + } + + /// Returns required port name and port number tuples depending on the role. + pub fn ports(&self, role: &HdfsRole) -> Vec<(String, u16)> { + match role { + HdfsRole::NameNode => vec![ + ( + String::from(SERVICE_PORT_NAME_METRICS), + DEFAULT_NAME_NODE_METRICS_PORT, + ), + ( + String::from(SERVICE_PORT_NAME_RPC), + DEFAULT_NAME_NODE_RPC_PORT, + ), + if self.has_https_enabled() { + ( + String::from(SERVICE_PORT_NAME_HTTPS), + DEFAULT_NAME_NODE_HTTPS_PORT, + ) + } else { + ( + String::from(SERVICE_PORT_NAME_HTTP), + DEFAULT_NAME_NODE_HTTP_PORT, + ) + }, + ], + HdfsRole::DataNode => vec![ + ( + String::from(SERVICE_PORT_NAME_METRICS), + DEFAULT_DATA_NODE_METRICS_PORT, + ), + ( + String::from(SERVICE_PORT_NAME_DATA), + DEFAULT_DATA_NODE_DATA_PORT, + ), + ( + String::from(SERVICE_PORT_NAME_IPC), + DEFAULT_DATA_NODE_IPC_PORT, + ), + if self.has_https_enabled() { + ( + String::from(SERVICE_PORT_NAME_HTTPS), + DEFAULT_DATA_NODE_HTTPS_PORT, + ) + } else { + ( + String::from(SERVICE_PORT_NAME_HTTP), + DEFAULT_DATA_NODE_HTTP_PORT, + ) + }, + ], + HdfsRole::JournalNode => vec![ + ( + String::from(SERVICE_PORT_NAME_METRICS), + DEFAULT_JOURNAL_NODE_METRICS_PORT, + ), + ( + String::from(SERVICE_PORT_NAME_RPC), + DEFAULT_JOURNAL_NODE_RPC_PORT, + ), + if self.has_https_enabled() { + ( + String::from(SERVICE_PORT_NAME_HTTPS), + DEFAULT_JOURNAL_NODE_HTTPS_PORT, + ) + } else { + ( + String::from(SERVICE_PORT_NAME_HTTP), + DEFAULT_JOURNAL_NODE_HTTP_PORT, + ) + }, + ], + } + } } /// Reference to a single `Pod` that is a component of a [`HdfsCluster`] /// diff --git a/rust/crd/src/security.rs b/rust/crd/src/security.rs new file mode 100644 index 00000000..456e08a9 --- /dev/null +++ b/rust/crd/src/security.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; +use stackable_operator::schemars::{self, JsonSchema}; + +#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct AuthenticationConfig { + /// Name of the SecretClass providing the tls certificates for the WebUIs. + #[serde(default = "default_tls_secret_class")] + pub tls_secret_class: String, + /// Kerberos configuration + pub kerberos: KerberosConfig, +} + +fn default_tls_secret_class() -> String { + "tls".to_string() +} + +#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct KerberosConfig { + /// Name of the SecretClass providing the keytab for the HDFS services. + pub secret_class: String, +} diff --git a/rust/operator/src/config.rs b/rust/operator/src/config.rs index f4760cbb..ec5f1997 100644 --- a/rust/operator/src/config.rs +++ b/rust/operator/src/config.rs @@ -1,12 +1,13 @@ use stackable_hdfs_crd::constants::{ - DEFAULT_JOURNAL_NODE_RPC_PORT, DEFAULT_NAME_NODE_HTTP_PORT, DEFAULT_NAME_NODE_RPC_PORT, - DFS_DATANODE_DATA_DIR, DFS_HA_NAMENODES, DFS_JOURNALNODE_EDITS_DIR, - DFS_JOURNALNODE_RPC_ADDRESS, DFS_NAMENODE_HTTP_ADDRESS, DFS_NAMENODE_NAME_DIR, - DFS_NAMENODE_RPC_ADDRESS, DFS_NAMENODE_SHARED_EDITS_DIR, DFS_NAME_SERVICES, DFS_REPLICATION, - FS_DEFAULT_FS, HA_ZOOKEEPER_QUORUM, JOURNALNODE_ROOT_DATA_DIR, NAMENODE_ROOT_DATA_DIR, + DEFAULT_JOURNAL_NODE_RPC_PORT, DEFAULT_NAME_NODE_HTTPS_PORT, DEFAULT_NAME_NODE_HTTP_PORT, + DEFAULT_NAME_NODE_RPC_PORT, DFS_DATANODE_DATA_DIR, DFS_HA_NAMENODES, DFS_JOURNALNODE_EDITS_DIR, + DFS_JOURNALNODE_RPC_ADDRESS, DFS_NAMENODE_HTTPS_ADDRESS, DFS_NAMENODE_HTTP_ADDRESS, + DFS_NAMENODE_NAME_DIR, DFS_NAMENODE_RPC_ADDRESS, DFS_NAMENODE_SHARED_EDITS_DIR, + DFS_NAME_SERVICES, DFS_REPLICATION, FS_DEFAULT_FS, HA_ZOOKEEPER_QUORUM, + JOURNALNODE_ROOT_DATA_DIR, NAMENODE_ROOT_DATA_DIR, }; use stackable_hdfs_crd::storage::{DataNodeStorageConfig, DataNodeStorageConfigInnerType}; -use stackable_hdfs_crd::HdfsPodRef; +use stackable_hdfs_crd::{HdfsCluster, HdfsPodRef}; use std::collections::BTreeMap; #[derive(Clone)] @@ -23,8 +24,8 @@ impl HdfsSiteConfigBuilder { } } - pub fn add(&mut self, property: &str, value: &str) -> &mut Self { - self.config.insert(property.to_string(), value.to_string()); + pub fn add(&mut self, property: impl Into, value: impl Into) -> &mut Self { + self.config.insert(property.into(), value.into()); self } @@ -144,12 +145,24 @@ impl HdfsSiteConfigBuilder { self } - pub fn dfs_namenode_http_address_ha(&mut self, namenode_podrefs: &[HdfsPodRef]) -> &mut Self { - self.dfs_namenode_address_ha( - namenode_podrefs, - DFS_NAMENODE_HTTP_ADDRESS, - DEFAULT_NAME_NODE_HTTP_PORT, - ); + pub fn dfs_namenode_http_address_ha( + &mut self, + hdfs: &HdfsCluster, + namenode_podrefs: &[HdfsPodRef], + ) -> &mut Self { + if hdfs.has_https_enabled() { + self.dfs_namenode_address_ha( + namenode_podrefs, + DFS_NAMENODE_HTTPS_ADDRESS, + DEFAULT_NAME_NODE_HTTPS_PORT, + ); + } else { + self.dfs_namenode_address_ha( + namenode_podrefs, + DFS_NAMENODE_HTTP_ADDRESS, + DEFAULT_NAME_NODE_HTTP_PORT, + ); + } self } @@ -193,6 +206,16 @@ impl CoreSiteConfigBuilder { } } + pub fn add(&mut self, property: impl Into, value: impl Into) -> &mut Self { + self.config.insert(property.into(), value.into()); + self + } + + pub fn extend(&mut self, properties: &BTreeMap) -> &mut Self { + self.config.extend(properties.clone()); + self + } + pub fn fs_default_fs(&mut self) -> &mut Self { self.config.insert( FS_DEFAULT_FS.to_string(), @@ -209,11 +232,6 @@ impl CoreSiteConfigBuilder { self } - pub fn extend(&mut self, properties: &BTreeMap) -> &mut Self { - self.config.extend(properties.clone()); - self - } - pub fn build_as_xml(&self) -> String { let transformed_config = transform_for_product_config(&self.config); stackable_operator::product_config::writer::to_hadoop_xml(transformed_config.iter()) diff --git a/rust/operator/src/container.rs b/rust/operator/src/container.rs index 88db0594..9174dd49 100644 --- a/rust/operator/src/container.rs +++ b/rust/operator/src/container.rs @@ -9,26 +9,32 @@ //! - Set resources //! - Add tcp probes and container ports (to the main containers) //! -use crate::product_logging::{ - FORMAT_NAMENODES_LOG4J_CONFIG_FILE, FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, HDFS_LOG4J_CONFIG_FILE, - MAX_LOG_FILES_SIZE_IN_MIB, STACKABLE_LOG_DIR, WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, - ZKFC_LOG4J_CONFIG_FILE, +use crate::{ + hdfs_controller::KEYSTORE_DIR_NAME, + product_logging::{ + FORMAT_NAMENODES_LOG4J_CONFIG_FILE, FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, + HDFS_LOG4J_CONFIG_FILE, MAX_LOG_FILES_SIZE_IN_MIB, STACKABLE_LOG_DIR, + WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, ZKFC_LOG4J_CONFIG_FILE, + }, }; use indoc::formatdoc; -use snafu::{ResultExt, Snafu}; +use snafu::{OptionExt, ResultExt, Snafu}; use stackable_hdfs_crd::{ constants::{ - CORE_SITE_XML, DATANODE_ROOT_DATA_DIR_PREFIX, DEFAULT_DATA_NODE_METRICS_PORT, - DEFAULT_JOURNAL_NODE_METRICS_PORT, DEFAULT_NAME_NODE_METRICS_PORT, HDFS_SITE_XML, - LOG4J_PROPERTIES, NAMENODE_ROOT_DATA_DIR, SERVICE_PORT_NAME_IPC, SERVICE_PORT_NAME_RPC, + DATANODE_ROOT_DATA_DIR_PREFIX, DEFAULT_DATA_NODE_METRICS_PORT, + DEFAULT_JOURNAL_NODE_METRICS_PORT, DEFAULT_NAME_NODE_METRICS_PORT, LOG4J_PROPERTIES, + NAMENODE_ROOT_DATA_DIR, SERVICE_PORT_NAME_IPC, SERVICE_PORT_NAME_RPC, STACKABLE_ROOT_DATA_DIR, }, storage::DataNodeStorageConfig, - DataNodeContainer, HdfsPodRef, HdfsRole, MergedConfig, NameNodeContainer, + DataNodeContainer, HdfsCluster, HdfsPodRef, HdfsRole, MergedConfig, NameNodeContainer, }; use stackable_operator::{ - builder::{ContainerBuilder, PodBuilder, VolumeBuilder, VolumeMountBuilder}, + builder::{ + ContainerBuilder, PodBuilder, SecretOperatorVolumeSourceBuilder, VolumeBuilder, + VolumeMountBuilder, + }, commons::product_image_selection::ResolvedProductImage, k8s_openapi::{ api::core::v1::{ @@ -51,6 +57,8 @@ use strum::{Display, EnumDiscriminants, IntoStaticStr}; #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { + #[snafu(display("object has no namespace"))] + ObjectHasNoNamespace, #[snafu(display("Invalid java heap config for [{role}]"))] InvalidJavaHeapConfig { source: stackable_operator::error::Error, @@ -138,6 +146,7 @@ impl ContainerConfig { #[allow(clippy::too_many_arguments)] pub fn add_containers_and_volumes( pb: &mut PodBuilder, + hdfs: &HdfsCluster, role: &HdfsRole, resolved_product_image: &ResolvedProductImage, merged_config: &(dyn MergedConfig + Send + 'static), @@ -150,6 +159,8 @@ impl ContainerConfig { let main_container_config = Self::from(role.clone()); pb.add_volumes(main_container_config.volumes(merged_config, object_name)); pb.add_container(main_container_config.main_container( + hdfs, + role, resolved_product_image, zk_config_map_name, env_overrides, @@ -166,6 +177,65 @@ impl ContainerConfig { )); } + if let Some(authentication_config) = hdfs.authentication_config() { + pb.add_volume( + VolumeBuilder::new("tls") + .ephemeral( + SecretOperatorVolumeSourceBuilder::new( + &authentication_config.tls_secret_class, + ) + .with_pod_scope() + .with_node_scope() + .build(), + ) + .build(), + ); + + pb.add_volume( + VolumeBuilder::new("keystore") + .with_empty_dir(Option::::None, None) + .build(), + ); + + pb.add_volume( + VolumeBuilder::new("kerberos") + .ephemeral( + SecretOperatorVolumeSourceBuilder::new( + &authentication_config.kerberos.secret_class, + ) + .with_service_scope(hdfs.name_any()) + .with_kerberos_service_name(role.kerberos_service_name()) + .with_kerberos_service_name("HTTP") + .build(), + ) + .build(), + ); + + let create_tls_cert_bundle_init_container = + ContainerBuilder::new("create-tls-cert-bundle") + .unwrap() + .image_from_product_image(resolved_product_image) + .command(Self::command()) + .args(vec![formatdoc!( + r###" + echo "Cleaning up truststore - just in case" + rm -f {KEYSTORE_DIR_NAME}/truststore.p12 + echo "Creating truststore" + keytool -importcert -file /stackable/tls/ca.crt -keystore {KEYSTORE_DIR_NAME}/truststore.p12 -storetype pkcs12 -noprompt -alias ca_cert -storepass changeit + echo "Creating certificate chain" + cat /stackable/tls/ca.crt /stackable/tls/tls.crt > {KEYSTORE_DIR_NAME}/chain.crt + echo "Cleaning up keystore - just in case" + rm -f {KEYSTORE_DIR_NAME}/keystore.p12 + echo "Creating keystore" + openssl pkcs12 -export -in {KEYSTORE_DIR_NAME}/chain.crt -inkey /stackable/tls/tls.key -out {KEYSTORE_DIR_NAME}/keystore.p12 --passout pass:changeit"### + )]) + // Only this init container needs the actual cert (from tls volume) to create the truststore + keystore from + .add_volume_mount("tls", "/stackable/tls") + .add_volume_mount("keystore", KEYSTORE_DIR_NAME) + .build(); + pb.add_init_container(create_tls_cert_bundle_init_container); + } + // role specific pod settings configured here match role { HdfsRole::NameNode => { @@ -173,6 +243,8 @@ impl ContainerConfig { let zkfc_container_config = Self::try_from(NameNodeContainer::Zkfc.to_string())?; pb.add_volumes(zkfc_container_config.volumes(merged_config, object_name)); pb.add_container(zkfc_container_config.main_container( + hdfs, + role, resolved_product_image, zk_config_map_name, env_overrides, @@ -186,6 +258,8 @@ impl ContainerConfig { format_namenodes_container_config.volumes(merged_config, object_name), ); pb.add_init_container(format_namenodes_container_config.init_container( + hdfs, + role, resolved_product_image, zk_config_map_name, env_overrides, @@ -200,6 +274,8 @@ impl ContainerConfig { format_zookeeper_container_config.volumes(merged_config, object_name), ); pb.add_init_container(format_zookeeper_container_config.init_container( + hdfs, + role, resolved_product_image, zk_config_map_name, env_overrides, @@ -215,6 +291,8 @@ impl ContainerConfig { wait_for_namenodes_container_config.volumes(merged_config, object_name), ); pb.add_init_container(wait_for_namenodes_container_config.init_container( + hdfs, + role, resolved_product_image, zk_config_map_name, env_overrides, @@ -253,6 +331,8 @@ impl ContainerConfig { /// - Journalnode main process fn main_container( &self, + hdfs: &HdfsCluster, + role: &HdfsRole, resolved_product_image: &ResolvedProductImage, zookeeper_config_map_name: &str, env_overrides: Option<&BTreeMap>, @@ -266,11 +346,16 @@ impl ContainerConfig { let resources = self.resources(merged_config); cb.image_from_product_image(resolved_product_image) - .command(self.command()) - .args(self.args(merged_config, &[])) - .add_env_vars(self.env(zookeeper_config_map_name, env_overrides, resources.as_ref())) - .add_volume_mounts(self.volume_mounts(merged_config)) - .add_container_ports(self.container_ports()); + .command(Self::command()) + .args(self.args(hdfs, role, merged_config, &[])?) + .add_env_vars(self.env( + hdfs, + zookeeper_config_map_name, + env_overrides, + resources.as_ref(), + )) + .add_volume_mounts(self.volume_mounts(hdfs, merged_config)) + .add_container_ports(self.container_ports(hdfs)); if let Some(resources) = resources { cb.resources(resources); @@ -287,8 +372,11 @@ impl ContainerConfig { /// Creates respective init containers for: /// - Namenode (format-namenodes, format-zookeeper) /// - Datanode (wait-for-namenodes) + #[allow(clippy::too_many_arguments)] fn init_container( &self, + hdfs: &HdfsCluster, + role: &HdfsRole, resolved_product_image: &ResolvedProductImage, zookeeper_config_map_name: &str, env_overrides: Option<&BTreeMap>, @@ -298,10 +386,10 @@ impl ContainerConfig { Ok(ContainerBuilder::new(self.name()) .with_context(|_| InvalidContainerNameSnafu { name: self.name() })? .image_from_product_image(resolved_product_image) - .command(self.command()) - .args(self.args(merged_config, namenode_podrefs)) - .add_env_vars(self.env(zookeeper_config_map_name, env_overrides, None)) - .add_volume_mounts(self.volume_mounts(merged_config)) + .command(Self::command()) + .args(self.args(hdfs, role, merged_config, namenode_podrefs)?) + .add_env_vars(self.env(hdfs, zookeeper_config_map_name, env_overrides, None)) + .add_volume_mounts(self.volume_mounts(hdfs, merged_config)) .build()) } @@ -328,61 +416,62 @@ impl ContainerConfig { } /// Returns the container command. - fn command(&self) -> Vec { - match self { - ContainerConfig::Hdfs { .. } | ContainerConfig::Zkfc { .. } => vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ], - ContainerConfig::FormatNameNodes { .. } - | ContainerConfig::FormatZooKeeper { .. } - | ContainerConfig::WaitForNameNodes { .. } => { - vec!["bash".to_string(), "-c".to_string()] - } - } + fn command() -> Vec { + vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ] } /// Returns the container command arguments. fn args( &self, + hdfs: &HdfsCluster, + role: &HdfsRole, merged_config: &(dyn MergedConfig + Send + 'static), namenode_podrefs: &[HdfsPodRef], - ) -> Vec { - let mut args = vec![ - self.create_config_directory_cmd(), - self.copy_hdfs_and_core_site_xml_cmd(), - ]; + ) -> Result, Error> { + let mut args = String::new(); + args.push_str(&self.create_config_directory_cmd()); + args.push_str(&self.copy_config_xml_cmd()); + + // We can't influence the order of the init containers. + // Some init containers - such as format-namenodes - need the tls certs or kerberos tickets, so let's wait for them to be properly set up + if hdfs.has_https_enabled() { + args.push_str(&Self::wait_for_trust_and_keystore_command()); + } + if hdfs.has_kerberos_enabled() { + args.push_str(&Self::export_kerberos_real_env_var_command()); + } match self { ContainerConfig::Hdfs { role, .. } => { - args.push(self.copy_log4j_properties_cmd( + args.push_str(&self.copy_log4j_properties_cmd( HDFS_LOG4J_CONFIG_FILE, merged_config.hdfs_logging(), )); - - args.push(format!( - "{hadoop_home}/bin/hdfs {role}", + args.push_str(&format!( + "{hadoop_home}/bin/hdfs {role}\n", hadoop_home = Self::HADOOP_HOME, )); } ContainerConfig::Zkfc { .. } => { if let Some(container_config) = merged_config.zkfc_logging() { - args.push( - self.copy_log4j_properties_cmd(ZKFC_LOG4J_CONFIG_FILE, container_config), + args.push_str( + &self.copy_log4j_properties_cmd(ZKFC_LOG4J_CONFIG_FILE, container_config), ); } - - args.push(format!( - "{hadoop_home}/bin/hdfs zkfc", + args.push_str(&format!( + "{hadoop_home}/bin/hdfs zkfc\n", hadoop_home = Self::HADOOP_HOME )); } ContainerConfig::FormatNameNodes { .. } => { if let Some(container_config) = merged_config.format_namenodes_logging() { - args.push(self.copy_log4j_properties_cmd( + args.push_str(&self.copy_log4j_properties_cmd( FORMAT_NAMENODES_LOG4J_CONFIG_FILE, container_config, )); @@ -391,25 +480,28 @@ impl ContainerConfig { // for e.g. scaling. It may fail if the active namenode is restarted and the standby // namenode takes over. // This is why in the second part we check if the node is formatted already via - // $NAMENODE_DIR/current/VERSION. Then we dont do anything. + // $NAMENODE_DIR/current/VERSION. Then we don't do anything. // If there is no active namenode, the current pod is not formatted we format as // active namenode. Otherwise as standby node. - args.push(formatdoc!(r###" + if hdfs.has_kerberos_enabled() { + args.push_str(&Self::get_kerberos_ticket(hdfs, role)?); + } + args.push_str(&formatdoc!( + r###" echo "Start formatting namenode $POD_NAME. Checking for active namenodes:" - for id in {pod_names} + for namenode_id in {pod_names} do - echo -n "Checking pod $id... " - SERVICE_STATE=$({hadoop_home}/bin/hdfs haadmin -getServiceState $id 2>/dev/null) + echo -n "Checking pod $namenode_id... " + {get_service_state_command} if [ "$SERVICE_STATE" == "active" ] then - ACTIVE_NAMENODE=$id + ACTIVE_NAMENODE=$namenode_id echo "active" break fi echo "" done - set -e if [ ! -f "{NAMENODE_ROOT_DATA_DIR}/current/VERSION" ] then if [ -z ${{ACTIVE_NAMENODE+x}} ] @@ -421,8 +513,11 @@ impl ContainerConfig { {hadoop_home}/bin/hdfs namenode -bootstrapStandby -nonInteractive fi else + cat "{NAMENODE_ROOT_DATA_DIR}/current/VERSION" echo "Pod $POD_NAME already formatted. Skipping..." - fi"###, + fi + "###, + get_service_state_command = Self::get_namenode_service_state_command(), hadoop_home = Self::HADOOP_HOME, pod_names = namenode_podrefs .iter() @@ -433,39 +528,56 @@ impl ContainerConfig { } ContainerConfig::FormatZooKeeper { .. } => { if let Some(container_config) = merged_config.format_zookeeper_logging() { - args.push(self.copy_log4j_properties_cmd( + args.push_str(&self.copy_log4j_properties_cmd( FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, container_config, )); } - args.push(formatdoc!( + args.push_str(&formatdoc!( r###" echo "Attempt to format ZooKeeper..." if [[ "0" -eq "$(echo $POD_NAME | sed -e 's/.*-//')" ]] ; then - {hadoop_home}/bin/hdfs zkfc -formatZK -nonInteractive || true + set +e + {hadoop_home}/bin/hdfs zkfc -formatZK -nonInteractive + EXITCODE=$? + set -e + if [[ $EXITCODE -eq 0 ]]; then + echo "Successfully formatted" + elif [[ $EXITCODE -eq 2 ]]; then + echo "ZNode already existed, did nothing" + else + echo "Zookeeper format failed with exit code $EXITCODE" + exit $EXITCODE + fi + else echo "ZooKeeper already formatted!" - fi"###, + fi + "###, hadoop_home = Self::HADOOP_HOME )); } ContainerConfig::WaitForNameNodes { .. } => { if let Some(container_config) = merged_config.wait_for_namenodes() { - args.push(self.copy_log4j_properties_cmd( + args.push_str(&self.copy_log4j_properties_cmd( WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, container_config, )); } - args.push(formatdoc!(r###" + if hdfs.has_kerberos_enabled() { + args.push_str(&Self::get_kerberos_ticket(hdfs, role)?); + } + args.push_str(&formatdoc!( + r###" echo "Waiting for namenodes to get ready:" n=0 while [ ${{n}} -lt 12 ]; do ALL_NODES_READY=true - for id in {pod_names} + for namenode_id in {pod_names} do - echo -n "Checking pod $id... " - SERVICE_STATE=$({hadoop_home}/bin/hdfs haadmin -getServiceState $id 2>/dev/null) + echo -n "Checking pod $namenode_id... " + {get_service_state_command} if [ "$SERVICE_STATE" = "active" ] || [ "$SERVICE_STATE" = "standby" ] then echo "$SERVICE_STATE" @@ -482,43 +594,124 @@ impl ContainerConfig { echo "" n=$(( n + 1)) sleep 5 - done"###, - hadoop_home = Self::HADOOP_HOME, - pod_names = namenode_podrefs - .iter() - .map(|pod_ref| pod_ref.pod_name.as_ref()) - .collect::>() - .join(" ") + done + "###, + get_service_state_command = Self::get_namenode_service_state_command(), + pod_names = namenode_podrefs + .iter() + .map(|pod_ref| pod_ref.pod_name.as_ref()) + .collect::>() + .join(" ") )); } } - vec![args.join(" && ")] + Ok(vec![args]) + } + + /// Wait until the init container has created global trust and keystore shared between all containers + fn wait_for_trust_and_keystore_command() -> String { + formatdoc!( + r###"until [ -f {KEYSTORE_DIR_NAME}/truststore.p12 ]; do + echo 'Waiting for truststore to be created' + sleep 1 + done + until [ -f {KEYSTORE_DIR_NAME}/keystore.p12 ]; do + echo 'Waiting for keystore to be created' + sleep 1 + done + "### + ) + } + + // Command to export `KERBEROS_REALM` env var to default real from krb5.conf, e.g. `CLUSTER.LOCAL` + fn export_kerberos_real_env_var_command() -> String { + "export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' /stackable/kerberos/krb5.conf)\n" + .to_string() + } + + /// Command to `kinit` a ticket using the principal created for the specified hdfs role + /// Needs the KERBEROS_REALM env var, which will be written with `export_kerberos_real_env_var_command` + /// Needs the POD_NAME env var to be present, which will be provided by the PodSpec + fn get_kerberos_ticket(hdfs: &HdfsCluster, role: &HdfsRole) -> Result { + let principal = format!( + "{service_name}/{hdfs_name}.{namespace}.svc.cluster.local@${{KERBEROS_REALM}}", + service_name = role.kerberos_service_name(), + hdfs_name = hdfs.name_any(), + namespace = hdfs.namespace().context(ObjectHasNoNamespaceSnafu)?, + ); + Ok(formatdoc!( + r###" + echo "Getting ticket for {principal}" from /stackable/kerberos/keytab + kinit "{principal}" -kt /stackable/kerberos/keytab + "###, + )) + } + + fn get_namenode_service_state_command() -> String { + formatdoc!( + r###" + SERVICE_STATE=$({hadoop_home}/bin/hdfs haadmin -getServiceState $namenode_id | tail -n1 || true)"###, + hadoop_home = Self::HADOOP_HOME, + ) } /// Returns the container env variables. fn env( &self, + hdfs: &HdfsCluster, zookeeper_config_map_name: &str, env_overrides: Option<&BTreeMap>, resources: Option<&ResourceRequirements>, ) -> Vec { - let mut env = Self::transform_env_overrides_to_env_vars(env_overrides); + let mut env = Vec::new(); env.extend(Self::shared_env_vars( self.volume_mount_dirs().final_config(), zookeeper_config_map_name, )); + // For the main container we use specialized env variables for every role + // (think of like HDFS_NAMENODE_OPTS or HDFS_DATANODE_OPTS) + // We do so, so that users shelling into the hdfs Pods will not have problems + // because the will read out the HADOOP_OPTS env var as well for the cli clients + // (but *not* the HDFS_NAMENODE_OPTS env var)! + // The hadoop opts contain a Prometheus metric emitter, which binds itself to a static port. + // When the users tries to start a cli tool the port is already taken by the hdfs services, + // so we don't want to stuff all the config into HADOOP_OPTS, but rather into the specialized env variables + // See https://github.com/stackabletech/hdfs-operator/issues/138 for details if let ContainerConfig::Hdfs { role, .. } = self { - if let Some(resources) = resources { - env.push(EnvVar { - name: role.hadoop_opts().to_string(), - value: self.build_hadoop_opts(resources).ok(), - ..EnvVar::default() - }); - } + env.push(EnvVar { + name: role.hadoop_opts_env_var_for_role().to_string(), + value: self.build_hadoop_opts(hdfs, resources).ok(), + ..EnvVar::default() + }); + } + // Additionally, any other init or sidecar container must have access to the following settings. + // As the Prometheus metric emitter is not part of this config it's safe to use for hdfs cli tools as well. + // This will not only enable the init containers to work, but also the user to run e.g. + // `bin/hdfs dfs -ls /` without getting `Caused by: java.lang.IllegalArgumentException: KrbException: Cannot locate default realm` + // because the `-Djava.security.krb5.conf` setting is missing + if hdfs.has_kerberos_enabled() { + env.push(EnvVar { + name: "HADOOP_OPTS".to_string(), + value: Some("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string()), + ..EnvVar::default() + }); + + env.push(EnvVar { + name: "KRB5_CONFIG".to_string(), + value: Some("/stackable/kerberos/krb5.conf".to_string()), + ..EnvVar::default() + }); + env.push(EnvVar { + name: "KRB5_CLIENT_KTNAME".to_string(), + value: Some("/stackable/kerberos/keytab".to_string()), + ..EnvVar::default() + }); } + // Overrides need to come last + env.extend(Self::transform_env_overrides_to_env_vars(env_overrides)); env } @@ -606,6 +799,7 @@ impl ContainerConfig { /// Returns the container volume mounts. fn volume_mounts( &self, + hdfs: &HdfsCluster, merged_config: &(dyn MergedConfig + Send + 'static), ) -> Vec { let mut volume_mounts = vec![ @@ -623,6 +817,15 @@ impl ContainerConfig { .build(), ]; + // Adding this for all containers, as not only the main container needs Kerberos or TLS + if hdfs.has_kerberos_enabled() { + volume_mounts.push(VolumeMountBuilder::new("kerberos", "/stackable/kerberos").build()); + } + if hdfs.has_https_enabled() { + // This volume will be propagated by the create-tls-cert-bundle container + volume_mounts.push(VolumeMountBuilder::new("keystore", KEYSTORE_DIR_NAME).build()); + } + match self { ContainerConfig::FormatNameNodes { .. } => { // As FormatNameNodes only runs on the Namenodes we can safely assume the only pvc is called "data". @@ -666,26 +869,18 @@ impl ContainerConfig { /// Create a config directory for the respective container. fn create_config_directory_cmd(&self) -> String { format!( - "mkdir -p {config_dir_name}", + "mkdir -p {config_dir_name}\n", config_dir_name = self.volume_mount_dirs().final_config() ) } - /// Copy the `core-site.xml` and `hdfs-site.xml` to the respective container config dir. - fn copy_hdfs_and_core_site_xml_cmd(&self) -> String { - vec![ - format!( - "cp {config_dir_mount}/{HDFS_SITE_XML} {config_dir_name}/{HDFS_SITE_XML}", - config_dir_mount = self.volume_mount_dirs().config_mount(), - config_dir_name = self.volume_mount_dirs().final_config() - ), - format!( - "cp {config_dir_mount}/{CORE_SITE_XML} {config_dir_name}/{CORE_SITE_XML}", - config_dir_mount = self.volume_mount_dirs().config_mount(), - config_dir_name = self.volume_mount_dirs().final_config() - ), - ] - .join(" && ") + /// Copy all the configuration files to the respective container config dir. + fn copy_config_xml_cmd(&self) -> String { + format!( + "cp {config_dir_mount}/*.xml {config_dir_name}\n", + config_dir_mount = self.volume_mount_dirs().config_mount(), + config_dir_name = self.volume_mount_dirs().final_config() + ) } /// Copy the `log4j.properties` to the respective container config dir. @@ -707,7 +902,7 @@ impl ContainerConfig { }; format!( - "cp {log4j_properties_dir}/{file_name} {config_dir}/{LOG4J_PROPERTIES}", + "cp {log4j_properties_dir}/{file_name} {config_dir}/{LOG4J_PROPERTIES}\n", log4j_properties_dir = source_log4j_properties_dir, file_name = log4j_config_file, config_dir = self.volume_mount_dirs().final_config() @@ -715,7 +910,11 @@ impl ContainerConfig { } /// Build HADOOP_{*node}_OPTS for each namenode, datanodes and journalnodes. - fn build_hadoop_opts(&self, resources: &ResourceRequirements) -> Result { + fn build_hadoop_opts( + &self, + hdfs: &HdfsCluster, + resources: Option<&ResourceRequirements>, + ) -> Result { match self { ContainerConfig::Hdfs { role, metrics_port, .. @@ -724,8 +923,14 @@ impl ContainerConfig { format!( "-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar={metrics_port}:/stackable/jmx/{role}.yaml", )]; - if let Some(Some(memory_limit)) = - resources.limits.as_ref().map(|limits| limits.get("memory")) + + if hdfs.has_kerberos_enabled() { + jvm_args.push( + "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string(), + ); + } + + if let Some(memory_limit) = resources.and_then(|r| r.limits.as_ref()?.get("memory")) { let memory_limit = MemoryQuantity::try_from(memory_limit).with_context(|_| { @@ -753,10 +958,10 @@ impl ContainerConfig { } /// Container ports for the main containers namenode, datanode and journalnode. - fn container_ports(&self) -> Vec { + fn container_ports(&self, hdfs: &HdfsCluster) -> Vec { match self { - ContainerConfig::Hdfs { role, .. } => role - .ports() + ContainerConfig::Hdfs { role, .. } => hdfs + .ports(role) .into_iter() .map(|(name, value)| ContainerPort { name: Some(name), diff --git a/rust/operator/src/discovery.rs b/rust/operator/src/discovery.rs index 13d973fc..7f732a0f 100644 --- a/rust/operator/src/discovery.rs +++ b/rust/operator/src/discovery.rs @@ -38,27 +38,33 @@ pub fn build_discovery_configmap( ) .add_data( HDFS_SITE_XML, - build_discovery_hdfs_site_xml(hdfs.name_any(), namenode_podrefs), + build_discovery_hdfs_site_xml(hdfs, hdfs.name_any(), namenode_podrefs), ) .add_data( CORE_SITE_XML, - build_discovery_core_site_xml(hdfs.name_any()), + build_discovery_core_site_xml(hdfs, hdfs.name_any()), ) .build() } -fn build_discovery_hdfs_site_xml(logical_name: String, namenode_podrefs: &[HdfsPodRef]) -> String { +fn build_discovery_hdfs_site_xml( + hdfs: &HdfsCluster, + logical_name: String, + namenode_podrefs: &[HdfsPodRef], +) -> String { HdfsSiteConfigBuilder::new(logical_name) .dfs_name_services() .dfs_ha_namenodes(namenode_podrefs) .dfs_namenode_rpc_address_ha(namenode_podrefs) - .dfs_namenode_http_address_ha(namenode_podrefs) + .dfs_namenode_http_address_ha(hdfs, namenode_podrefs) .dfs_client_failover_proxy_provider() + .security_discovery_config(hdfs) .build_as_xml() } -fn build_discovery_core_site_xml(logical_name: String) -> String { +fn build_discovery_core_site_xml(hdfs: &HdfsCluster, logical_name: String) -> String { CoreSiteConfigBuilder::new(logical_name) .fs_default_fs() + .security_discovery_config(hdfs) .build_as_xml() } diff --git a/rust/operator/src/hdfs_controller.rs b/rust/operator/src/hdfs_controller.rs index 54eef2b4..ce99aa51 100644 --- a/rust/operator/src/hdfs_controller.rs +++ b/rust/operator/src/hdfs_controller.rs @@ -4,6 +4,7 @@ use crate::{ container::ContainerConfig, discovery::build_discovery_configmap, event::{build_invalid_replica_message, publish_event}, + kerberos, product_logging::{extend_role_group_config_map, resolve_vector_aggregator_address}, OPERATOR_NAME, }; @@ -54,6 +55,8 @@ const RESOURCE_MANAGER_HDFS_CONTROLLER: &str = "hdfs-operator-hdfs-controller"; const HDFS_CONTROLLER: &str = "hdfs-controller"; const DOCKER_IMAGE_BASE_NAME: &str = "hadoop"; +pub(crate) const KEYSTORE_DIR_NAME: &str = "/stackable/keystore"; + #[derive(Snafu, Debug, EnumDiscriminants)] #[strum_discriminants(derive(IntoStaticStr))] pub enum Error { @@ -97,6 +100,8 @@ pub enum Error { }, #[snafu(display("Object has no name"))] ObjectHasNoName { obj_ref: ObjectRef }, + #[snafu(display("Object has no namespace"))] + ObjectHasNoNamespace { obj_ref: ObjectRef }, #[snafu(display("Cannot build config map for role [{role}] and role group [{role_group}]"))] BuildRoleGroupConfigMap { source: stackable_operator::error::Error, @@ -150,6 +155,10 @@ pub enum Error { BuildRbacResources { source: stackable_operator::error::Error, }, + #[snafu(display( + "kerberos not supported for HDFS versions < 3.3.x. Please use at least version 3.3.x" + ))] + KerberosNotSupported {}, } impl ReconcilerError for Error { @@ -170,6 +179,9 @@ pub async fn reconcile_hdfs(hdfs: Arc, ctx: Arc) -> HdfsOperat let client = &ctx.client; let resolved_product_image = hdfs.spec.image.resolve(DOCKER_IMAGE_BASE_NAME); + if hdfs.has_kerberos_enabled() { + kerberos::check_if_supported(&resolved_product_image)?; + } let vector_aggregator_address = resolve_vector_aggregator_address(&hdfs, client) .await @@ -369,7 +381,7 @@ fn rolegroup_service( type_: Some("ClusterIP".to_string()), cluster_ip: Some("None".to_string()), ports: Some( - role.ports() + hdfs.ports(role) .into_iter() .map(|(name, value)| ServicePort { name: Some(name), @@ -399,7 +411,6 @@ fn rolegroup_config_map( vector_aggregator_address: Option<&str>, ) -> HdfsOperatorResult { tracing::info!("Setting up ConfigMap for {:?}", rolegroup_ref); - let hdfs_name = hdfs .metadata .name @@ -407,17 +418,25 @@ fn rolegroup_config_map( .with_context(|| ObjectHasNoNameSnafu { obj_ref: ObjectRef::from_obj(hdfs), })?; + let hdfs_namespace = hdfs + .namespace() + .with_context(|| ObjectHasNoNamespaceSnafu { + obj_ref: ObjectRef::from_obj(hdfs), + })?; let mut hdfs_site_xml = String::new(); let mut core_site_xml = String::new(); + let mut hadoop_policy_xml = String::new(); + let mut ssl_server_xml = String::new(); + let mut ssl_client_xml = String::new(); for (property_name_kind, config) in rolegroup_config { match property_name_kind { PropertyNameKind::File(file_name) if file_name == HDFS_SITE_XML => { + // IMPORTANT: these folders must be under the volume mount point, otherwise they will not + // be formatted by the namenode, or used by the other services. + // See also: https://github.com/apache-spark-on-k8s/kubernetes-HDFS/commit/aef9586ecc8551ca0f0a468c3b917d8c38f494a0 hdfs_site_xml = HdfsSiteConfigBuilder::new(hdfs_name.to_string()) - // IMPORTANT: these folders must be under the volume mount point, otherwise they will not - // be formatted by the namenode, or used by the other services. - // See also: https://github.com/apache-spark-on-k8s/kubernetes-HDFS/commit/aef9586ecc8551ca0f0a468c3b917d8c38f494a0 .dfs_namenode_name_dir() .dfs_datanode_data_dir(merged_config.data_node_resources().map(|r| r.storage)) .dfs_journalnode_edits_dir() @@ -434,8 +453,9 @@ fn rolegroup_config_map( .dfs_namenode_shared_edits_dir(journalnode_podrefs) .dfs_namenode_name_dir_ha(namenode_podrefs) .dfs_namenode_rpc_address_ha(namenode_podrefs) - .dfs_namenode_http_address_ha(namenode_podrefs) + .dfs_namenode_http_address_ha(hdfs, namenode_podrefs) .dfs_client_failover_proxy_provider() + .security_config(hdfs) .add("dfs.ha.fencing.methods", "shell(/bin/true)") .add("dfs.ha.nn.not-become-active-in-safemode", "true") .add("dfs.ha.automatic-failover.enabled", "true") @@ -448,10 +468,66 @@ fn rolegroup_config_map( core_site_xml = CoreSiteConfigBuilder::new(hdfs_name.to_string()) .fs_default_fs() .ha_zookeeper_quorum() + .security_config(hdfs, hdfs_name, &hdfs_namespace) // the extend with config must come last in order to have overrides working!!! .extend(config) .build_as_xml(); } + PropertyNameKind::File(file_name) if file_name == HADOOP_POLICY_XML => { + // We don't add any settings here, the main purpose is to have a configOverride for users. + let mut config_opts: BTreeMap> = BTreeMap::new(); + config_opts.extend(config.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + hadoop_policy_xml = + stackable_operator::product_config::writer::to_hadoop_xml(config_opts.iter()); + } + PropertyNameKind::File(file_name) if file_name == SSL_SERVER_XML => { + let mut config_opts = BTreeMap::new(); + if hdfs.has_https_enabled() { + config_opts.extend([ + ( + "ssl.server.truststore.location".to_string(), + Some(format!("{KEYSTORE_DIR_NAME}/truststore.p12")), + ), + ( + "ssl.server.keystore.location".to_string(), + Some(format!("{KEYSTORE_DIR_NAME}/keystore.p12")), + ), + ( + "ssl.server.keystore.password".to_string(), + Some("changeit".to_string()), + ), + ( + "ssl.server.keystore.type".to_string(), + Some("pkcs12".to_string()), + ), + ]); + } + config_opts.extend(config.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + ssl_server_xml = + stackable_operator::product_config::writer::to_hadoop_xml(config_opts.iter()); + } + PropertyNameKind::File(file_name) if file_name == SSL_CLIENT_XML => { + let mut config_opts = BTreeMap::new(); + if hdfs.has_https_enabled() { + config_opts.extend([ + ( + "ssl.client.truststore.location".to_string(), + Some(format!("{KEYSTORE_DIR_NAME}/truststore.p12")), + ), + ( + "ssl.client.truststore.password".to_string(), + Some("changeit".to_string()), + ), + ( + "ssl.client.truststore.type".to_string(), + Some("pkcs12".to_string()), + ), + ]); + } + config_opts.extend(config.iter().map(|(k, v)| (k.clone(), Some(v.clone())))); + ssl_client_xml = + stackable_operator::product_config::writer::to_hadoop_xml(config_opts.iter()); + } _ => {} } } @@ -477,7 +553,10 @@ fn rolegroup_config_map( .build(), ) .add_data(CORE_SITE_XML.to_string(), core_site_xml) - .add_data(HDFS_SITE_XML.to_string(), hdfs_site_xml); + .add_data(HDFS_SITE_XML.to_string(), hdfs_site_xml) + .add_data(HADOOP_POLICY_XML.to_string(), hadoop_policy_xml) + .add_data(SSL_SERVER_XML, ssl_server_xml) + .add_data(SSL_CLIENT_XML, ssl_client_xml); extend_role_group_config_map( rolegroup_ref, @@ -530,6 +609,7 @@ fn rolegroup_statefulset( // Adds all containers and volumes to the pod builder ContainerConfig::add_containers_and_volumes( &mut pb, + hdfs, role, resolved_product_image, merged_config, diff --git a/rust/operator/src/kerberos.rs b/rust/operator/src/kerberos.rs new file mode 100644 index 00000000..fbf11bc2 --- /dev/null +++ b/rust/operator/src/kerberos.rs @@ -0,0 +1,133 @@ +use stackable_hdfs_crd::{ + constants::{SSL_CLIENT_XML, SSL_SERVER_XML}, + HdfsCluster, +}; +use stackable_operator::commons::product_image_selection::ResolvedProductImage; + +use crate::{ + config::{CoreSiteConfigBuilder, HdfsSiteConfigBuilder}, + hdfs_controller::Error, +}; + +pub fn check_if_supported(resolved_product_image: &ResolvedProductImage) -> Result<(), Error> { + // We only support Kerberos for HDFS >= 3.3.x + // With HDFS 3.2.2 we got weird errors, which *might* be caused by DNS lookup issues + // The Stacktrace is documented in rust/operator/src/kerberos_hdfs_3.2_stacktrace.txt + + if resolved_product_image.product_version.starts_with("3.2.") { + Err(Error::KerberosNotSupported {}) + } else { + Ok(()) + } +} + +impl HdfsSiteConfigBuilder { + pub fn security_config(&mut self, hdfs: &HdfsCluster) -> &mut Self { + if hdfs.has_kerberos_enabled() { + self.add("dfs.block.access.token.enable", "true") + .add("dfs.http.policy", "HTTPS_ONLY") + .add("hadoop.kerberos.keytab.login.autorenewal.enabled", "true") + .add("dfs.https.server.keystore.resource", SSL_SERVER_XML) + .add("dfs.https.client.keystore.resource", SSL_CLIENT_XML); + self.add_wire_encryption_settings(); + } + self + } + + pub fn security_discovery_config(&mut self, hdfs: &HdfsCluster) -> &mut Self { + if hdfs.has_kerberos_enabled() { + // We want e.g. hbase to automatically renew the Kerberos tickets. + // This shouldn't harm any other consumers. + self.add("hadoop.kerberos.keytab.login.autorenewal.enabled", "true"); + self.add_wire_encryption_settings(); + } + self + } + + fn add_wire_encryption_settings(&mut self) -> &mut Self { + self.add("dfs.data.transfer.protection", "privacy"); + self.add("dfs.encrypt.data.transfer", "true"); + self + } +} + +impl CoreSiteConfigBuilder { + pub fn security_config( + &mut self, + hdfs: &HdfsCluster, + hdfs_name: &str, + hdfs_namespace: &str, + ) -> &mut Self { + if hdfs.authentication_config().is_some() { + // For a long time we tried using `_HOST` in principals, e.g. `jn/_HOST@REALM.COM`. + // Turns out there are a lot of code paths that check the principal of the requester using a reverse lookup of the incoming IP address + // and getting a different hostname than the principal has. + // What ultimately killed this approach was + // + // 2023-05-30 09:23:01,745 ERROR namenode.EditLogInputStream (EditLogFileInputStream.java:nextOpImpl(220)) - caught exception initializing https://hdfs-journalnode-default-1.hdfs-journalnode-default.kuttl-test-fine-rat.svc.cluster.local:8481/getJournal?jid=hdfs&segmentTxId=1&storageInfo=-65%3A595659877%3A1685437352616%3ACID-90c52400-5b07-49bf-bdbe-3469bbdc5ebb&inProgressOk=true + // org.apache.hadoop.hdfs.server.common.HttpGetFailedException: Fetch of https://hdfs-journalnode-default-1.hdfs-journalnode-default.kuttl-test-fine-rat.svc.cluster.local:8481/getJournal?jid=hdfs&segmentTxId=1&storageInfo=-65%3A595659877%3A1685437352616%3ACID-90c52400-5b07-49bf-bdbe-3469bbdc5ebb&inProgressOk=true failed with status code 403 + // Response message: + // Only Namenode and another JournalNode may access this servlet + // + // After we have switched to using the following principals everything worked without problems + + let principal_host_part = + format!("{hdfs_name}.{hdfs_namespace}.svc.cluster.local@${{env.KERBEROS_REALM}}"); + self.add("hadoop.security.authentication", "kerberos") + .add("hadoop.registry.kerberos.realm", "${env.KERBEROS_REALM}") + .add( + "dfs.journalnode.kerberos.principal", + format!("jn/{principal_host_part}"), + ) + .add( + "dfs.journalnode.kerberos.internal.spnego.principal", + format!("jn/{principal_host_part}"), + ) + .add( + "dfs.namenode.kerberos.principal", + format!("nn/{principal_host_part}"), + ) + .add( + "dfs.datanode.kerberos.principal", + format!("dn/{principal_host_part}"), + ) + .add( + "dfs.web.authentication.kerberos.principal", + format!("HTTP/{principal_host_part}"), + ) + .add("dfs.journalnode.keytab.file", "/stackable/kerberos/keytab") + .add("dfs.namenode.keytab.file", "/stackable/kerberos/keytab") + .add("dfs.datanode.keytab.file", "/stackable/kerberos/keytab") + .add( + "dfs.journalnode.kerberos.principal.pattern", + format!("jn/{principal_host_part}"), + ) + .add( + "dfs.namenode.kerberos.principal.pattern", + format!("nn/{principal_host_part}"), + ) + // Otherwise we fail with `java.io.IOException: No groups found for user nn` + // Default value is `dr.who=`, so we include that here + .add( + "hadoop.user.group.static.mapping.overrides", + "dr.who=;nn=;nm=;jn=;", + ); + + self.add_wire_encryption_settings(); + } + self + } + + pub fn security_discovery_config(&mut self, hdfs: &HdfsCluster) -> &mut Self { + if hdfs.has_kerberos_enabled() { + self.add("hadoop.security.authentication", "kerberos"); + self.add_wire_encryption_settings(); + } + self + } + + fn add_wire_encryption_settings(&mut self) -> &mut Self { + self.add("hadoop.rpc.protection", "privacy"); + self + } +} diff --git a/rust/operator/src/kerberos_hdfs_3.2_stacktrace.txt b/rust/operator/src/kerberos_hdfs_3.2_stacktrace.txt new file mode 100644 index 00000000..0d583dac --- /dev/null +++ b/rust/operator/src/kerberos_hdfs_3.2_stacktrace.txt @@ -0,0 +1,75 @@ +2023-05-31 12:34:18,319 ERROR namenode.EditLogInputStream (EditLogFileInputStream.java:nextOpImpl(220)) - caught exception initializing https://hdfs-journalnode-default-2.hdfs-journalnode-default.kuttl-test-nice-eft.svc.cluster.local:8481/getJournal?jid=hdfs&segmentTxId=1&storageInfo=-65%3A1740831343%3A1685535647411%3ACID-5bb822a0-549e-41ce-9997-ee657b6fc23f&inProgressOk=true +java.io.IOException: org.apache.hadoop.security.authentication.client.AuthenticationException: Error while authenticating with endpoint: https://hdfs-journalnode-default-2.hdfs-journalnode-default.kuttl-test-nice-eft.svc.cluster.local:8481/getJournal?jid=hdfs&segmentTxId=1&storageInfo=-65%3A1740831343%3A1685535647411%3ACID-5bb822a0-549e-41ce-9997-ee657b6fc23f&inProgressOk=true + at org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream$URLLog$1.run(EditLogFileInputStream.java:482) + at org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream$URLLog$1.run(EditLogFileInputStream.java:474) + at java.base/java.security.AccessController.doPrivileged(Native Method) + at java.base/javax.security.auth.Subject.doAs(Subject.java:423) + at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1762) + at org.apache.hadoop.security.SecurityUtil.doAsUser(SecurityUtil.java:535) + at org.apache.hadoop.security.SecurityUtil.doAsCurrentUser(SecurityUtil.java:529) + at org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream$URLLog.getInputStream(EditLogFileInputStream.java:473) + at org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream.init(EditLogFileInputStream.java:157) + at org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream.nextOpImpl(EditLogFileInputStream.java:218) + at org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream.nextOp(EditLogFileInputStream.java:276) + at org.apache.hadoop.hdfs.server.namenode.EditLogInputStream.readOp(EditLogInputStream.java:85) + at org.apache.hadoop.hdfs.server.namenode.EditLogInputStream.skipUntil(EditLogInputStream.java:151) + at org.apache.hadoop.hdfs.server.namenode.RedundantEditLogInputStream.nextOp(RedundantEditLogInputStream.java:190) + at org.apache.hadoop.hdfs.server.namenode.EditLogInputStream.readOp(EditLogInputStream.java:85) + at org.apache.hadoop.hdfs.server.namenode.EditLogInputStream.skipUntil(EditLogInputStream.java:151) + at org.apache.hadoop.hdfs.server.namenode.RedundantEditLogInputStream.nextOp(RedundantEditLogInputStream.java:190) + at org.apache.hadoop.hdfs.server.namenode.EditLogInputStream.readOp(EditLogInputStream.java:85) + at org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.loadEditRecords(FSEditLogLoader.java:243) + at org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.loadFSEdits(FSEditLogLoader.java:182) + at org.apache.hadoop.hdfs.server.namenode.FSImage.loadEdits(FSImage.java:914) + at org.apache.hadoop.hdfs.server.namenode.FSImage.loadFSImage(FSImage.java:761) + at org.apache.hadoop.hdfs.server.namenode.FSImage.recoverTransitionRead(FSImage.java:338) + at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.loadFSImage(FSNamesystem.java:1135) + at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.loadFromDisk(FSNamesystem.java:750) + at org.apache.hadoop.hdfs.server.namenode.NameNode.loadNamesystem(NameNode.java:658) + at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:734) + at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:977) + at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:950) + at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1716) + at org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:1783) +Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: Error while authenticating with endpoint: https://hdfs-journalnode-default-2.hdfs-journalnode-default.kuttl-test-nice-eft.svc.cluster.local:8481/getJournal?jid=hdfs&segmentTxId=1&storageInfo=-65%3A1740831343%3A1685535647411%3ACID-5bb822a0-549e-41ce-9997-ee657b6fc23f&inProgressOk=true + at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) + at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) + at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490) + at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.wrapExceptionWithMessage(KerberosAuthenticator.java:232) + at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:219) + at org.apache.hadoop.security.authentication.client.AuthenticatedURL.openConnection(AuthenticatedURL.java:348) + at org.apache.hadoop.hdfs.web.URLConnectionFactory.openConnection(URLConnectionFactory.java:186) + at org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream$URLLog$1.run(EditLogFileInputStream.java:480) + ... 30 more +Caused by: org.apache.hadoop.security.authentication.client.AuthenticationException: GSSException: No valid credentials provided (Mechanism level: Server not found in Kerberos database (7) - LOOKING_UP_SERVER) + at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:360) + at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.authenticate(KerberosAuthenticator.java:204) + ... 33 more +Caused by: GSSException: No valid credentials provided (Mechanism level: Server not found in Kerberos database (7) - LOOKING_UP_SERVER) + at java.security.jgss/sun.security.jgss.krb5.Krb5Context.initSecContext(Krb5Context.java:773) + at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:266) + at java.security.jgss/sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:196) + at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:336) + at org.apache.hadoop.security.authentication.client.KerberosAuthenticator$1.run(KerberosAuthenticator.java:310) + at java.base/java.security.AccessController.doPrivileged(Native Method) + at java.base/javax.security.auth.Subject.doAs(Subject.java:423) + at org.apache.hadoop.security.authentication.client.KerberosAuthenticator.doSpnegoSequence(KerberosAuthenticator.java:310) + ... 34 more +Caused by: KrbException: Server not found in Kerberos database (7) - LOOKING_UP_SERVER + at java.security.jgss/sun.security.krb5.KrbTgsRep.(KrbTgsRep.java:73) + at java.security.jgss/sun.security.krb5.KrbTgsReq.getReply(KrbTgsReq.java:226) + at java.security.jgss/sun.security.krb5.KrbTgsReq.sendAndGetCreds(KrbTgsReq.java:237) + at java.security.jgss/sun.security.krb5.internal.CredentialsUtil.serviceCredsSingle(CredentialsUtil.java:477) + at java.security.jgss/sun.security.krb5.internal.CredentialsUtil.serviceCreds(CredentialsUtil.java:340) + at java.security.jgss/sun.security.krb5.internal.CredentialsUtil.serviceCreds(CredentialsUtil.java:314) + at java.security.jgss/sun.security.krb5.internal.CredentialsUtil.acquireServiceCreds(CredentialsUtil.java:169) + at java.security.jgss/sun.security.krb5.Credentials.acquireServiceCreds(Credentials.java:490) + at java.security.jgss/sun.security.jgss.krb5.Krb5Context.initSecContext(Krb5Context.java:697) + ... 41 more +Caused by: KrbException: Identifier doesn't match expected value (906) + at java.security.jgss/sun.security.krb5.internal.KDCRep.init(KDCRep.java:140) + at java.security.jgss/sun.security.krb5.internal.TGSRep.init(TGSRep.java:65) + at java.security.jgss/sun.security.krb5.internal.TGSRep.(TGSRep.java:60) + at java.security.jgss/sun.security.krb5.KrbTgsRep.(KrbTgsRep.java:55) + ... 49 more diff --git a/rust/operator/src/lib.rs b/rust/operator/src/lib.rs index d40191b4..52e06d0c 100644 --- a/rust/operator/src/lib.rs +++ b/rust/operator/src/lib.rs @@ -3,6 +3,7 @@ mod container; mod discovery; mod event; mod hdfs_controller; +mod kerberos; mod pod_svc_controller; mod product_logging; diff --git a/tests/templates/kuttl/kerberos/00-assert.yaml.j2 b/tests/templates/kuttl/kerberos/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/kerberos/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/kerberos/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/kerberos/01-assert.yaml.j2 b/tests/templates/kuttl/kerberos/01-assert.yaml.j2 new file mode 100644 index 00000000..d34c1c63 --- /dev/null +++ b/tests/templates/kuttl/kerberos/01-assert.yaml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 300 +{% if test_scenario['values']['kerberos-backend'] == 'mit' %} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +status: + readyReplicas: 1 + replicas: 1 +{% endif %} diff --git a/tests/templates/kuttl/kerberos/01-install-krb5-kdc.yaml.j2 b/tests/templates/kuttl/kerberos/01-install-krb5-kdc.yaml.j2 new file mode 100644 index 00000000..d8070ec6 --- /dev/null +++ b/tests/templates/kuttl/kerberos/01-install-krb5-kdc.yaml.j2 @@ -0,0 +1,135 @@ +{% if test_scenario['values']['kerberos-backend'] == 'mit' %} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: krb5-kdc +spec: + selector: + matchLabels: + app: krb5-kdc + template: + metadata: + labels: + app: krb5-kdc + spec: + initContainers: + - name: init + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + args: + - sh + - -euo + - pipefail + - -c + - | + test -e /var/kerberos/krb5kdc/principal || kdb5_util create -s -P asdf + kadmin.local get_principal -terse root/admin || kadmin.local add_principal -pw asdf root/admin + # stackable-secret-operator principal must match the keytab specified in the SecretClass + kadmin.local get_principal -terse stackable-secret-operator || kadmin.local add_principal -e aes256-cts-hmac-sha384-192:normal -pw asdf stackable-secret-operator + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + containers: + - name: kdc + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + args: + - krb5kdc + - -n + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + - name: kadmind + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + args: + - kadmind + - -nofork + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + - mountPath: /var/kerberos/krb5kdc + name: data + - name: client + image: docker.stackable.tech/stackable/krb5:1.18.2-stackable0.0.0-dev + tty: true + stdin: true + env: + - name: KRB5_CONFIG + value: /stackable/config/krb5.conf + volumeMounts: + - mountPath: /stackable/config + name: config + volumes: + - name: config + configMap: + name: krb5-kdc + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: krb5-kdc +spec: + selector: + app: krb5-kdc + ports: + - name: kadmin + port: 749 + - name: kdc + port: 88 + - name: kdc-udp + port: 88 + protocol: UDP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: krb5-kdc +data: + krb5.conf: | + [logging] + default = STDERR + kdc = STDERR + admin_server = STDERR + # default = FILE:/var/log/krb5libs.log + # kdc = FILE:/var/log/krb5kdc.log + # admin_server = FILE:/vaggr/log/kadmind.log + [libdefaults] + dns_lookup_realm = false + ticket_lifetime = 24h + renew_lifetime = 7d + forwardable = true + rdns = false + default_realm = {{ test_scenario['values']['kerberos-realm'] }} + spake_preauth_groups = edwards25519 + [realms] + {{ test_scenario['values']['kerberos-realm'] }} = { + acl_file = /stackable/config/kadm5.acl + disable_encrypted_timestamp = false + } + [domain_realm] + .cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + cluster.local = {{ test_scenario['values']['kerberos-realm'] }} + kadm5.acl: | + root/admin *e + stackable-secret-operator *e +{% endif %} diff --git a/tests/templates/kuttl/kerberos/02-create-kerberos-secretclass.yaml.j2 b/tests/templates/kuttl/kerberos/02-create-kerberos-secretclass.yaml.j2 new file mode 100644 index 00000000..04ae9a63 --- /dev/null +++ b/tests/templates/kuttl/kerberos/02-create-kerberos-secretclass.yaml.j2 @@ -0,0 +1,72 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 }} + resources: + cpu: + min: '300m' + max: '600m' + memory: + limit: '512Mi' + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/kerberos/11-assert.yaml b/tests/templates/kuttl/kerberos/11-assert.yaml new file mode 100644 index 00000000..47260144 --- /dev/null +++ b/tests/templates/kuttl/kerberos/11-assert.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-namenode-default +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-journalnode-default +status: + readyReplicas: 3 + replicas: 3 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: hdfs-datanode-default +status: + readyReplicas: 2 + replicas: 2 diff --git a/tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2 b/tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2 new file mode 100644 index 00000000..99faadb6 --- /dev/null +++ b/tests/templates/kuttl/kerberos/11-install-hdfs.yaml.j2 @@ -0,0 +1,70 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 0 }} + resources: + cpu: + max: '1' + min: '300m' + memory: + limit: '512Mi' + configOverrides: &configOverrides + core-site.xml: + hadoop.user.group.static.mapping.overrides: "dr.who=;nn=;nm=;jn=;testuser=supergroup;" + roleGroups: + default: + replicas: 2 + dataNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + resources: + cpu: + max: '1' + min: '300m' + memory: + limit: '512Mi' + configOverrides: *configOverrides + roleGroups: + default: + replicas: 2 + journalNodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + resources: + cpu: + max: '1' + min: '300m' + memory: + limit: '512Mi' + configOverrides: *configOverrides + roleGroups: + default: + replicas: 3 + EOF diff --git a/tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2 b/tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2 new file mode 100644 index 00000000..1a94ea8b --- /dev/null +++ b/tests/templates/kuttl/kerberos/20-access-hdfs.yaml.j2 @@ -0,0 +1,68 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 20 minutes) +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 3600 +commands: + # First, let's delete the first pod of every HDFS service + # Should trigger failover of the namenode to 1 + - script: kubectl -n $NAMESPACE delete pod --force hdfs-journalnode-default-0 hdfs-namenode-default-0 hdfs-datanode-default-0 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs hdfs --timeout 10m + timeout: 600 + + # Also delete the last pod of every HDFS service + # Should trigger failover of the namenode back to 0 + - script: kubectl -n $NAMESPACE delete pod --force hdfs-journalnode-default-2 hdfs-namenode-default-1 hdfs-datanode-default-1 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs hdfs --timeout 10m + timeout: 600 + + # Also delete the Zookeeper + - script: kubectl -n $NAMESPACE delete pod --force hdfs-zk-server-default-0 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available zookeepercluster hdfs-zk --timeout 10m + timeout: 600 + + # And now everything +{% for n in range(3) %} + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hdfs + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=zookeeper + timeout: 600 + - script: sleep 10 + # Delete just after they have started up again, just to make things worse + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hdfs + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=zookeeper + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available zookeepercluster hdfs-zk --timeout 10m + timeout: 600 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs hdfs --timeout 10m + timeout: 600 +{% endfor %} diff --git a/tests/templates/kuttl/kerberos/22-assert.yaml b/tests/templates/kuttl/kerberos/22-assert.yaml new file mode 100644 index 00000000..374d2e79 --- /dev/null +++ b/tests/templates/kuttl/kerberos/22-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: check-hdfs +status: + succeeded: 1 diff --git a/tests/templates/kuttl/kerberos/22-check-file.yaml.j2 b/tests/templates/kuttl/kerberos/22-check-file.yaml.j2 new file mode 100644 index 00000000..6477d384 --- /dev/null +++ b/tests/templates/kuttl/kerberos/22-check-file.yaml.j2 @@ -0,0 +1,65 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl apply -n $NAMESPACE -f - < 20 minutes) +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 3600 +commands: + # First, let's delete the first pod of every HDFS service + # Should trigger failover of the namenode to 1 + - script: kubectl -n $NAMESPACE delete pod --force hdfs-journalnode-default-0 hdfs-namenode-default-0 hdfs-datanode-default-0 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs hdfs --timeout 10m + timeout: 600 + + # Also delete the last pod of every HDFS service + # Should trigger failover of the namenode back to 0 + - script: kubectl -n $NAMESPACE delete pod --force hdfs-namenode-default-1 hdfs-datanode-default-1 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs hdfs --timeout 10m + timeout: 600 + + # Also delete the Zookeeper + - script: kubectl -n $NAMESPACE delete pod --force hdfs-zk-server-default-0 + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available zookeepercluster hdfs-zk --timeout 10m + timeout: 600 + + # And now everything +{% for n in range(3) %} + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hdfs + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=zookeeper + timeout: 600 + - script: sleep 10 + # Delete just after they have started up again, just to make things worse + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=hdfs + timeout: 600 + - script: kubectl -n $NAMESPACE delete pod --force -l app.kubernetes.io/name=zookeeper + timeout: 600 + - script: sleep 10 + - script: kubectl -n $NAMESPACE wait --for=condition=Available zookeepercluster hdfs-zk --timeout 10m + timeout: 600 + - script: kubectl -n $NAMESPACE wait --for=condition=Available hdfs hdfs --timeout 10m + timeout: 600 +{% endfor %} +{% endif %} diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index b8ed0428..5a1483d1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -3,7 +3,6 @@ dimensions: - name: hadoop values: - 3.2.2-stackable0.0.0-dev - - 3.3.3-stackable0.0.0-dev - 3.3.4-stackable0.0.0-dev - name: hadoop-latest values: @@ -27,14 +26,31 @@ dimensions: values: - "cluster-internal" - "external-unstable" + - name: kerberos-realm + values: + - "CLUSTER.LOCAL" + - "PROD.MYCORP" + - name: kerberos-backend + values: + - mit + # Requires manual setup, see create-kerberos-secretclass.yaml + # This will *not* respect the kerberos-realm test attribute, but instead use a hard-coded realm + # - activeDirectory tests: - name: smoke dimensions: - hadoop - zookeeper + - zookeeper-latest # Needed for smoke test to detect if zk versions is the latest we support - number-of-datanodes - datanode-pvcs - listener-class + - name: kerberos + dimensions: + - hadoop-latest # We only support Kerberos for HDFS >= 3.3.x. See rust/operator/src/kerberos.rs for details + - zookeeper-latest + - kerberos-realm + - kerberos-backend - name: orphaned-resources dimensions: - hadoop-latest