diff --git a/modules/ROOT/pages/getting-started.adoc b/modules/ROOT/pages/getting-started.adoc index 7b2855295..13de4a25c 100644 --- a/modules/ROOT/pages/getting-started.adoc +++ b/modules/ROOT/pages/getting-started.adoc @@ -32,11 +32,11 @@ Install the Stackable command line utility xref:management:stackablectl:index.ad The Stackable operators are components that translate the service definitions deployed via Kubernetes into deploy services on the worker nodes. These can be installed on any node that has access to the Kubernetes control plane. In this example we will install them on the controller node. -Stackable operators can be installed using `stackablectl`. Run the following commands to install ZooKeeper, Kafka and NiFi from the Stackable 24.11 release. +Stackable operators can be installed using `stackablectl`. Run the following commands to install ZooKeeper, Kafka and NiFi from the Stackable 25.3 release. [source,bash] ---- -stackablectl release install -i commons -i secret -i listener -i zookeeper -i kafka -i nifi 24.11 +stackablectl release install -i commons -i secret -i listener -i zookeeper -i kafka -i nifi 25.3 ---- .Using Helm instead @@ -50,12 +50,12 @@ Install the operators: [source,bash] ---- -helm install zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version=24.11.1 -helm install kafka-operator oci://oci.stackable.tech/sdp-charts/kafka-operator --version=24.11.1 -helm install secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version=24.11.1 -helm install listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version=24.11.1 -helm install commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version=24.11.1 -helm install nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version=24.11.1 +helm install zookeeper-operator oci://oci.stackable.tech/sdp-charts/zookeeper-operator --version=25.3.0 +helm install kafka-operator oci://oci.stackable.tech/sdp-charts/kafka-operator --version=25.3.0 +helm install secret-operator oci://oci.stackable.tech/sdp-charts/secret-operator --version=25.3.0 +helm install listener-operator oci://oci.stackable.tech/sdp-charts/listener-operator --version=25.3.0 +helm install commons-operator oci://oci.stackable.tech/sdp-charts/commons-operator --version=25.3.0 +helm install nifi-operator oci://oci.stackable.tech/sdp-charts/nifi-operator --version=25.3.0 ---- ==== @@ -64,12 +64,12 @@ You can check which operators are installed using `stackablectl operator install [source,console] ---- OPERATOR VERSION NAMESPACE STATUS LAST UPDATED -commons 24.11.1 default deployed 2024-11-30 17:58:32.916032854 +0100 CET -kafka 24.11.1 default deployed 2024-11-30 17:58:55.036115353 +0100 CET -listener 24.11.1 default deployed 2024-11-30 17:59:18.136775259 +0100 CET -nifi 24.11.1 default deployed 2024-11-30 17:59:51.927081648 +0100 CET -secret 24.11.1 default deployed 2024-11-30 18:00:05.060241771 +0100 CET -zookeeper 24.11.1 default deployed 2024-11-30 18:00:08.425686918 +0100 CET +commons 25.3.0 default deployed 2024-11-30 17:58:32.916032854 +0100 CET +kafka 25.3.0 default deployed 2024-11-30 17:58:55.036115353 +0100 CET +listener 25.3.0 default deployed 2024-11-30 17:59:18.136775259 +0100 CET +nifi 25.3.0 default deployed 2024-11-30 17:59:51.927081648 +0100 CET +secret 25.3.0 default deployed 2024-11-30 18:00:05.060241771 +0100 CET +zookeeper 25.3.0 default deployed 2024-11-30 18:00:08.425686918 +0100 CET ---- == Deploying Stackable Services @@ -90,7 +90,7 @@ metadata: name: simple-zk spec: image: - productVersion: 3.9.2 + productVersion: 3.9.3 clusterConfig: tls: serverSecretClass: null @@ -278,7 +278,7 @@ To get the IP address we need to connect to (in this case `172.18.0.2`), run: ---- $ kubectl get nodes -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME -quickstart-control-plane Ready control-plane 4d18h v1.30.0 172.18.0.2 Debian GNU/Linux 12 (bookworm) 6.11.3 containerd://1.7.15 +quickstart-control-plane Ready control-plane 4d18h v1.32.0 172.18.0.2 Debian GNU/Linux 12 (bookworm) 6.13.2 containerd://1.7.24 ---- diff --git a/modules/ROOT/pages/release-notes.adoc b/modules/ROOT/pages/release-notes.adoc index ea0b0ea88..fc4bd73b9 100644 --- a/modules/ROOT/pages/release-notes.adoc +++ b/modules/ROOT/pages/release-notes.adoc @@ -6,13 +6,25 @@ The Stackable platform consists of multiple operators that work together. Periodically a platform release is made, including all components of the platform at a specific version. +// WARNING: Please keep the empty newlines, otherwise headings are broken. +include::partial$release-notes/release-25.3.adoc[] + include::partial$release-notes/release-24.11.adoc[] + include::partial$release-notes/release-24.7.adoc[] + include::partial$release-notes/release-24.3.adoc[] + include::partial$release-notes/release-23.11.adoc[] + include::partial$release-notes/release-23.7.adoc[] + include::partial$release-notes/release-23.4.adoc[] + include::partial$release-notes/release-23.1.adoc[] + include::partial$release-notes/release-22.11.adoc[] + include::partial$release-notes/release-22.9.adoc[] + include::partial$release-notes/release-22.6.adoc[] diff --git a/modules/ROOT/partials/release-notes/release-22.11.adoc b/modules/ROOT/partials/release-notes/release-22.11.adoc index c229407fc..a4c23f309 100644 --- a/modules/ROOT/partials/release-notes/release-22.11.adoc +++ b/modules/ROOT/partials/release-notes/release-22.11.adoc @@ -9,7 +9,7 @@ This is the third release of the Stackable Data Platform, which this time focuse The following new major platform features were added: -CPU and memory limits configurable:: +===== CPU and memory limits configurable The operators now https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/[request] resources from Kubernetes for the products and required CPU and memory can now also be configured for all products. If your product instances are less performant after the update, the new defaults might be set too low and we recommend to xref:kafka:usage-guide/storage-resources.adoc[set custom requests] for your cluster. @@ -24,7 +24,7 @@ If your product instances are less performant after the update, the new defaults * https://github.com/stackabletech/airflow-operator/pull/167[Apache Airflow] * https://github.com/stackabletech/superset-operator/pull/273[Apache Superset] -Orphaned Resources:: +===== Orphaned Resources The operators now properly clean up after scaling down products. This means for example deleting StatefulSets that were left over after scaling down. @@ -37,7 +37,7 @@ This means for example deleting StatefulSets that were left over after scaling d * https://github.com/stackabletech/trino-operator/pull/310[Trino] * https://github.com/stackabletech/airflow-operator/pull/174[Apache Airflow] -New Versions:: +===== New Versions New product versions are supported. @@ -47,7 +47,7 @@ New product versions are supported. * https://github.com/stackabletech/druid-operator/pull/317[Apache Druid 24.0.0] * https://github.com/stackabletech/airflow-operator/pull/179[Apache Airflow 2.4.1] -Product features:: +===== Product features Additionally there are some individual product features that are noteworthy diff --git a/modules/ROOT/partials/release-notes/release-22.6.adoc b/modules/ROOT/partials/release-notes/release-22.6.adoc index 497af6dc4..ac71fac2d 100644 --- a/modules/ROOT/partials/release-notes/release-22.6.adoc +++ b/modules/ROOT/partials/release-notes/release-22.6.adoc @@ -13,37 +13,37 @@ While we are very proud of this release it is our first one and we'll add new fe ==== Platform features -Easily install production ready data applications:: +===== Easily install production ready data applications Using a familiar declarative approach, users can easily install data applications such as Apache Kafka or Trino across multiple cloud Kubernetes providers or on their own data centers. The installation process is fully automated while also providing the flexibility for the user to tune relevant aspects of each application. -Monitoring:: +===== Monitoring All products have monitoring with prometheus enabled. xref:operators:monitoring.adoc[Learn more] -Service discovery:: +===== Service discovery Products on the Stackable platform use service discovery to easily interconnect with each other. xref:concepts:service_discovery.adoc[Learn more] -Configuration overrides:: +===== Configuration overrides All operators support configuration overrides, these are documented in the specific operator documentation pages. -Common S3 configuration:: +===== Common S3 configuration Many products support connecting to S3 to load and/or store data. There is a common resource for S3 connections and buckets across all operators that can be reused. xref:concepts:s3.adoc[Learn more] -Roles and role groups:: +===== Roles and role groups To support hybrid hardware clusters, the Stackable platform uses the concept of role groups. Services and applications can be configured to maximize hardware efficiency. -Standardized:: +===== Standardized Learn once reuse everywhere. We use the same conventions in all our operators. diff --git a/modules/ROOT/partials/release-notes/release-22.9.adoc b/modules/ROOT/partials/release-notes/release-22.9.adoc index 6e0d6250b..8605f027d 100644 --- a/modules/ROOT/partials/release-notes/release-22.9.adoc +++ b/modules/ROOT/partials/release-notes/release-22.9.adoc @@ -10,7 +10,7 @@ The main features focus on OpenShift support and security. The following new major platform features were added: -OpenShift compatibility:: +===== OpenShift compatibility We have made continued progress towards OpenShift compability, and the following operators can now be previewed on OpenShift. Further improvements are expected in future releases, but no stability or compatibility guarantees are currently made for OpenShift clusters. @@ -20,7 +20,7 @@ Further improvements are expected in future releases, but no stability or compat * https://github.com/stackabletech/hdfs-operator/pull/225[Apache HDFS] * https://github.com/stackabletech/spark-k8s-operator/pull/126[Apache Spark on K8s] -Support for internal and external TLS:: +===== Support for internal and external TLS The following operators support operating the products at a maximal level of transport security by using TLS certificates to secure internal and external communication: @@ -28,7 +28,7 @@ The following operators support operating the products at a maximal level of tra * https://github.com/stackabletech/kafka-operator/pull/442[Apache Kafka] * https://github.com/stackabletech/zookeeper-operator/pull/479[Apache ZooKeeper] -LDAP authentication:: +===== LDAP authentication Use a central LDAP server to manage all of your user identities in a single place. The following operators added support for LDAP authentication: diff --git a/modules/ROOT/partials/release-notes/release-23.1.adoc b/modules/ROOT/partials/release-notes/release-23.1.adoc index ef601bff9..10cc9a96b 100644 --- a/modules/ROOT/partials/release-notes/release-23.1.adoc +++ b/modules/ROOT/partials/release-notes/release-23.1.adoc @@ -20,7 +20,7 @@ The focus in this platform release is on the support of offline (or on-premise) The following new major platform features were added: -Product image selection:: +===== Product image selection Product image selection has been expanded to cover different scenarios: @@ -32,7 +32,7 @@ These options are described in more detail xref:contributor:adr/ADR023-product-i *N.B.* this is a breaking change across all operators as `spec.version` has been replaced by `spec.image`. -Logging Aggregation:: +===== Logging Aggregation Component activity within the platform is logged in a way that makes it difficult to find, persist and consolidate this information. Log configuration is also a challenge. @@ -47,19 +47,19 @@ In this release this has been added to the following components: Support for other products will be added in future releases. -New Versions:: +===== New Versions The following new product version is now supported: * https://github.com/stackabletech/trino-operator/pull/358[Trino 403] -Deprecated Versions:: +===== Deprecated Versions The following product version is no longer supported: * https://github.com/stackabletech/druid-operator/pull/339[Druid 0.22.1] -Product features:: +===== Product features Additionally, there are some individual product features that are noteworthy diff --git a/modules/ROOT/partials/release-notes/release-23.11.adoc b/modules/ROOT/partials/release-notes/release-23.11.adoc index 565091628..ccc559f84 100644 --- a/modules/ROOT/partials/release-notes/release-23.11.adoc +++ b/modules/ROOT/partials/release-notes/release-23.11.adoc @@ -11,45 +11,45 @@ Released 2023-11-30. The following new major platform features were added: -PodDisruptionBudgets:: +===== PodDisruptionBudgets Kubernetes has mechanisms to ensure minimal planned downtime. Our product operators deploy so-called PodDisruptionBudget (PDB) resources alongside the products. For every role that you specify (e.g. HDFS namenodes or Trino workers) a PDB is created. This will determine the extent to which roles for a given application may be inactive at any given time. See xref:concepts:operations/pod_disruptions.adoc[the documentation] for more details. -Graceful shutdown:: +===== Graceful shutdown Graceful shutdown refers to the managed, controlled shutdown of service instances in the manner intended by the software authors. Typically, an instance will receive a signal indicating the intent for the server to shut down, and it will initiate a controlled shutdown. Our operators configure a sensible amount of time Pods are granted to properly shut down without disrupting the availability of the product. See xref:concepts:operations/graceful_shutdown.adoc[the documentation] for more details. -Signed SDP product images:: +===== Signed SDP product images As of this release all Stackable product images are signed (the signing of operator images was delivered in SDP 23.7). Please see this xref:guides:enabling-verification-of-image-signatures.adoc[tutorial] for more information. -Airflow KubernetesExecutor:: +===== Airflow KubernetesExecutor Airflow clusters can now be configured to use Kubernetes executors, whereby pods are spun up for job tasks and terminated when complete, thus offering an alternative way to use resources without the need for job queuing. -Overridable Java security settings:: +===== Overridable Java security settings For JVM-based products (i.e. Druid, HBase, HDFS, Hive, Kafka, NiFi, Spark, Trino and ZooKeeper) it is now possible to provide custom security settings that override the default values. This allows the user to control things such as DNS lookup caches. -Stackable Cockpit:: +===== Stackable Cockpit This release includes a very early preview version of Stackable Cockpit, a browser-based management tool which interacts with the Stackable data platform to display e.g. deployed stacklets and their status. -stackablectl:: +===== stackablectl Our command line tool has been re-worked to use the same backbone as Stackable Cockpit: you can find out about the recent enhancements by visiting the online xref:management:stackablectl:index.adoc[documentation]. -Listener operator:: +===== Listener operator The listener-operator was introduced in release 23.1 and the associated ServiceType field in 23.4. In this release we introduce configurable ListenerClass _presets_ that map to the service types appropriate for different environments. This is discussed in more detail in the xref:listener-operator:listenerclass.adoc[documentation]. -Openshift certification:: +===== Openshift certification All Stackable operators in the 23.11 release have been certified for Openshift versions 4.11-4.13 and can be installed directly from the OperatorHub UI: image:openshift_operatorhub.png[OperatorHub in Openshift portal] -Product features:: +===== Product features Additionally, there are some other individual product features that are noteworthy: @@ -692,7 +692,6 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-ope [source,console] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced -customresourcedefinition.apiextensions.k8s.io "airflowdbs.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced ... diff --git a/modules/ROOT/partials/release-notes/release-23.4.adoc b/modules/ROOT/partials/release-notes/release-23.4.adoc index 35166ff1f..9f70d0a74 100644 --- a/modules/ROOT/partials/release-notes/release-23.4.adoc +++ b/modules/ROOT/partials/release-notes/release-23.4.adoc @@ -29,19 +29,19 @@ It is recommended to install <> instead, as it contains relevant The following new major platform features were added: -Cluster Operation:: +===== Cluster Operation The first part of xref:concepts:operations/cluster_operations.adoc[Cluster operations] was rolled out in every applicable Stackable Operator. This supports pausing the cluster reconciliation and stopping the cluster completely. Pausing reconciliation will not apply any changes to the Kubernetes resources (e.g. when changing the custom resource). Stopping the cluster will set all replicas of StatefulSets, Deployments or DaemonSets to zero and therefore result in the deletion of all Pods belonging to that cluster (not the PVCs). -Status Field:: +===== Status Field Operators of the Stackable Data Platform create, manage and delete Kubernetes resources: in order to easily query the health state of the products - and react accordingly - Stackable Operators use several predefined condition types to capture different aspects of a product's availability. See this xref:contributor:adr/ADR027-status[ADR] for more information. -Default / Custom Affinities:: +===== Default / Custom Affinities In Kubernetes there are different ways to influence how Pods are assigned to Nodes. In some cases it makes sense to co-locate certain services that communicate a lot with each other, such as HBase regionservers with HDFS datanodes. @@ -50,13 +50,13 @@ There may also be additional requirements e.g. placing important services - such This release implements default affinities that should suffice for many scenarios out-of-the box, while also allowing for custom affinity rules at a role and/or role-group level. See this xref:contributor:adr/ADR026-affinities.adoc[ADR] for more information. -Log Aggregation:: +===== Log Aggregation The logging framework (added to the platform in Release 23.1) offers a consistent custom resource configuration and a separate, persisted sink (defaulting to OpenSearch). This has now been rolled out across all products. See this xref:contributor:adr/adr025-logging_architecture[ADR] and this xref:concepts:logging.adoc[concepts page] for more information. -Service Type:: +===== Service Type The Service type can now be specified in all products. This currently differentiates between the internal ClusterIP and the external NodePort and is forward compatible with the xref:listener-operator:listenerclass.adoc[ListenerClass] for the automatic exposure of Services via the Listener Operator. @@ -64,15 +64,15 @@ This change is not backwards compatible with older platform releases. For security reasons, the default is set to the cluster-internal (ClusterIP) ListenerClass. A cluster can be exposed outside of Kubernetes by setting clusterConfig.listenerClass to external-unstable (NodePort) or external-stable (LoadBalancer). -New Versions:: +===== New Versions No new product versions are supported in this platform release. -Deprecated Versions:: +===== Deprecated Versions No product versions have been deprecated in this platform release. -Product features:: +===== Product features Additionally, there are some individual product features that are noteworthy: diff --git a/modules/ROOT/partials/release-notes/release-23.7.adoc b/modules/ROOT/partials/release-notes/release-23.7.adoc index e05a87838..3c699dd2d 100644 --- a/modules/ROOT/partials/release-notes/release-23.7.adoc +++ b/modules/ROOT/partials/release-notes/release-23.7.adoc @@ -11,14 +11,14 @@ Released on 2024-07-25. The following new major platform features were added: -Resource Quotas:: +===== Resource Quotas Explicit resources are now applied to all containers, for both operators and products. This allows running the Stackable Data Platform on Kubernetes clusters with a ResourceQuota or LimitRange set. Where these are not specified directly, defaults will be used. See https://github.com/stackabletech/issues/issues/368[this issue] for more information. -Pod Overrides:: +===== Pod Overrides It is now possible to add custom settings which specify elements of a pod template (Service, StatefulSet etc.) on roles or rolegroups, which the operator then merges with the objects it writes before actually applying them. This provides the user with a possibility for specifying any property that can be set on a regular Kubernetes Pod, but which is not directly exposed via the Stackable custom resource definition. @@ -41,17 +41,17 @@ For example, with HDFS: cpu: 410m ``` -Openshift certification:: +===== Openshift certification OLM bundles - a pre-requisite for the Openshift certification process - have been created for each operator. All 15 SDP operators in release 23.4.1 are now Openshift-certified and deployable directly from within an Openshift cluster. -Signed SDP operator images:: +===== Signed SDP operator images As of this release all Stackable operator images are signed (this feature will be added to product images in a subsequent release). More information about this, including how to verify the image signatures, can be found in this xref:guides:enabling-verification-of-image-signatures.adoc[guide]. -New Versions:: +===== New Versions The following new product versions are now supported: @@ -65,7 +65,7 @@ The following new product versions are now supported: * https://github.com/stackabletech/trino-operator/pull/423[Trino: 414] * https://github.com/stackabletech/zookeeper-operator/pull/689[ZooKeeper: 3.8.1] -Deprecated Versions:: +===== Deprecated Versions The following product versions are deprecated and will be removed in a later release: @@ -82,11 +82,11 @@ The following product versions are deprecated and will be removed in a later rel * Trino: 377, 387, 395, 396, 403 * Zookeeper: 3.5.8, 3.6.3, 3.7.0, 3.8.0 -Removed Versions:: +===== Removed Versions No product versions have been removed. -Product features:: +===== Product features Additionally, there are some individual product features that are noteworthy: diff --git a/modules/ROOT/partials/release-notes/release-24.11.adoc b/modules/ROOT/partials/release-notes/release-24.11.adoc index 030b6ac39..aaa77e922 100644 --- a/modules/ROOT/partials/release-notes/release-24.11.adoc +++ b/modules/ROOT/partials/release-notes/release-24.11.adoc @@ -70,7 +70,6 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-ope [source,console] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced -customresourcedefinition.apiextensions.k8s.io "airflowdbs.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced ... @@ -126,7 +125,6 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-ope [source,console] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced -customresourcedefinition.apiextensions.k8s.io "airflowdbs.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced ... @@ -162,7 +160,7 @@ Also consult the release notes for the <<_24_11_1>> patch release above. ==== New platform features -Authentication:: +===== Authentication In this release we introduced several authentication mechanisms in different products: @@ -170,30 +168,30 @@ In this release we introduced several authentication mechanisms in different pro * Apache Kafka: https://github.com/stackabletech/kafka-operator/issues/655[Kerberos support] * Apache NiFi: https://github.com/stackabletech/nifi-operator/issues/633[OIDC support] -Security:: +===== Security * The Stackable Data Platform now supports provisioning TLS certificates using https://cert-manager.io/[cert-manager]. * Support has been added for customizing `sAMAccountName` generation in secret operator. * The Stackable Secret Operator now requests permission to read Listeners, which is required to provision secrets for listener volumes with the `listeners.stackable.tech/listener-name` annotation. * The RSA key length for generated key pairs can now be customized to 2048, 3072 or 4096 bits. The default is 2048 bits. -Listener:: +===== Listener * The Stackable Operator for Kafka now uses the Stackable Listener Operator, allowing connectivity to be customized. * Listeners can now be configured to use either IP addresses or fully qualified domain names (FQDNs). -Dependencies:: +===== Dependencies Apache HBase: The hadoop-azure module was added to the image and is contained in the classpath. This makes it possible to use the Azure Data Lake Storage Gen2 (ADLS) instead of HDFS. See the xref:hbase:usage-guide/adls.adoc[usage guide] for detailed information. -Operations:: +===== Operations The Stackable Operator for HDFS now supports upgrading existing HDFS installations. However, this process requires some manual intervention as described in xref:hdfs:usage-guide/upgrading.adoc[Upgrading HDFS]. -Miscellaneous:: +===== Miscellaneous * Apache NiFi: Permit users to configure allowed hosts when NiFi is running behind a proxy. The proxy host check is now disabled by default. @@ -201,7 +199,7 @@ Miscellaneous:: * Apache Airflow: Allow custom arbitrary python code in `webserver_config.py`. * Apache Superset: Allow custom arbitrary python code in `superset_config.py`. -Images:: +===== Images Support the `restricted-v2` SecurityContextConstraint (SCC) in OpenShift. Stackable currently defaults to the `nonroot-v2` SCC but we plan on migrating to the `restricted-v2` SCC in the future. @@ -213,24 +211,24 @@ Stackable currently defaults to the `nonroot-v2` SCC but we plan on migrating to ==== Platform improvements -Vulnerabilities:: +===== Vulnerabilities More than 142 CVEs were fixed in the Stackable product images. This includes 11 critical and 55 high-severity CVEs. -Authorization:: +===== Authorization * The performance of the xref:hdfs:usage-guide/security.adoc#\_authorization[HDFS OPA Authorizer] has been greatly improved. This can in some cases be a breaking change so please make sure to read the hdfs-utils https://github.com/stackabletech/hdfs-utils/releases/tag/v0.4.0[release notes] for details. * The User Info Fetcher HTTP API has been replaced with a Rego library. Please see xref:opa:usage-guide/user-info-fetcher#_user_info_fetcher_api[user-info-fetcher API] for more information. -Logging:: +===== Logging * Apache NiFi: The default size of ephemeral EmptyDir Volumes used to store log files before aggregation has been increased from 33 MiB to 500 MiB. Additionally the interval in which Logback checks if the maximum log file size has been reached has been reduced from 60 seconds to 5 seconds. * Apache NiFi: the create-reporting-task Job (and podOverrides on that Job) can now be disabled. -Monitoring:: +===== Monitoring https://github.com/prometheus/jmx_exporter[JMX Exporter] is a tool which allows us to expose JMX metrics as Prometheus metrics. It is used by the following products: Hadoop, HBase, Hive, Kafka, Spark, Trino and ZooKeeper. @@ -238,15 +236,15 @@ In the previous SDP release (24.7) we upgraded JMX Exporter from 0.20 to 1.0.1. Unfortunately version 1.0.1 has a severe performance degradation which has been https://github.com/prometheus/jmx_exporter/pull/1009[fixed upstream] but is not yet released. This SDP release (24.11) contains a fixed version bringing performance back to normal levels. -Listener:: +===== Listener The `ListenerClass.spec.serviceAnnotations` are now correctly propagated to created Service objects. -Miscellaneous:: +===== Miscellaneous The size of the operator deployed CRDs was reduced significantly (see: https://github.com/stackabletech/issues/issues/627[stackabletech/issues#627]). -Bug fixes:: +===== Bug fixes * Apache Spark: Ensure Spark applications are submitted only once. Reconciling applications after the corresponding Job objects have been recycled doesn't lead to the creation of new Job objects. @@ -263,7 +261,7 @@ Bug fixes:: ==== Platform deprecations -Commons:: +===== Commons Pod Enrichment is now deprecated, and will be removed in the next release. Once removed, the SDP will no longer set any `enrichment.stackable.tech/` annotations on Pods. @@ -432,7 +430,6 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-ope [source,console] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced -customresourcedefinition.apiextensions.k8s.io "airflowdbs.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced ... @@ -488,7 +485,6 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-ope [source,console] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced -customresourcedefinition.apiextensions.k8s.io "airflowdbs.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced ... diff --git a/modules/ROOT/partials/release-notes/release-24.3.adoc b/modules/ROOT/partials/release-notes/release-24.3.adoc index bdb0a8148..90d428961 100644 --- a/modules/ROOT/partials/release-notes/release-24.3.adoc +++ b/modules/ROOT/partials/release-notes/release-24.3.adoc @@ -6,14 +6,14 @@ Released on 2024-03-27. ==== New / extended platform features -Support for many new product versions:: +===== Support for many new product versions Almost all product images have been updated to their latest versions. Some notable examples are Apache Airflow 2.8.1, Trino 442 and Apache Spark 3.5.1. In addition, we started building some product binaries from source instead of packaging them from the official releases. This enables greater control over the features and the security aspects of the products. Currently, Apache Hadoop and Apache HBase are built from source and others will follow in coming releases. -Security:: +===== Security We put a special emphasis on security in this release. For this purpose we addressed the following topics: * Authorization @@ -55,26 +55,26 @@ In an upcoming release we will enable authentication and encryption by default w To ensure a smooth transition to future releases, we strongly encourage you to enable security features wherever possible in your deployments. -Storage:: +===== Storage HDFS deployments now support __rack awareness__. This is another unique feature that brings the SDP platform closer to feature parity with bare metal HDFS deployments. Of course, the exact meaning of __rack__ is different in Kubernetes environments, but the effect is the same: DataNodes are brought closer to the data they are reading and writing thus improving performance and reliability. A new https://github.com/stackabletech/hdfs-topology-provider[topology provider] is bundled with the HDFS image that maps Kubernetes labels to a cluster topology. // TODO: Link to docs - apparently the README in in the linked repository is all there is. -Documentation:: +===== Documentation We are constantly working on improving the platform documentation and custom resource definitions are a significant part of that. The CRD documentation is now generated automatically and can be found at https://crds.stackable.tech. -Command line tools:: +===== Command line tools The `stackablectl` command line tool has been overhauled and can now list endpoints provided by the listener operator. Also operator installation is parallelized, which considerably speeds up the process of setting up SDP on fresh Kubernetes clusters. -Custom labels for Helm charts:: +===== Custom labels for Helm charts In the past, Helm users could not assign custom labels to stacklets. This is now possible and it enables better component management with third party tools. -Bugfixes:: +===== Bugfixes * Apache Airflow Operator: Using git-sync with the KubernetesExecutor is now possible. * Apache Hadoop Operator: ** Kerberos principals are now included in the discovery ConfigMap. @@ -85,7 +85,7 @@ Bugfixes:: * Trino Operator: Add HDFS configuration files to the `hive.config.resources` property when connecting to a HDFS cluster. -Product features:: +===== Product features The following are selected product features provided by new versions available in this release: * Apache Airflow: Introducing Airflow Object Storage and Listener hooks for Datasets plus various bug fixes. @@ -110,7 +110,7 @@ The following are selected product features provided by new versions available i ** Also, starting from release 440, there is now row filtering and column masking in Open Policy Agent. * Apache ZooKeeper: Security and bug fixes. -Support for the ARM architecture:: +===== Support for the ARM architecture During the development of this release, we started introducing support for the arm64 architecture. Currently support is experimental, and we only provide arm64 images for the previous release (23.11). For more information on how to use the ARM images, refer to the xref:concepts:container-images.adoc#multi-platform-support[documentation]. @@ -382,7 +382,6 @@ kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-ope [source,console] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced -customresourcedefinition.apiextensions.k8s.io "airflowdbs.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced ... diff --git a/modules/ROOT/partials/release-notes/release-24.7.adoc b/modules/ROOT/partials/release-notes/release-24.7.adoc index 7572a77f7..a74e1479c 100644 --- a/modules/ROOT/partials/release-notes/release-24.7.adoc +++ b/modules/ROOT/partials/release-notes/release-24.7.adoc @@ -6,7 +6,7 @@ Released on 2024-07-25. ==== New / extended platform features -Vulnerability management:: +===== Vulnerability management In this release we have worked on significantly reducing the number of vulnerabilities in our product binaries. This will be more manageable going forward as we now build all Java-based binaries from source, which gives us greater flexibility when creating patches. @@ -21,7 +21,7 @@ NOTE: In this release we have eliminated 75% of all vulnerabilities that were pr This work will continue in the next version and the progress made in this release enables us to do more work in the future (workflow improvements, tooling etc.). -Build products from source:: +===== Build products from source All Java-based product binaries are now built from source instead of packaging them from the official releases. The status is summarised below: @@ -39,13 +39,13 @@ The status is summarised below: * Apache Superset: official release (Python-based) * Apache ZooKeeper: built from source from release 24.7 -Multi-platform images:: +===== Multi-platform images This release is the first multi-platform release of the Stackable Data Platform, supporting AMD64 and ARM64 architectures. Each image has a manifest list which wraps the architecture-specific image. The status is still xref:concepts:multi-platform-support.adoc[experimental], as we work to fine-tune the necessary workflows. -Security:: +===== Security Support for OIDC with/without TLS has been added to Apache Druid in this release. @@ -58,7 +58,7 @@ Check xref:hbase:usage-guide/security.adoc#_authorization[the documentation on H In an upcoming release we will enable authentication and encryption by default where possible. To ensure a smooth transition to future releases, we strongly encourage you to enable security features wherever possible in your deployments. -Documentation:: +===== Documentation * Apache Hive and Trino operators: we have provided non-trivial sample Rego rules for these operators, together with an in-depth explanation and links @@ -69,7 +69,7 @@ Documentation:: * Open Policy Agent: N.B. As mentioned in the release 24.3, we will be actively building out the backends supported by the User Info Fetcher. This feature should be therefore be treated as experimental as we continue to extend and consolidate back-end handling and fine-tune the tool in general. -Other product features:: +===== Other product features The following are selected product features provided by new versions available in this release: @@ -95,7 +95,7 @@ NOTE: There is currently a known problem with using git-sync credentials in 24.7 * Apache ZooKeeper: allow the overriding of the ZNode path by setting status.znodePath -Bugfixes:: +===== Bugfixes * Apache Druid: move the DB credentials user and password out of the CRD into a secret containing the keys username and password * Apache Hive: move the metastore user and password DB credentials out of the CRD into a Secret containing the keys username and password * Apache Kafka: remove field/arg controller_config from kafka_controller::Ctx struct and create_controller function @@ -285,7 +285,6 @@ kubectl delete -f https://raw.githubusercontent.com/stackabletech/hello-world-op [source,console] ---- customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced -customresourcedefinition.apiextensions.k8s.io "airflowdbs.airflow.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced ... diff --git a/modules/ROOT/partials/release-notes/release-25.3.adoc b/modules/ROOT/partials/release-notes/release-25.3.adoc new file mode 100644 index 000000000..d6ef0fb44 --- /dev/null +++ b/modules/ROOT/partials/release-notes/release-25.3.adoc @@ -0,0 +1,404 @@ +// Here are the headings you can use for the next release. Saves time checking indentation levels. +// Take a look at release 24.11 to see how to structure patch releases. + +== Release 25.3 + +=== 25.3.0 + +Released on 2025-03-21. + +==== New platform features + +===== General + +* Operators deployed by Helm will contain an extra pod annotation: `checksum/config`. + This field triggers a rollout of Deployments when the ConfigMap contents change. +* Operators for Java products now support setting JVM arguments. + See the xref:concepts:overrides.adoc[override concept page] and https://github.com/stackabletech/issues/issues/584[issues#584]. + +===== Authorization + +* Apache Airflow: Authorization can now be delegated to an Open Policy Agent. + See https://github.com/stackabletech/airflow-operator/issues/446[airflow-operator#446]. +* Apache Superset: Support Open Policy Agent role mapping. + See https://github.com/stackabletech/superset-operator/pull/582[superset-operator#582]. + +===== Security + +* Additional trust roots can be specified in an `autoTls` SecretClass. + See the xref:secret-operator:secretclass.adoc[`autoTls` backend documentation]. +* The Stackable Secret Operator's `experimentalCertManager` backend now supports specifying custom key lengths. + The `autoTls` backend has supported this since 24.11. +* Users can now configure the lifetime of self-signed certificates directly in the product's custom resources which influences the frequency of pod restarts. + Details can be found on the xref:concepts:operations/temporary_credentials_lifetime.adoc[temporary credentials lifetime page]. + +===== Observability + +Products now log information about their xref:concepts:containerdebug.adoc[container environment], on startup and on an interval after that. + +===== NiFi + +Add Hadoop libraries for accessing Azure and GCP. +See https://github.com/stackabletech/docker-images/pull/943[docker-images#943]. + +==== Platform improvements + +===== General + +====== OCI Registry + +Starting with this release, our OCI registry located at `oci.stackable.tech` was promoted to stable. +This means that our operators, the product image selection and tools will use the registry by default instead of the previous Docker repository located at `docker.stackable.tech`. + +[NOTE] +==== +To ease the transition, operator images, product images, and operator Helm charts are published both on the old Docker repository as well as the new OCI registry. +Subsequent releases will **only** be published on `oci.stackable.tech`. +==== + +Using the old Docker repository is still possible: + +* Products can use the old images by setting the `spec.image.repo` to `docker.stackable.tech/stackable` in the respective product CustomResource. +* Our `stackablectl` tool can pull Helm charts from the old repository by providing the `--chart-source repo` argument during operator installs. + Unfortunately, some Helm chart values point to the OCI registry and `stackablectl` doesn't support overriding these values. + Use `helm` directly instead. +* Operators can be installed via `helm install` with a custom `image.repository` value set. + +[source,console] +---- +$ helm repo add stackable-stable https://repo.stackable.tech/repository/helm-stable/ +$ helm repo update stackable-stable +$ helm install --wait airflow-operator stackable-stable/airflow-operator --version 25.3 --set image.repository=docker.stackable.tech/stackable/airflow-operator +---- + +====== DNS lookup performance + +Users can now opt-in to experimental support for improved DNS performance by specifying a fully qualified domain name (with a trailing dot) as Kubernetes cluster domain for our operators. +This can reduce the amount of DNS queries within the cluster, but might have side effects, hence we consider the support experimental for now. +Take a look at xref:guides:kubernetes-cluster-domain.adoc[our documentation] to find out how to enable this improvement. + +===== Vulnerabilities + +41 CVEs were fixed in the Stackable product images. +This includes 6 critical and 16 high-severity CVEs. + +===== Authorization + +OPA Rego rules no longer require the `future.keywords` import. +They have been stabilized in OPA 1.0 and are now implicitly imported. + +===== Druid + +The default memory limits have been increased for following roles. +See https://github.com/stackabletech/druid-operator/pull/685[druid-operator#685]. + +* Coordinator: From `512Mi` to `768Mi` +* Middle Manager: From `1Gi` to `1500Mi`. + +NOTE: Upgrades to existing deployments could cause memory limits to be reached on the node/namespace. + +===== Hive + +The the default memory reservation of Hive metastore has been increased from `512Mi` to `768Mi` to avoid OOMKilled events. +See https://github.com/stackabletech/hive-operator/pull/578[hive-operator#578]. + +===== Bug fixes + +* Previously, pods with a Listener volume were stuck in an "Unknown" state after their node was restarted. + With this release, Listener volumes are correctly republished and the pods restart as expected. + See https://github.com/stackabletech/listener-operator/issues/262[listener-operator#262]. +* In 24.11 we used a custom build of jmx_exporter to resolve a https://github.com/stackabletech/issues/issues/649[performance degradation]. + In this release, Java products ship with the fixed upstream https://github.com/prometheus/jmx_exporter/releases/tag/1.1.0[jmx_exporter 1.1.0] which includes https://github.com/prometheus/jmx_exporter/pull/1009[the fix]. + +==== Platform deprecations + +===== Kafka operator + +The `-nodeport` discovery ConfigMaps have been deprecated for removal. +Use the primary discovery CMs instead. +See the https://github.com/stackabletech/kafka-operator/issues/765[deprecation tracking issue ]for more information. + +==== Product versions + +As with previous SDP releases, many product images have been updated to their latest versions. +The LTS version has in many cases also been adjusted in line with our xref:ROOT:policies.adoc[support policy]. + +Refer to the xref:operators:supported_versions.adoc[supported versions] documentation for a complete overview including LTS versions or deprecations. + +===== New versions + +The following new product versions are now supported: + +* Apache Airflow: https://github.com/stackabletech/docker-images/issues/1017[2.10.4] +* Apache Druid: https://github.com/stackabletech/docker-images/issues/965[31.0.1], https://github.com/stackabletech/docker-images/issues/965[30.0.1 (LTS)] +* Apache Hadoop: https://github.com/stackabletech/docker-images/issues/1018[3.4.1 (LTS)] +* Apache HBase: https://github.com/stackabletech/docker-images/issues/972[2.6.1 (LTS)] +* Apache Hive: https://github.com/stackabletech/docker-images/issues/1019[4.0.0 (LTS)], https://github.com/stackabletech/docker-images/issues/1019[4.0.1 (experimental)] +* Apache Kafka: https://github.com/stackabletech/docker-images/issues/968[3.7.2 (LTS)], https://github.com/stackabletech/docker-images/issues/968[3.9.0] +* Apache NiFi: https://github.com/stackabletech/docker-images/issues/966[1.28.1], https://github.com/stackabletech/docker-images/issues/966[2.2.0 (experimental)] +* Apache Spark: https://github.com/stackabletech/docker-images/issues/1016[3.5.5 (LTS)] +* Apache Superset: https://github.com/stackabletech/docker-images/issues/970[4.1.1] +* Apache ZooKeeper: https://github.com/stackabletech/docker-images/issues/1020[3.9.3 (LTS)] +* Open Policy Agent: https://github.com/stackabletech/docker-images/issues/998[1.0.1] +* Trino: https://github.com/stackabletech/docker-images/issues/971[470] + +===== Deprecated versions + +The following product versions are deprecated and will be removed in a later release: + +* Apache Druid: https://github.com/stackabletech/docker-images/issues/965[30.0.0] +* Apache Hadoop: https://github.com/stackabletech/docker-images/issues/1018[3.4.0] +* Apache HBase: https://github.com/stackabletech/docker-images/issues/972[2.4.18] +* Apache Hive: https://github.com/stackabletech/docker-images/issues/1019[3.1.3] +* Apache Kafka: https://github.com/stackabletech/docker-images/issues/968[3.8.0], https://github.com/stackabletech/docker-images/issues/968[3.7.1] +* Apache Spark: https://github.com/stackabletech/docker-images/issues/1016[3.5.2] +* Apache ZooKeeper: https://github.com/stackabletech/docker-images/issues/1020[3.9.2] +* Open Policy Agent: https://github.com/stackabletech/docker-images/issues/969[0.67.1] +* Trino: https://github.com/stackabletech/docker-images/issues/971[455] + +===== Removed versions + +The following product versions are no longer supported (although images for released product versions remain available https://repo.stackable.tech/#browse/browse:docker:v2%2Fstackable[here]): + +* Apache Airflow: https://github.com/stackabletech/docker-images/issues/1017[2.10.2], https://github.com/stackabletech/docker-images/issues/1017[2.9.2] +* Apache Druid: https://github.com/stackabletech/docker-images/issues/965[26.0.0] +* Apache HBase: https://github.com/stackabletech/docker-images/issues/972[2.6.0] +* Apache NiFi: https://github.com/stackabletech/docker-images/issues/966[2.0.0] +* Apache Spark: https://github.com/stackabletech/docker-images/issues/1016[3.5.1] +* Open Policy Agent: https://github.com/stackabletech/docker-images/issues/969[0.66.0] + +==== stackablectl + +* A new demo called `jupyterhub-keycloak` was added and is available via `stackablectl`. + The JupyterHub-Keycloak integration demo offers a comprehensive and secure multi-user data science environment on Kubernetes, integrating Single Sign-on Jupyter notebooks with Stackable Spark and S3 storage. + The demo can be installed by running `stackablectl demo install jupyterhub-keycloak`. + See https://github.com/stackabletech/demos/pull/155[demos#155] and https://github.com/stackabletech/documentation/pull/715[documentation#715]. +* Demos and stacks are now versioned and the main branch is considered unstable. + `stackablectl` by default installs the latest stable demo and/or stack. + A specific release can be targeted by providing the `--release` argument. + See https://github.com/stackabletech/stackable-cockpit/pull/340[stackable-cockpit#340]. +* Add new argument --chart-source so that operator charts can be pulled either from an OCI registry (the default) or from a index.yaml-based repository. + See https://github.com/stackabletech/stackable-cockpit/pull/344[stackable-cockpit#344]. +* Use `rustls-native-certs` so that `stackablectl` can be used in environments with internal PKI. + See https://github.com/stackabletech/stackable-cockpit/pull/351[stackable-cockpit#351]. +* Use `heritage` label when looking up the `minio-console` stacklet. + See https://github.com/stackabletech/stackable-cockpit/pull/364[stackable-cockpit#364]. +* Improve tracing and log output. + See https://github.com/stackabletech/stackable-cockpit/pull/365[stackable-cockpit#365]. + +==== Supported Kubernetes versions + +This release supports the following Kubernetes versions: + +* `1.32` +* `1.31` +* `1.30` + +These Kubernetes versions are no longer supported: + +* `1.29` + +==== Supported OpenShift versions + +This release is available in the RedHat Certified Operator Catalog for the following OpenShift versions: + +* `4.17` +* `4.16` +* `4.15` +* `4.14` + +==== Breaking changes + +Of the changes mentioned above, the following are breaking (or could lead to breaking behaviour), and you will need to adapt your existing CRDs accordingly: + +===== General + +S3 bucket `region` can now be configured for `S3Connection`, `S3Bucket`, and inline S3 references. +It defaults to `us-east-1`. +See the https://github.com/stackabletech/issues/issues/696[tracking issue]. + +NOTE: Products that use the Hadoop S3 implementation previously defaulted to `us-east-2`, so if there are bucket connectivity problems, you will need to set the region `us-east-2` explicitly. + +===== Airflow operator + +The field `.spec.clusterConfig.dagsGitSync[].wait` changed from `uint8` to our human-readable xref:reference:duration.adoc[`Duration`] type. +If you have specified a time without a unit, eg: `wait: 20`, you will need to add the applicable unit, eg: `wait: 20s`. + +===== Druid operator + +NOTE: All Druid versions are affected. + +If druid-opa-authorizer is used, `input.user` needs to be replaced by `input.authenticationResult.identity` in applicable Rego rules. +Change in https://github.com/stackabletech/druid-opa-authorizer/pull/85[druid-opa-authorizer#85]. + +===== OPA operator + +* Using `if` for all rules and `contains` for multi-value rules is now mandatory. +* `strict` mode is now enabled by default. + For more upgrade information, read the https://www.openpolicyagent.org/docs/latest/v0-upgrade/#upgrading-rego[Upgrading Rego] section of the official documentation. + +===== Trino operator + +Trino now uses the native S3 implementation which has the following requirements for S3 connections: + +* TLS is always enabled and cannot be disabled. +* Client-side encryption is not supported. + Server-side encryption (SSE) is the recommended alternative. +* Multipart (non-streaming) writes and upload are not supported. + +Legacy S3 support (via Hadoop) has been disabled and will be removed in a future version of Trino. + +==== Upgrade from 24.11 + +===== Using stackablectl + +Uninstall the `24.11` release + +[source,console] +---- +$ stackablectl release uninstall 24.11 + +Uninstalled release '24.11' + +Use "stackablectl release list" to list available releases. +# ... +---- + +Afterwards you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. +The reason for this is that helm will uninstall the operators but not the CRDs. +This can be done using `kubectl replace`. + +[source] +---- +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/25.3.0/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/25.3.0/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/25.3.0/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/25.3.0/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/25.3.0/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/25.3.0/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/25.3.0/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.3.0/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.3.0/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.3.0/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.3.0/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.3.0/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.3.0/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.3.0/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.3.0/deploy/helm/zookeeper-operator/crds/crds.yaml +---- + +[source,console] +---- +customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced +... +---- + +Install the `25.3` release + +[source,console] +---- +$ stackablectl release install 25.3 + +Installed release '25.3' + +Use "stackablectl operator installed" to list installed operators. +---- + +===== Using Helm + +Use `helm list` to list the currently installed operators. + +You can use the following command to uninstall all operators that are part of the `24.11` release: + +[source,console] +---- +$ helm uninstall airflow-operator commons-operator druid-operator hbase-operator hdfs-operator hive-operator kafka-operator listener-operator nifi-operator opa-operator secret-operator spark-k8s-operator superset-operator trino-operator zookeeper-operator +release "airflow-operator" uninstalled +release "commons-operator" uninstalled +... +---- + +Afterward you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. +The reason for this is that helm will uninstall the operators but not the CRDs. +This can be done using `kubectl replace`: + +[source] +---- +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/25.3.0/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/25.3.0/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/25.3.0/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/25.3.0/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/25.3.0/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/25.3.0/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/25.3.0/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/25.3.0/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/25.3.0/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/25.3.0/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/25.3.0/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/25.3.0/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/25.3.0/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/25.3.0/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/25.3.0/deploy/helm/zookeeper-operator/crds/crds.yaml +---- + +[source,console] +---- +customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced +... +---- + +Install the `25.3` release + +[source,console] +---- +helm repo add stackable-stable https://repo.stackable.tech/repository/helm-stable/ +helm repo update stackable-stable +helm install --wait airflow-operator stackable-stable/airflow-operator --version 25.3.0 +helm install --wait commons-operator stackable-stable/commons-operator --version 25.3.0 +helm install --wait druid-operator stackable-stable/druid-operator --version 25.3.0 +helm install --wait hbase-operator stackable-stable/hbase-operator --version 25.3.0 +helm install --wait hdfs-operator stackable-stable/hdfs-operator --version 25.3.0 +helm install --wait hive-operator stackable-stable/hive-operator --version 25.3.0 +helm install --wait kafka-operator stackable-stable/kafka-operator --version 25.3.0 +helm install --wait listener-operator stackable-stable/listener-operator --version 25.3.0 +helm install --wait nifi-operator stackable-stable/nifi-operator --version 25.3.0 +helm install --wait opa-operator stackable-stable/opa-operator --version 25.3.0 +helm install --wait secret-operator stackable-stable/secret-operator --version 25.3.0 +helm install --wait spark-k8s-operator stackable-stable/spark-k8s-operator --version 25.3.0 +helm install --wait superset-operator stackable-stable/superset-operator --version 25.3.0 +helm install --wait trino-operator stackable-stable/trino-operator --version 25.3.0 +helm install --wait zookeeper-operator stackable-stable/zookeeper-operator --version 25.3.0 +---- + +==== Known issues + +===== Hive operator + +In Hive 4.0.1 with Kerberos enabled, health checks cause excessive error logs: + +[source] +---- +ERROR [Metastore-Handler-Pool: Thread-65] server.TThreadPoolServer: Thrift Error occurred during processing of message. +---- + +This is because the health check doesn't complete SASL authentication. +The error is ignorable, though it can be hidden with the following configuration: + +[source,yaml] +---- +spec: + metastore: + config: + logging: + containers: + hive: + loggers: + org.apache.thrift.server.TThreadPoolServer: + level: NONE +---- + +NOTE: This will suppress all logging from `TThreadPoolServer`, including log events that might be useful for diagnosing issues. diff --git a/modules/ROOT/partials/release-notes/release-template.adoc b/modules/ROOT/partials/release-notes/release-template.adoc index fd5452384..2ea6f9e22 100644 --- a/modules/ROOT/partials/release-notes/release-template.adoc +++ b/modules/ROOT/partials/release-notes/release-template.adoc @@ -3,11 +3,11 @@ == Release YY.M +=== YY.M.X + Released on YYYY-MM-DD. (Optional description / introduction) -=== YY.M.X - ==== New platform features ==== Platform improvements @@ -16,18 +16,51 @@ Released on YYYY-MM-DD. ==== Product versions +As with previous SDP releases, many product images have been updated to their latest versions. +The LTS version has in many cases also been adjusted in line with our xref:ROOT:policies.adoc[support policy]. + +Refer to the xref:operators:supported_versions.adoc[supported versions] documentation for a complete overview including LTS versions or deprecations. + ===== New versions +The following new product versions are now supported: + +* ... + ===== Deprecated versions +The following product versions are deprecated and will be removed in a later release: + +* ... + ===== Removed versions +The following product versions are no longer supported (although images for released product versions remain available https://repo.stackable.tech/#browse/browse:docker:v2%2Fstackable[here]): + +* ... + ==== stackablectl ==== Supported Kubernetes versions +This release supports the following Kubernetes versions: + +* `1.XX` + +These Kubernetes versions are no longer supported: + +* `1.XX` + ==== Supported OpenShift versions +This release is available in the RedHat Certified Operator Catalog for the following OpenShift versions: + +* `4.XX` + +These OpenShift versions are no longer supported: + +* `4.XX` + ==== Breaking changes Of the changes mentioned above, the following are breaking (or could lead to breaking behaviour), and you will need to adapt your existing CRDs accordingly: @@ -48,6 +81,125 @@ Of the changes mentioned above, the following are breaking (or could lead to bre ===== Using stackablectl +Uninstall the `OO.M` release + +[source,console] +---- +$ stackablectl release uninstall OO.M + +Uninstalled release 'OO.M' + +Use "stackablectl release list" to list available releases. +# ... +---- + +Afterwards you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. +The reason for this is that helm will uninstall the operators but not the CRDs. +This can be done using `kubectl replace`. + +[source] +---- +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/OO.M.X/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/OO.M.X/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/OO.M.X/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/OO.M.X/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/OO.M.X/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/OO.M.X/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/OO.M.X/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/OO.M.X/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/OO.M.X/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/OO.M.X/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/OO.M.X/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/OO.M.X/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/OO.M.X/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/OO.M.X/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/OO.M.X/deploy/helm/zookeeper-operator/crds/crds.yaml +---- + +[source,console] +---- +customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced +... +---- + +Install the `YY.M` release + +[source,console] +---- +$ stackablectl release install YY.M + +Installed release 'YY.M' + +Use "stackablectl operator installed" to list installed operators. +---- + ===== Using Helm +Use `helm list` to list the currently installed operators. + +You can use the following command to uninstall all operators that are part of the `OO.M` release: + +[source,console] +---- +$ helm uninstall airflow-operator commons-operator druid-operator hbase-operator hdfs-operator hive-operator kafka-operator listener-operator nifi-operator opa-operator secret-operator spark-k8s-operator superset-operator trino-operator zookeeper-operator +release "airflow-operator" uninstalled +release "commons-operator" uninstalled +... +---- + +Afterward you will need to upgrade the CustomResourceDefinitions (CRDs) installed by the Stackable Platform. +The reason for this is that helm will uninstall the operators but not the CRDs. This can be done using `kubectl replace`: + +[source] +---- +kubectl replace -f https://raw.githubusercontent.com/stackabletech/airflow-operator/OO.M.X/deploy/helm/airflow-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/commons-operator/OO.M.X/deploy/helm/commons-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/druid-operator/OO.M.X/deploy/helm/druid-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hbase-operator/OO.M.X/deploy/helm/hbase-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hdfs-operator/OO.M.X/deploy/helm/hdfs-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/hive-operator/OO.M.X/deploy/helm/hive-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/kafka-operator/OO.M.X/deploy/helm/kafka-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/listener-operator/OO.M.X/deploy/helm/listener-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/nifi-operator/OO.M.X/deploy/helm/nifi-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/opa-operator/OO.M.X/deploy/helm/opa-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/secret-operator/OO.M.X/deploy/helm/secret-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/spark-k8s-operator/OO.M.X/deploy/helm/spark-k8s-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/superset-operator/OO.M.X/deploy/helm/superset-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/trino-operator/OO.M.X/deploy/helm/trino-operator/crds/crds.yaml +kubectl replace -f https://raw.githubusercontent.com/stackabletech/zookeeper-operator/OO.M.X/deploy/helm/zookeeper-operator/crds/crds.yaml +---- + +[source,console] +---- +customresourcedefinition.apiextensions.k8s.io "airflowclusters.airflow.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "authenticationclasses.authentication.stackable.tech" replaced +customresourcedefinition.apiextensions.k8s.io "s3connections.s3.stackable.tech" replaced +... +---- + +Install the `YY.M` release + +[source,console] +---- +helm repo add stackable-stable https://repo.stackable.tech/repository/helm-stable/ +helm repo update stackable-stable +helm install --wait airflow-operator stackable-stable/airflow-operator --version OO.M.X +helm install --wait commons-operator stackable-stable/commons-operator --version OO.M.X +helm install --wait druid-operator stackable-stable/druid-operator --version OO.M.X +helm install --wait hbase-operator stackable-stable/hbase-operator --version OO.M.X +helm install --wait hdfs-operator stackable-stable/hdfs-operator --version OO.M.X +helm install --wait hive-operator stackable-stable/hive-operator --version OO.M.X +helm install --wait kafka-operator stackable-stable/kafka-operator --version OO.M.X +helm install --wait listener-operator stackable-stable/listener-operator --version OO.M.X +helm install --wait nifi-operator stackable-stable/nifi-operator --version OO.M.X +helm install --wait opa-operator stackable-stable/opa-operator --version OO.M.X +helm install --wait secret-operator stackable-stable/secret-operator --version OO.M.X +helm install --wait spark-k8s-operator stackable-stable/spark-k8s-operator --version OO.M.X +helm install --wait superset-operator stackable-stable/superset-operator --version OO.M.X +helm install --wait trino-operator stackable-stable/trino-operator --version OO.M.X +helm install --wait zookeeper-operator stackable-stable/zookeeper-operator --version OO.M.X +---- + ==== Known issues diff --git a/ui b/ui index 94047125e..ebadd01d7 160000 --- a/ui +++ b/ui @@ -1 +1 @@ -Subproject commit 94047125e9561a6b73d583dc4e9652c6a687c008 +Subproject commit ebadd01d74f1183a9dbd5e0e0c26bb792a7245e0