Merge branch 'main' into spike/lb-operator

Author: siegfriedweber

.dockerignore

@@ -5,5 +5,10 @@ target/
 .idea/
 *.iws
 
+Cargo.nix
+crate-hashes.json
+result
+image.tar
+
 # We do NOT want to ignore .git because we use the `built` crate to gather the current git commit hash at built time
 # This means we need the .git directory in our Docker image, it will be thrown away and won't be included in the final image

.github/ISSUE_TEMPLATE/new_version.md

@@ -29,6 +29,7 @@ Not all of these steps are necessary for all versions.
 
 - [ ] Update the Docker image
 - [ ] Update documentation to include supported version(s)
+- [ ] Update and test getting started guide with updated version(s)
 - [ ] Update operator to support the new version (if needed)
 - [ ] Update integration tests to test use the new versions (in addition or replacing old versions
 - [ ] Update examples to use new versions

.github/workflows/build.yml

@@ -33,13 +33,17 @@ jobs:
     env:
       RUSTC_BOOTSTRAP: 1
     steps:
+      - name: Install protoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "1.63.0"
           override: true
       - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
         with:
@@ -61,7 +65,7 @@ jobs:
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
-      - uses: actions/setup-python@c4e89fac7e8767b327bbad6cb4d859eda999cf08 # tag=v4
+      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # tag=v4
       - name: Install requirements for version tool
         run: pip install -r python/requirements.txt
       - id: prerelease
@@ -146,7 +150,7 @@ jobs:
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "1.63.0"
           components: rustfmt
           override: true
       - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # renovate: tag=v1.0.3
@@ -158,13 +162,17 @@ jobs:
     name: Run Clippy
     runs-on: ubuntu-latest
     steps:
+      - name: Install protoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "1.63.0"
           components: clippy
           override: true
       - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
@@ -188,13 +196,17 @@ jobs:
     name: Run RustDoc
     runs-on: ubuntu-latest
     steps:
+      - name: Install protoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "1.63.0"
           components: rustfmt
           override: true
       - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
@@ -214,13 +226,17 @@ jobs:
       - run_rustdoc
     runs-on: ubuntu-latest
     steps:
+      - name: Install protoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "1.63.0"
           override: true
       - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
         with:
@@ -229,6 +245,31 @@ jobs:
         with:
           command: test
 
+  # Similar to check_charts, this tries to render the README, and see if there are unintended changes.
+  # This will save us from merging changes to the wrong file (instead of the templated source), and from
+  # forgetting to render out modifications to the README.
+  check_readme:
+    name: Check if committed README is the one we would render from the available parts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+        with:
+          submodules: recursive
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: Regenerate charts
+        run: make render-readme
+      - name: Check if committed README were up to date
+        run: git diff --exit-code
+      - name: Git Diff showed uncommitted changes
+        if: ${{ failure() }}
+        uses: actions/github-script@d50f485531ba88479582bc2da03ff424389af5c1 # tag=v6
+        with:
+          script: |
+            core.setFailed('Committed README are not up to date, please make sure to apply them to the templated partials, and re-commit!')
+
   # This job cleans up the CRDs, Helm charts and Kustomize manifests, followed by rebuilding them
   # It then runs a `git diff` and fails the entire workflow, if any difference is encountered.
   #
@@ -248,19 +289,23 @@ jobs:
       - run_rustdoc
     runs-on: ubuntu-latest
     steps:
+      - name: Install protoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
       - name: Set up Helm
-        uses: azure/setup-helm@e4f3964f67aac34233ac06d6614802b7040f2a7c # tag=v3.1
+        uses: azure/setup-helm@b5b231a831f96336bbfeccc1329990f0005c5bb1 # tag=v3.3
         with:
           version: v3.6.2
       - name: Set up cargo
         uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "1.63.0"
           override: true
       - name: Set up rust-cache
         uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
@@ -272,7 +317,7 @@ jobs:
         run: git diff --exit-code
       - name: Git Diff showed uncommitted changes
         if: ${{ failure() }}
-        uses: actions/github-script@7a5c598405937d486b0331594b5da2b14db670da # tag=v6
+        uses: actions/github-script@7dff1a87643417cf3b95bb10b29f4c4bc60d8ebd # tag=v6
         with:
           script: |
             core.setFailed('Committed charts were not up to date, please regenerate and re-commit!')
@@ -312,16 +357,20 @@ jobs:
       REPO: ${{ needs.select_repo.outputs.repository }}
     if: needs.select_repo.outputs.repository != 'skip'
     steps:
+      - name: Install protoc
+        run: |
+          sudo apt-get update
+          sudo apt-get install protobuf-compiler
       - name: Checkout
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
         with:
           submodules: recursive
-      - uses: actions/setup-python@c4e89fac7e8767b327bbad6cb4d859eda999cf08 # tag=v4
+      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # tag=v4
         if: ${{ github.event_name == 'pull_request' }}
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: "1.63.0"
           components: rustfmt
           override: true
       - name: Install requirements for version tool

.github/workflows/reviewdog.yaml

@@ -14,15 +14,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
-      - uses: reviewdog/action-actionlint@12f228ecba8c567a103efafb3fb0bf5b60dc16b7 # tag=v1.27.0
+      - uses: reviewdog/action-actionlint@fe5c2db6029881e0b5634d7cb0d119b0c16b6922 # tag=v1.32.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   flake8:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
-      - uses: actions/setup-python@c4e89fac7e8767b327bbad6cb4d859eda999cf08 # tag=v4
+      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # tag=v4
         with:
           python-version: "3.9"
       - uses: reviewdog/action-flake8@b6435e67f0cfda225b9e0c9283cfb7ea7c551bdb # tag=v3.6.0
@@ -41,9 +41,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
-      - uses: reviewdog/action-markdownlint@b8f945b8bee2a2967214f35956209bc31d3c4d26 # tag=v0.7.1
+      - uses: reviewdog/action-markdownlint@97e3df02fe1573d505a7b268046a44fa5409cbc3 # tag=v0.9.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
+          markdownlint_flags: '-i README.md .'
 
   shellcheck:
     runs-on: ubuntu-latest

.gitignore

@@ -13,3 +13,4 @@ target/
 Cargo.nix
 crate-hashes.json
 result
+image.tar

.readme/README.md.j2

@@ -0,0 +1,12 @@
+{%- set title="Stackable Operator for Apache Kafka" -%}
+{%- set operator_name="kafka" -%}
+{%- set related_reading_links=[] -%}
+
+{% include "partials/borrowed/header.md.j2" %}
+{% include "partials/borrowed/links.md.j2" %}
+
+{% include "partials/main.md.j2" %}
+
+{% include "partials/borrowed/footer.md.j2" %}
+
+{% include "partials/borrowed/related_reading.md.j2" %}

.readme/partials/borrowed/documentation.md.j2

@@ -0,0 +1,6 @@
+## Documentation
+
+The stable documentation for this operator can be found [here](https://docs.stackable.tech/{{operator_name}}/stable/index.html).
+If you are interested in the most recent state of this repository, check out the [nightly docs](https://docs.stackable.tech/{{operator_name}}/nightly/index.html) instead.
+
+The documentation for all Stackable products can be found at [docs.stackable.tech](https://docs.stackable.tech).

.readme/partials/borrowed/footer.md.j2

@@ -0,0 +1,62 @@
+## About The Stackable Data Platform
+
+This operator is written and maintained by [Stackable](https://www.stackable.tech) and it is part of a larger data platform.
+
+![Stackable Data Platform Overview](./.readme/static/borrowed/sdp_overview.png)
+
+Stackable makes it easy to operate data applications in any Kubernetes cluster.
+
+The data platform offers many operators, new ones being added continuously. All our operators are designed and built to be easily interconnected and to be consistent to work with.
+
+The [Stackable GmbH](https://stackable.tech/) is the company behind the Stackable Data Platform. Offering professional services, paid support plans and custom development.
+
+We love open-source!
+
+## Supported Platforms
+
+We develop and test our operators on the following cloud platforms:
+
+* Kubernetes 1.21-1.24
+* EKS on Amazon Web Services (AWS)
+* GKE on Google Cloud Platform (GCP)
+* AKS on Microsoft Azure
+* K3s
+
+We are currently working to support:
+
+* OpenShift
+
+## Other Operators
+
+These are the operators that are currently part of the Stackable Data Platform:
+
+- [Stackable Operator for Apache Airflow](https://github.com/stackabletech/airflow-operator)
+- [Stackable Operator for Apache Druid](https://github.com/stackabletech/druid-operator)
+- [Stackable Operator for Apache HBase](https://github.com/stackabletech/hbase-operator)
+- [Stackable Operator for Apache Hadoop HDFS](https://github.com/stackabletech/hdfs-operator)
+- [Stackable Operator for Apache Hive](https://github.com/stackabletech/hive-operator)
+- [Stackable Operator for Apache Kafka](https://github.com/stackabletech/kafka-operator)
+- [Stackable Operator for Apache NiFi](https://github.com/stackabletech/nifi-operator)
+- [Stackable Operator for Apache Spark](https://github.com/stackabletech/spark-k8s-operator)
+- [Stackable Operator for Apache Superset](https://github.com/stackabletech/superset-operator)
+- [Stackable Operator for Trino](https://github.com/stackabletech/trino-operator)
+- [Stackable Operator for Apache ZooKeeper](https://github.com/stackabletech/zookeeper-operator)
+
+And our internal operators:
+
+- [Commons Operator](https://github.com/stackabletech/commons-operator)
+- [Listener Operator](https://github.com/stackabletech/listener-operator)
+- [OpenPolicyAgent Operator](https://github.com/stackabletech/opa-operator)
+- [Secret Operator](https://github.com/stackabletech/secret-operator)
+
+## Contributing
+
+Contributions are welcome. Follow our [Contributors Guide](https://docs.stackable.tech/home/stable/contributor/index.html) to learn how you can contribute.
+
+## License
+
+[Open Software License version 3.0](./LICENSE).
+
+## Support
+
+You can use this project under different licenses. Get started with the community edition! If you want professional support, [we offer subscription plans](https://stackable.tech/en/plans/).

.readme/partials/borrowed/header.md.j2

@@ -0,0 +1,5 @@
+<p align="center">
+  <img width="150" src="./.readme/static/borrowed/Icon_Stackable.svg" alt="Stackable Logo"/>
+</p>
+
+<h1 align="center">{{title}}</h1>

.readme/partials/borrowed/links.md.j2

@@ -0,0 +1,6 @@
+[![Build Actions Status](https://ci.stackable.tech/job/{{operator_name}}%2doperator%2dit%2dnightly/badge/icon?subject=Integration%20Tests)](https://ci.stackable.tech/job/{{operator_name}}%2doperator%2dit%2dnightly)
+[![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://GitHub.com/stackabletech/{{operator_name}}-operator/graphs/commit-activity)
+[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-green.svg)](https://docs.stackable.tech/home/stable/contributor/index.html)
+[![License OSL3.0](https://img.shields.io/badge/license-OSL3.0-green)](./LICENSE)
+
+[Documentation](https://docs.stackable.tech/{{operator_name}}-operator/stable/index.html) {% if quickstart_link %}| [Quickstart]({{quickstart_link}}) {% endif %}| [Stackable Data Platform](https://stackable.tech/) | [Platform Docs](https://docs.stackable.tech/)

.readme/partials/borrowed/related_reading.md.j2

@@ -0,0 +1,6 @@
+{%- if related_reading_links -%}
+## Related Reading
+{% for (text, link) in related_reading_links %}
+* [{{text}}]({{link}})
+{%- endfor %}
+{%- endif -%}

.readme/partials/main.md.j2

@@ -0,0 +1,48 @@
+This is a Kubernetes operator to manage [Apache Kafka](https://kafka.apache.org/) clusters.
+
+It is part of the Stackable Data Platform, a curated selection of the best open source data apps like Kafka, Druid, Trino or Spark, [all](#other-operators) working together seamlessly. Based on Kubernetes, it runs everywhere – [on prem or in the cloud]((#supported-platforms)).
+
+## Installation
+
+You can install the operator using [stackablectl or helm](https://docs.stackable.tech/kafka/stable/getting_started/installation.html).
+
+Read on to get started with it, or see it in action in one of our [demos](https://stackable.tech/en/demos/).
+
+## Getting Started
+
+You can follow this [tutorial](https://docs.stackable.tech/kafka/stable/getting_started/first_steps.html) to create a Kafka cluster with three replicas.
+
+{% include "partials/borrowed/documentation.md.j2" %}
+
+## What Does This Do?
+
+This operator is managing Kafka in Kubernetes. It is part of the Stackable Data Platform, a 
+
+Operators observe custom resources in a Kubernetes cluster, and manages the application it's responsible for according to the desired state specified in those custom resources.
+
+Here's an example how a custom resource to bring up a 3-replica Kafka cluster can look like:
+
+```
+---
+apiVersion: kafka.stackable.tech/v1alpha1
+kind: KafkaCluster
+metadata:
+  name: simple-kafka
+spec:
+  version: 3.2.0-stackable0.1.0
+  zookeeperConfigMapName: simple-kafka-znode
+  config:
+    tls: null
+  brokers:
+    roleGroups:
+      default:
+        replicas: 3
+```
+
+If you want to find out more about it, take a look at this [getting started guide](https://docs.stackable.tech/kafka/stable/getting_started/first_steps.html).
+
+## Behind the Scenes
+
+This operator is written by [Stackable](https://www.stackable.tech). It uses [kube.rs](https://kube.rs/) to interact with the Kubernetes control plane.
+
+We test it [extensively](https://ci.stackable.tech/) using [Kuttl](https://kuttl.dev/)-powered integration tests on managed Kubernetes of multiple cloud platforms and our own bare-metal clusters.