More demo fixes (#69)

* set hadoop to 3.3.6 and use 24.3 image for distcp job due to map reduce missing

* hdfs 3.3.6 -> 3.3.4

* fix pyspark version in jupyterhub demo

* fix to latest available

* bump kafka version in data-lakehouse demo

---------

Co-authored-by: xeniape <xenia.fischer@stackable.tech>

Author: maltesander

demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml

@@ -8,7 +8,10 @@ spec:
     spec:
       containers:
         - name: distcp-cycling-data
-          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.7.0
+          # We use 24.3.0 here which contains the distcp MapReduce components
+          # This is not included in the 24.7 images and will fail.
+          # See: https://github.com/stackabletech/docker-images/issues/793
+          image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.3.0
           env:
             - name: HADOOP_USER_NAME
               value: stackable

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile

@@ -1,6 +1,6 @@
-# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.3.0-stackable0.0.0-dev .
+# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.4.0-stackable0.0.0-dev .
 
-FROM docker.stackable.tech/stackable/pyspark-k8s:3.5.0-stackable24.7.0
+FROM docker.stackable.tech/stackable/pyspark-k8s:3.4.0-stackable23.7.0
 
 COPY requirements.txt .
 

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: load-ny-taxi-data
-          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.7.0
+          image: docker.stackable.tech/stackable/hadoop:3.3.4-stackable24.7.0
           command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \
           && cd /tmp \
           && for month in 2020-09; do \

stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml

@@ -13,7 +13,7 @@ metadata:
   name: kafka
 spec:
   image:
-    productVersion: 3.6.1
+    productVersion: 3.7.1
   clusterConfig:
     zookeeperConfigMapName: kafka-znode
     authentication:

stacks/dual-hive-hdfs-s3/hdfs.yaml

@@ -25,7 +25,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.4.0
+    productVersion: 3.3.4
   clusterConfig:
     listenerClass: external-unstable
     dfsReplication: 1

stacks/end-to-end-security/hdfs.yaml

@@ -4,7 +4,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.4.0
+    productVersion: 3.3.4
   clusterConfig:
     zookeeperConfigMapName: hdfs-znode
     authentication:

stacks/hdfs-hbase/hdfs.yaml

@@ -4,7 +4,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.4.0
+    productVersion: 3.3.4
   clusterConfig:
     dfsReplication: 1
     zookeeperConfigMapName: hdfs-znode

stacks/jupyterhub-pyspark-hdfs/hdfs.yaml

@@ -13,7 +13,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.4.0
+    productVersion: 3.3.4
   clusterConfig:
     dfsReplication: 1
     zookeeperConfigMapName: hdfs-znode

stacks/keycloak-opa-poc/hdfs.yaml

@@ -5,7 +5,7 @@ metadata:
   name: hdfs
 spec:
   image:
-    productVersion: 3.4.0
+    productVersion: 3.3.4
   clusterConfig:
     dfsReplication: 1
     zookeeperConfigMapName: hdfs-znode