From f57e4a0a73a379dfb45b1ff88b285d0efd2ea8ca Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 11:13:41 +0200 Subject: [PATCH 01/24] dummy change on index --- modules/ROOT/pages/index.adoc | 248 +--------------------------------- 1 file changed, 1 insertion(+), 247 deletions(-) diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index cbd1a4b1d..c94406d03 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -1,250 +1,4 @@ = Stackable Documentation :page-layout: landing -Welcome to Stackable! -This documentation gives you an overview of the Stackable Data Platform, how to install and manage it as well as some tutorials. - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Introduction

-++++ - -The Stackable Data Platform allows you to deploy, scale and manage Data infrastructure in any environment running https://kubernetes.io/[Kubernetes]. - -You can find an overview of the supported components <>, as well as a full list of all supported product versions xref:operators:supported_versions.adoc[here]. - -If you have any feedback regarding the documentation please either open an https://github.com/stackabletech/documentation/issues[issue], ask a https://github.com/stackabletech/documentation/discussions[question] or look at the source for this documentation in its https://github.com/stackabletech/documentation[repository]. - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Goal of the project

-++++ - -We are building a distribution of existing Open Source tools that together comprise the components of a modern data platform. - -There are components to ingest data, to store data, to process and visualize and much more. -While the platform got started in the _Big Data_ ecosystem we are in no way limited to big data workloads. - -You can declaratively build these environments, and we don't stop at the tool level as we also provide ways for the users to interact with the platform in the "as Code"-approach. - -We are leveraging the https://www.openpolicyagent.org/[Open Policy Agent] to provide Security-as-Code. - -We are building a distribution that includes the “best of breed” of existing Open Source tools, but bundles them in a way, so it is easy to deploy a fully working stack of software. Most of the existing tools are “single purpose” tools, which often do not play nicely together out-of-the-box. - -++++ -
-++++ - -++++ -
-++++ - -== Components - -We are using Kubernetes as our deployment platform. -And we're building https://kubernetes.io/docs/concepts/extend-kubernetes/operator/[Operators] for each of the products we support. - -The Stackable Data Platform supports the following products: - -++++ -
-++++ - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache Airflow

-++++ - -Airflow is a workflow engine and your replacement should you be using Apache Oozie. - -xref:airflow::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache Druid

-++++ - -Apache Druid is a real-time database to power modern analytics applications. - -xref:druid::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache HBase

-++++ - -HBase is a distributed, scalable, big data store. - -xref:hbase::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache Hadoop HDFS

-++++ - -HDFS is a distributed file system that provides high-throughput access to application data. - -xref:hdfs::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache Hive

-++++ - -The Apache Hive data warehouse software facilitates reading, writing, and managing large datasets residing in distributed storage using SQL. We support the Hive Metastore. - -xref:hive::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache Kafka

-++++ - -Apache Kafka is an open-source distributed event streaming platform used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications. - -xref:kafka::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache NiFi

-++++ - -An easy to use, powerful, and reliable system to process and distribute data. - -xref:nifi::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache Spark

-++++ - -Apache Spark is a multi-language engine for executing data engineering, data science, and machine learning on single-node machines or clusters. - -xref:spark-k8s::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache Superset

-++++ - -Apache Superset is a modern data exploration and visualization platform. - -xref:superset::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Trino

-++++ - -Fast distributed SQL query engine for big data analytics that helps you explore your data universe. - -xref:trino::index.adoc[Read more] - -++++ -
-++++ - -++++ -
-++++ - -++++ -

Apache ZooKeeper

-++++ - -ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. - -xref:zookeeper::index.adoc[Read more] - -++++ -
-++++ - - -++++ -
-++++ +Hello world!! \ No newline at end of file From cc91ebb1b3e1cf523de1dc9b350c7ff902dd6373 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 11:20:13 +0200 Subject: [PATCH 02/24] maybe fixed it --- antora-playbook.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/antora-playbook.yml b/antora-playbook.yml index 6521fea91..44bd59447 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -15,7 +15,8 @@ urls: content: sources: - - url: https://github.com/stackabletech/documentation.git + - url: . + worktrees: false branches: - HEAD - release/23.4 From 5ab8b1d1560d70d9bc7aaedbbd196c0dd81c1d78 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 11:31:54 +0200 Subject: [PATCH 03/24] maybe fixed it now? --- antora-playbook.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/antora-playbook.yml b/antora-playbook.yml index 44bd59447..59a26128e 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -16,7 +16,6 @@ urls: content: sources: - url: . - worktrees: false branches: - HEAD - release/23.4 From 9eb664c9c513503776fc7f980d1a9ce32e0ce2f7 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 12:16:41 +0200 Subject: [PATCH 04/24] maybe fixed it now??? --- netlify.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/netlify.toml b/netlify.toml index 8adf31c96..b66a6e441 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,3 +1,7 @@ +[build] + command = "git checkout . && for remote in `git branch -r | grep -v /HEAD`; do git checkout --track $remote ; done && make ANTORAFLAGS=--fetch PLAYBOOK=antora-playbook.yml" + publish = "build/site" + [[redirects]] from = "/airflow/nightly/*" to = "/home/nightly/airflow/:splat" From b550053fa88135577c1c5d4a90daa72ba3021589 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 14:36:34 +0200 Subject: [PATCH 05/24] ... --- netlify.toml | 2 +- scripts/netlify_git_fetch.sh | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 scripts/netlify_git_fetch.sh diff --git a/netlify.toml b/netlify.toml index b66a6e441..c469c9470 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,5 +1,5 @@ [build] - command = "git checkout . && for remote in `git branch -r | grep -v /HEAD`; do git checkout --track $remote ; done && make ANTORAFLAGS=--fetch PLAYBOOK=antora-playbook.yml" + command = "bash scripts/netlify_git_fetch.sh && make ANTORAFLAGS=--fetch PLAYBOOK=antora-playbook.yml" publish = "build/site" [[redirects]] diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh new file mode 100644 index 000000000..ae52e50d6 --- /dev/null +++ b/scripts/netlify_git_fetch.sh @@ -0,0 +1,15 @@ +#!/usr/bin/bash + +current_branch=$(git branch --show-current) + +git fetch --all + +git stash + +for remote in $(git branch -r | grep release/); do + git switch "${remote#origin/}" +done + +git switch "$current_branch" + +git stash pop From 7393ac6f0d238b92c204650cbc7e298f7c1373e2 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 14:42:27 +0200 Subject: [PATCH 06/24] ... --- scripts/netlify_git_fetch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index ae52e50d6..851a0b75d 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -1,6 +1,6 @@ #!/usr/bin/bash -current_branch=$(git branch --show-current) +current_commit=$(git rev-parse HEAD) git fetch --all @@ -10,6 +10,6 @@ for remote in $(git branch -r | grep release/); do git switch "${remote#origin/}" done -git switch "$current_branch" +git checkout "$current_commit" git stash pop From 0fe1fd66b3ef24aabf4978cd17a1719594e88f9a Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 14:47:52 +0200 Subject: [PATCH 07/24] ... --- scripts/netlify_git_fetch.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 851a0b75d..4fca16e4e 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -1,15 +1,19 @@ #!/usr/bin/bash +# netlify messes with some files, restore everything to how it was +git restore . + +# save current commit for later current_commit=$(git rev-parse HEAD) +# update git fetch --all -git stash - +# checkout all release branches once, so we fetch the files for remote in $(git branch -r | grep release/); do git switch "${remote#origin/}" done +# go back to the initial commit to start the build git checkout "$current_commit" -git stash pop From 56665d5d345f04fafabb6472608b364ecf89fa4c Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:01:31 +0200 Subject: [PATCH 08/24] ... --- scripts/netlify_git_fetch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 4fca16e4e..4216be23e 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -2,6 +2,7 @@ # netlify messes with some files, restore everything to how it was git restore . +git submodule foreach git reset --hard # save current commit for later current_commit=$(git rev-parse HEAD) From 6d0e63fd4523327b2247552d265f2ffaf9daa831 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:05:16 +0200 Subject: [PATCH 09/24] ... --- scripts/netlify_git_fetch.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 4216be23e..47f11eb84 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -1,8 +1,7 @@ #!/usr/bin/bash # netlify messes with some files, restore everything to how it was -git restore . -git submodule foreach git reset --hard +git reset --hard --recurse-submodule # save current commit for later current_commit=$(git rev-parse HEAD) From fbf85a58b4fdbd7f09b928115118c17d5d432ca7 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:16:30 +0200 Subject: [PATCH 10/24] ... --- scripts/netlify_git_fetch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 47f11eb84..37fe57f4a 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -15,5 +15,5 @@ for remote in $(git branch -r | grep release/); do done # go back to the initial commit to start the build -git checkout "$current_commit" +git checkout -c advice.detachedHead=false "$current_commit" From 68071c5b8964dd95149c3e830393ca2dd8141f07 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:17:30 +0200 Subject: [PATCH 11/24] ... --- scripts/netlify_git_fetch.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 37fe57f4a..31b858965 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -3,6 +3,11 @@ # netlify messes with some files, restore everything to how it was git reset --hard --recurse-submodule +cd ui + +git diff +cd - + # save current commit for later current_commit=$(git rev-parse HEAD) From 7d9d31e818fbce01a94ebc9e5db0566d93e0f765 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:20:17 +0200 Subject: [PATCH 12/24] ... --- scripts/netlify_git_fetch.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 31b858965..3981554ea 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -1,10 +1,12 @@ #!/usr/bin/bash # netlify messes with some files, restore everything to how it was +echo "reset" git reset --hard --recurse-submodule cd ui +echo "diff" git diff cd - @@ -20,5 +22,5 @@ for remote in $(git branch -r | grep release/); do done # go back to the initial commit to start the build -git checkout -c advice.detachedHead=false "$current_commit" +git -c advice.detachedHead=false checkout "$current_commit" From 56a4e8460d6899008bdf135a538af20f1c4f814e Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:23:35 +0200 Subject: [PATCH 13/24] ... --- scripts/netlify_git_fetch.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 3981554ea..389efd329 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -1,5 +1,8 @@ #!/usr/bin/bash +# init submodule +git submodule update --init --recursive + # netlify messes with some files, restore everything to how it was echo "reset" git reset --hard --recurse-submodule From 00258d98132a80b11cf7c993feea104ea5e1de31 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:27:27 +0200 Subject: [PATCH 14/24] ... --- scripts/netlify_git_fetch.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 389efd329..daf0c2ed1 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -7,10 +7,14 @@ git submodule update --init --recursive echo "reset" git reset --hard --recurse-submodule +echo "diffff" +git diff + cd ui echo "diff" git diff +git clean cd - # save current commit for later From 187abc925e6f943337fb55508546932785f6cc15 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:35:13 +0200 Subject: [PATCH 15/24] ... --- scripts/netlify_git_fetch.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index daf0c2ed1..9ba3bfdc1 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -4,19 +4,8 @@ git submodule update --init --recursive # netlify messes with some files, restore everything to how it was -echo "reset" git reset --hard --recurse-submodule -echo "diffff" -git diff - -cd ui - -echo "diff" -git diff -git clean -cd - - # save current commit for later current_commit=$(git rev-parse HEAD) @@ -25,7 +14,7 @@ git fetch --all # checkout all release branches once, so we fetch the files for remote in $(git branch -r | grep release/); do - git switch "${remote#origin/}" + git checkout --recurse-submodules "${remote#origin/}" done # go back to the initial commit to start the build From 647818b6f2d3d76e2e5db10fcc64e3831ef57579 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:39:23 +0200 Subject: [PATCH 16/24] ... --- scripts/netlify_git_fetch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 9ba3bfdc1..7be91bc1c 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -18,5 +18,5 @@ for remote in $(git branch -r | grep release/); do done # go back to the initial commit to start the build -git -c advice.detachedHead=false checkout "$current_commit" +git -c advice.detachedHead=false checkout --recurse-submodules "$current_commit" From fe3e76e5169f98eccdb2ae4aad345670c4ab5979 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:43:54 +0200 Subject: [PATCH 17/24] added documentation --- scripts/netlify_git_fetch.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh index 7be91bc1c..23588da2a 100644 --- a/scripts/netlify_git_fetch.sh +++ b/scripts/netlify_git_fetch.sh @@ -1,4 +1,9 @@ #!/usr/bin/bash +# The netlify repo is checked out without any blobs. This script +# iterates through the release branches and checks them out one-by-one +# to fetch all the files. +# Then we can build directly from here, making it possible to build +# with antora using the HEAD (enabling branch previews!) # init submodule git submodule update --init --recursive From f6634593f36afe486d3f23cbf3d237459a41e5fc Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 15:45:12 +0200 Subject: [PATCH 18/24] reverted index change --- modules/ROOT/pages/index.adoc | 248 +++++++++++++++++++++++++++++++++- 1 file changed, 247 insertions(+), 1 deletion(-) diff --git a/modules/ROOT/pages/index.adoc b/modules/ROOT/pages/index.adoc index c94406d03..cbd1a4b1d 100644 --- a/modules/ROOT/pages/index.adoc +++ b/modules/ROOT/pages/index.adoc @@ -1,4 +1,250 @@ = Stackable Documentation :page-layout: landing -Hello world!! \ No newline at end of file +Welcome to Stackable! +This documentation gives you an overview of the Stackable Data Platform, how to install and manage it as well as some tutorials. + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Introduction

+++++ + +The Stackable Data Platform allows you to deploy, scale and manage Data infrastructure in any environment running https://kubernetes.io/[Kubernetes]. + +You can find an overview of the supported components <>, as well as a full list of all supported product versions xref:operators:supported_versions.adoc[here]. + +If you have any feedback regarding the documentation please either open an https://github.com/stackabletech/documentation/issues[issue], ask a https://github.com/stackabletech/documentation/discussions[question] or look at the source for this documentation in its https://github.com/stackabletech/documentation[repository]. + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Goal of the project

+++++ + +We are building a distribution of existing Open Source tools that together comprise the components of a modern data platform. + +There are components to ingest data, to store data, to process and visualize and much more. +While the platform got started in the _Big Data_ ecosystem we are in no way limited to big data workloads. + +You can declaratively build these environments, and we don't stop at the tool level as we also provide ways for the users to interact with the platform in the "as Code"-approach. + +We are leveraging the https://www.openpolicyagent.org/[Open Policy Agent] to provide Security-as-Code. + +We are building a distribution that includes the “best of breed” of existing Open Source tools, but bundles them in a way, so it is easy to deploy a fully working stack of software. Most of the existing tools are “single purpose” tools, which often do not play nicely together out-of-the-box. + +++++ +
+++++ + +++++ +
+++++ + +== Components + +We are using Kubernetes as our deployment platform. +And we're building https://kubernetes.io/docs/concepts/extend-kubernetes/operator/[Operators] for each of the products we support. + +The Stackable Data Platform supports the following products: + +++++ +
+++++ + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache Airflow

+++++ + +Airflow is a workflow engine and your replacement should you be using Apache Oozie. + +xref:airflow::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache Druid

+++++ + +Apache Druid is a real-time database to power modern analytics applications. + +xref:druid::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache HBase

+++++ + +HBase is a distributed, scalable, big data store. + +xref:hbase::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache Hadoop HDFS

+++++ + +HDFS is a distributed file system that provides high-throughput access to application data. + +xref:hdfs::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache Hive

+++++ + +The Apache Hive data warehouse software facilitates reading, writing, and managing large datasets residing in distributed storage using SQL. We support the Hive Metastore. + +xref:hive::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache Kafka

+++++ + +Apache Kafka is an open-source distributed event streaming platform used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications. + +xref:kafka::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache NiFi

+++++ + +An easy to use, powerful, and reliable system to process and distribute data. + +xref:nifi::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache Spark

+++++ + +Apache Spark is a multi-language engine for executing data engineering, data science, and machine learning on single-node machines or clusters. + +xref:spark-k8s::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache Superset

+++++ + +Apache Superset is a modern data exploration and visualization platform. + +xref:superset::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Trino

+++++ + +Fast distributed SQL query engine for big data analytics that helps you explore your data universe. + +xref:trino::index.adoc[Read more] + +++++ +
+++++ + +++++ +
+++++ + +++++ +

Apache ZooKeeper

+++++ + +ZooKeeper is a centralized service for maintaining configuration information, naming, providing distributed synchronization, and providing group services. + +xref:zookeeper::index.adoc[Read more] + +++++ +
+++++ + + +++++ +
+++++ From f8354b396df56a03c83ec70f66c5bb13449cc8aa Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 16:09:08 +0200 Subject: [PATCH 19/24] moved script to makefile --- Makefile | 24 ++++++++++++++++++++++++ netlify.toml | 2 +- scripts/netlify_git_fetch.sh | 27 --------------------------- 3 files changed, 25 insertions(+), 28 deletions(-) delete mode 100644 scripts/netlify_git_fetch.sh diff --git a/Makefile b/Makefile index ec78921fa..be7b01af6 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,35 @@ PLAYBOOK := local-antora-playbook.yml ANTORAFLAGS := +CURRENT_COMMIT := $(shell git rev-parse HEAD) + + +# The netlify repo is checked out without any blobs. This script +# iterates through the release branches and checks them out one-by-one +# to fetch all the files. +# Then we can build directly from here, making it possible to build +# with antora using the HEAD (enabling branch previews!) +netlify-fetch: + git submodule update --init --recursive + # netlify messes with some files, restore everything to how it was: + git reset --hard --recurse-submodule + # fetch, because netlify does caching and we want to get the latest commits + git fetch --all + # checkout all release branches once, so we fetch the files + for remote in $(git branch -r | grep release/); do \ + git checkout --recurse-submodules "${remote#origin/}" ;\ + done + # go back to the initial commit to start the build + git -c advice.detachedHead=false checkout --recurse-submodules "$current_commit" +.PHONY: netlify-fetch + build: node_modules/.bin/gulp --cwd ui bundle node_modules/.bin/antora generate $(PLAYBOOK) $(ANTORAFLAGS) .PHONY: clean + + clean: rm -r build # 'cache' is the configured cache dir in the playbook diff --git a/netlify.toml b/netlify.toml index c469c9470..0eeac26ac 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,5 +1,5 @@ [build] - command = "bash scripts/netlify_git_fetch.sh && make ANTORAFLAGS=--fetch PLAYBOOK=antora-playbook.yml" + command = "make netlify-fetch && make ANTORAFLAGS=--fetch PLAYBOOK=antora-playbook.yml" publish = "build/site" [[redirects]] diff --git a/scripts/netlify_git_fetch.sh b/scripts/netlify_git_fetch.sh deleted file mode 100644 index 23588da2a..000000000 --- a/scripts/netlify_git_fetch.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/bash -# The netlify repo is checked out without any blobs. This script -# iterates through the release branches and checks them out one-by-one -# to fetch all the files. -# Then we can build directly from here, making it possible to build -# with antora using the HEAD (enabling branch previews!) - -# init submodule -git submodule update --init --recursive - -# netlify messes with some files, restore everything to how it was -git reset --hard --recurse-submodule - -# save current commit for later -current_commit=$(git rev-parse HEAD) - -# update -git fetch --all - -# checkout all release branches once, so we fetch the files -for remote in $(git branch -r | grep release/); do - git checkout --recurse-submodules "${remote#origin/}" -done - -# go back to the initial commit to start the build -git -c advice.detachedHead=false checkout --recurse-submodules "$current_commit" - From d729e91276569d343ae0d892364eb8469218dfad Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 16:13:30 +0200 Subject: [PATCH 20/24] fixes --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index be7b01af6..ccc08c8b3 100644 --- a/Makefile +++ b/Makefile @@ -15,11 +15,11 @@ netlify-fetch: # fetch, because netlify does caching and we want to get the latest commits git fetch --all # checkout all release branches once, so we fetch the files - for remote in $(git branch -r | grep release/); do \ + for remote in $(shell git branch -r | grep release/); do \ git checkout --recurse-submodules "${remote#origin/}" ;\ done # go back to the initial commit to start the build - git -c advice.detachedHead=false checkout --recurse-submodules "$current_commit" + git -c advice.detachedHead=false checkout --recurse-submodules $(CURRENT_COMMIT) .PHONY: netlify-fetch From 5c2d21d8c8508695a3c0754466b309f473c3ca28 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 16:16:39 +0200 Subject: [PATCH 21/24] fixes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ccc08c8b3..5c3bc7862 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ netlify-fetch: git fetch --all # checkout all release branches once, so we fetch the files for remote in $(shell git branch -r | grep release/); do \ - git checkout --recurse-submodules "${remote#origin/}" ;\ + git checkout --recurse-submodules "$${remote#origin/}" ;\ done # go back to the initial commit to start the build git -c advice.detachedHead=false checkout --recurse-submodules $(CURRENT_COMMIT) From 2a1cd71451e69ba92e3261a9b71a796bf30534b9 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 16:28:08 +0200 Subject: [PATCH 22/24] fixes --- Makefile | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 5c3bc7862..ec4a3e89f 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,20 @@ ANTORAFLAGS := CURRENT_COMMIT := $(shell git rev-parse HEAD) +build: + node_modules/.bin/gulp --cwd ui bundle + node_modules/.bin/antora generate $(PLAYBOOK) $(ANTORAFLAGS) + +.PHONY: build + + +clean: + rm -r build + # 'cache' is the configured cache dir in the playbook + rm -r cache +.PHONY: clean + + # The netlify repo is checked out without any blobs. This script # iterates through the release branches and checks them out one-by-one # to fetch all the files. @@ -21,18 +35,3 @@ netlify-fetch: # go back to the initial commit to start the build git -c advice.detachedHead=false checkout --recurse-submodules $(CURRENT_COMMIT) .PHONY: netlify-fetch - - -build: - node_modules/.bin/gulp --cwd ui bundle - node_modules/.bin/antora generate $(PLAYBOOK) $(ANTORAFLAGS) - -.PHONY: clean - - -clean: - rm -r build - # 'cache' is the configured cache dir in the playbook - rm -r cache - -.PHONY: build From f1f261d179e9c1f2092720296e867978e3fba056 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 16:50:35 +0200 Subject: [PATCH 23/24] .. --- Makefile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index ec4a3e89f..ff78a700a 100644 --- a/Makefile +++ b/Makefile @@ -7,15 +7,10 @@ build: node_modules/.bin/gulp --cwd ui bundle node_modules/.bin/antora generate $(PLAYBOOK) $(ANTORAFLAGS) -.PHONY: build - - clean: rm -r build # 'cache' is the configured cache dir in the playbook rm -r cache -.PHONY: clean - # The netlify repo is checked out without any blobs. This script # iterates through the release branches and checks them out one-by-one @@ -34,4 +29,7 @@ netlify-fetch: done # go back to the initial commit to start the build git -c advice.detachedHead=false checkout --recurse-submodules $(CURRENT_COMMIT) -.PHONY: netlify-fetch + +netlify-build: netlify-fetch build + +.PHONY: build clean netlify-fetch From 13927b9b4254b5924d08692bb7ba63ed99957435 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Mon, 24 Apr 2023 16:55:13 +0200 Subject: [PATCH 24/24] reorganized makefile --- Makefile | 14 ++++++++------ netlify.toml | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ff78a700a..d9f3bb686 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,13 @@ -PLAYBOOK := local-antora-playbook.yml -ANTORAFLAGS := CURRENT_COMMIT := $(shell git rev-parse HEAD) +build-dev: build-ui + node_modules/.bin/antora generate local-antora-playbook.yml -build: +build-prod: build-ui + node_modules/.bin/antora generate antora-playbook.yml --fetch + +build-ui: node_modules/.bin/gulp --cwd ui bundle - node_modules/.bin/antora generate $(PLAYBOOK) $(ANTORAFLAGS) clean: rm -r build @@ -30,6 +32,6 @@ netlify-fetch: # go back to the initial commit to start the build git -c advice.detachedHead=false checkout --recurse-submodules $(CURRENT_COMMIT) -netlify-build: netlify-fetch build +netlify-build: netlify-fetch build-prod -.PHONY: build clean netlify-fetch +.PHONY: build-dev build-prod build-ui clean netlify-build netlify-fetch diff --git a/netlify.toml b/netlify.toml index 0eeac26ac..acd39ef84 100644 --- a/netlify.toml +++ b/netlify.toml @@ -1,5 +1,5 @@ [build] - command = "make netlify-fetch && make ANTORAFLAGS=--fetch PLAYBOOK=antora-playbook.yml" + command = "make netlify-build" publish = "build/site" [[redirects]]