Build system overhaul

.github/workflows/cron.yml

@@ -0,0 +1,39 @@
+name: Cron Build
+
+on: 
+  schedule:
+#             ┌───────────── minute (0 - 59)
+#             │ ┌───────────── hour (0 - 23)
+#             │ │ ┌───────────── day of the month (1 - 31)
+#             │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
+#             │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
+#             │ │ │ │ │                                   
+#             │ │ │ │ │
+#             │ │ │ │ │
+#             * * * * *
+    - cron:  '0 */6 * * *'
+
+jobs:
+  run:
+    name: Build with IDF ${{ matrix.idf_branch }}
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        idf_branch: [release/v3.2]
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install dependencies
+      run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
+    - name: Install Python Wheel
+      run: pip install wheel
+    - name: Build
+      env:
+        GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+        IDF_BRANCH: ${{ matrix.idf_branch }}
+      run: bash ./tools/cron.sh
+    - name: Upload archive
+      uses: actions/upload-artifact@v1
+      with:
+        name: artifacts
+        path: dist

.github/workflows/main.yml → .github/workflows/push.yml

@@ -6,7 +6,6 @@ on:
     - master
     - release/*
   pull_request:
-  repository_dispatch:
 
 jobs:
 
@@ -20,11 +19,9 @@ jobs:
     - name: Install Python Wheel
       run: pip install wheel
     - name: Build Arduino Libs
-      env:
-        TRAVIS_BUILD_DIR: ${{ github.workspace }}
-      run: bash $TRAVIS_BUILD_DIR/build.sh
-    - name: Archive libs
+      run: bash ./build.sh
+    - name: Upload archive
       uses: actions/upload-artifact@v1
       with:
-        name: arduino-libs
+        name: artifacts
         path: dist

.github/workflows/repository_dispatch.yml

@@ -0,0 +1,23 @@
+name: Remote Trigger
+
+on: repository_dispatch
+
+jobs:
+  run:
+    name: Dispatch Event
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - name: Install dependencies
+      run: sudo apt-get install git wget curl libssl-dev libncurses-dev flex bison gperf python python-pip python-setuptools python-serial python-click python-cryptography python-future python-pyparsing python-pyelftools cmake ninja-build ccache
+    - name: Install Python Wheel
+      run: pip install wheel
+    - name: Handle Event
+      env:
+        GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+      run: bash ./tools/repository_dispatch.sh
+    - name: Upload archive
+      uses: actions/upload-artifact@v1
+      with:
+        name: artifacts
+        path: dist

build.sh

@@ -35,14 +35,16 @@ if ! [ -x "$(command -v stat)" ]; then
   	exit 1
 fi
 
-# install esp-idf and gcc toolchain
-source ./tools/install-esp-idf.sh
-if [ $? -ne 0 ]; then exit 1; fi
+mkdir -p dist
 
 # update components from git
 ./tools/update-components.sh
 if [ $? -ne 0 ]; then exit 1; fi
 
+# install esp-idf and gcc toolchain
+source ./tools/install-esp-idf.sh
+if [ $? -ne 0 ]; then exit 1; fi
+
 # build and prepare libs
 ./tools/build-libs.sh
 if [ $? -ne 0 ]; then exit 1; fi

tools/archive-build.sh

@@ -1,10 +1,16 @@
 #!/bin/bash
 
+IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
+IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD)
+
 idf_version_string=${IDF_BRANCH//\//_}"-$IDF_COMMIT"
 archive_path="dist/arduino-esp32-libs-$idf_version_string.tar.gz"
+build_archive_path="dist/arduino-esp32-build-$idf_version_string.tar.gz"
 
-mkdir -p dist && \
-rm -rf $archive_path && \
-cd out && \
-tar zcf ../$archive_path * \
-&& cd ..
+mkdir -p dist && rm -rf "$archive_path" "$build_archive_path"
+if [ -d "out" ]; then
+	cd out && tar zcf "../$archive_path" * && cd ..
+fi
+if [ -d "build" ]; then
+	cd build && tar zcf "../$build_archive_path" * && cd ..
+fi

tools/config.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-IDF_REPO="https://github.com/espressif/esp-idf.git"
-IDF_BRANCH="release/v3.2"
+
 IDF_COMPS="$IDF_PATH/components"
 IDF_TOOLCHAIN="xtensa-esp32-elf"
 IDF_TOOLCHAIN_LINUX_ARMEL="https://dl.espressif.com/dl/xtensa-esp32-elf-linux-armel-1.22.0-87-gb57bad3-5.2.0.tar.gz"
@@ -9,10 +8,25 @@ IDF_TOOLCHAIN_LINUX64="https://dl.espressif.com/dl/xtensa-esp32-elf-linux64-1.22
 IDF_TOOLCHAIN_WIN32="https://dl.espressif.com/dl/xtensa-esp32-elf-win32-1.22.0-80-g6c4433a-5.2.0.zip"
 IDF_TOOLCHAIN_MACOS="https://dl.espressif.com/dl/xtensa-esp32-elf-osx-1.22.0-80-g6c4433a-5.2.0.tar.gz"
 
-CAMERA_REPO="https://github.com/espressif/esp32-camera.git"
-FACE_REPO="https://github.com/espressif/esp-face.git"
+if [ -z $IDF_BRANCH ]; then
+	IDF_BRANCH="release/v3.2"
+fi
+
+# Owner of the target ESP32 Arduino repository
+AR_USER="espressif"
+
+# The full name of the repository
+AR_REPO="$AR_USER/arduino-esp32"
+
+IDF_REPO_URL="https://github.com/espressif/esp-idf.git"
+CAMERA_REPO_URL="https://github.com/espressif/esp32-camera.git"
+FACE_REPO_URL="https://github.com/espressif/esp-face.git"
+AR_REPO_URL="https://github.com/$AR_REPO.git"
+
+if [ -n $GITHUB_TOKEN ]; then
+	AR_REPO_URL="https://$GITHUB_TOKEN@github.com/$AR_REPO.git"
+fi
 
-AR_REPO="https://github.com/espressif/arduino-esp32.git"
 AR_ROOT="$PWD"
 AR_COMPS="$AR_ROOT/components"
 AR_OUT="$AR_ROOT/out"
@@ -56,3 +70,38 @@ if [[ "$AR_OS" == "macos" ]]; then
 	export SED="gsed"
 	export SSTAT="stat -f %z"
 fi
+
+function git_commit_exists(){ #git_commit_exists <repo-path> <commit-message>
+	local repo_path="$1"
+	local commit_message="$2"
+	local commits_found=`git -C "$repo_path" log --all --grep="$commit_message" | grep commit`
+	if [ -n "$commits_found" ]; then echo 1; else echo 0; fi
+}
+
+function git_branch_exists(){ # git_branch_exists <repo-path> <branch-name>
+	local repo_path="$1"
+	local branch_name="$2"
+	local branch_found=`git -C "$repo_path" ls-remote --heads origin "$branch_name"`
+	if [ -n "$branch_found" ]; then echo 1; else echo 0; fi
+}
+
+function git_pr_exists(){ # git_pr_exists <branch-name>
+	local pr_num=`curl -s -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" "https://api.github.com/repos/$AR_REPO/pulls?head=$AR_USER:$1&state=open" | jq -r '.[].number'`
+	if [ ! "$pr_num" == "" ] && [ ! "$pr_num" == "null" ]; then echo 1; else echo 0; fi
+}
+
+function git_create_pr(){ # git_create_pr <branch> <title>
+	local pr_branch="$1"
+	local pr_title="$2"
+	local pr_body=""
+	for component in `ls "$AR_COMPS"`; do
+		if [ ! $component == "arduino" ] && [ -d "$AR_COMPS/$component/.git" ]; then
+			pr_body+="$component: "$(git -C "$AR_COMPS/$component" symbolic-ref --short HEAD)" "$(git -C "$AR_COMPS/$component" rev-parse --short HEAD)"\r\n"
+		fi
+	done
+	local pr_data="{\"title\": \"$pr_title\", \"body\": \"$pr_body\", \"head\": \"$AR_USER:$pr_branch\", \"base\": \"master\"}"
+	git_create_pr_res=`echo "$pr_data" | curl -k -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3.raw+json" --data @- "https://api.github.com/repos/$AR_REPO/pulls"`
+	local done_pr=`echo "$git_create_pr_res" | jq -r '.title'`
+	if [ ! "$done_pr" == "" ] && [ ! "$done_pr" == "null" ]; then echo 1; else echo 0; fi
+}
+

tools/cron.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+if [ ! "$GITHUB_EVENT_NAME" == "schedule" ]; then
+    echo "Wrong event '$GITHUB_EVENT_NAME'!"
+    exit 1
+fi
+
+git checkout "$IDF_BRANCH" #local branches should match what the matrix wants to build
+source ./build.sh
+bash ./tools/push-to-arduino.sh

tools/install-esp-idf.sh

@@ -7,26 +7,102 @@ if ! [ -x "$(command -v $SED)" ]; then
   	exit 1
 fi
 
+#
+# CLONE ESP-IDF
+#
+
 if [ -z "$IDF_PATH" ]; then
 	echo "ESP-IDF is not installed! Installing local copy"
+	idf_was_installed="1"
 	if ! [ -d esp-idf ]; then
-		git clone $IDF_REPO -b $IDF_BRANCH
+		git clone $IDF_REPO_URL -b $IDF_BRANCH
 	fi
 	export IDF_PATH="$AR_ROOT/esp-idf"
-	cd $IDF_PATH
-	git fetch origin && git pull origin $IDF_BRANCH
-	git submodule update --init --recursive
-	python -m pip install -r requirements.txt
-	cd "$AR_ROOT"
 fi
 
 if [ "$IDF_COMMIT" ]; then
-    git -C $IDF_PATH checkout $IDF_COMMIT
-    git -C $IDF_PATH submodule update
+    git -C "$IDF_PATH" checkout "$IDF_COMMIT"
+    commit_predefined="1"
+fi
+
+export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
+export IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD)
+
+#
+# SETUP ARDUINO DEPLOY
+#
+
+if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "repository_dispatch" -a "$GITHUB_EVENT_ACTION" == "deploy" ]; then
+	# format new branch name and pr title
+	if [ -x $commit_predefined ]; then #commit was not specified at build time
+		AR_NEW_BRANCH_NAME="idf-$IDF_BRANCH"
+		AR_NEW_COMMIT_MESSAGE="IDF $IDF_BRANCH $IDF_COMMIT"
+		AR_NEW_PR_TITLE="IDF $IDF_BRANCH"
+	else
+		AR_NEW_BRANCH_NAME="idf-$IDF_COMMIT"
+		AR_NEW_COMMIT_MESSAGE="IDF $IDF_COMMIT"
+		AR_NEW_PR_TITLE="$AR_NEW_COMMIT_MESSAGE"
+	fi
+
+	AR_HAS_COMMIT=`git_commit_exists "$AR_COMPS/arduino" "$AR_NEW_COMMIT_MESSAGE"`
+	AR_HAS_BRANCH=`git_branch_exists "$AR_COMPS/arduino" "$AR_NEW_BRANCH_NAME"`
+	AR_HAS_PR=`git_pr_exists "$AR_NEW_BRANCH_NAME"`
+
+	if [ "$AR_HAS_COMMIT" == "1" ]; then
+		echo "Commit '$AR_NEW_COMMIT_MESSAGE' Already Exists"
+		exit 0
+	fi
+
+	if [ "$AR_HAS_BRANCH" == "1" ]; then
+		echo "Branch '$AR_NEW_BRANCH_NAME' Already Exists"
+	fi
+
+	if [ "$AR_HAS_PR" == "1" ]; then
+		echo "PR '$AR_NEW_PR_TITLE' Already Exists"
+	fi
+
+	# setup git for pushing
+	git config --global github.user "$GITHUB_ACTOR"
+	git config --global user.name "$GITHUB_ACTOR"
+	git config --global user.email "$GITHUB_ACTOR@github.com"
+
+	# create or checkout the branch
+	if [ ! $AR_HAS_BRANCH == "0" ]; then
+		echo "Switching to arduino branch '$AR_NEW_BRANCH_NAME'..."
+		git -C "$AR_COMPS/arduino" checkout $AR_NEW_BRANCH_NAME
+	else
+		echo "Creating arduino branch '$AR_NEW_BRANCH_NAME'..."
+		git -C "$AR_COMPS/arduino" checkout -b $AR_NEW_BRANCH_NAME
+	fi
+	if [ $? -ne 0 ]; then
+	    echo "ERROR: Checkout of branch '$AR_NEW_BRANCH_NAME' failed"
+		exit 1
+	fi
+
+	export AR_NEW_BRANCH_NAME
+	export AR_NEW_COMMIT_MESSAGE
+	export AR_NEW_PR_TITLE
+
+	export AR_HAS_COMMIT
+	export AR_HAS_BRANCH
+	export AR_HAS_PR
+fi
+
+#
+# UPDATE IDF MODULES
+#
+
+if [ -x $idf_was_installed ]; then
+	git -C $IDF_PATH fetch origin && git -C $IDF_PATH pull origin $IDF_BRANCH
+	git -C $IDF_PATH submodule update --init --recursive
+else
+	git -C $IDF_PATH submodule update --init --recursive
+	cd $IDF_PATH && python -m pip install -r requirements.txt && cd "$AR_ROOT"
 fi
 
-export IDF_COMMIT=$(git -C $IDF_PATH rev-parse --short HEAD)
-export IDF_BRANCH=$(git -C $IDF_PATH symbolic-ref --short HEAD)
+#
+# INSTALL TOOLCHAIN
+#
 
 if ! [ -x "$(command -v $IDF_TOOLCHAIN-gcc)" ]; then
   	echo "GCC toolchain is not installed! Installing local copy"

tools/prepare-libs.sh

@@ -16,7 +16,7 @@ cat pio_start.txt > "$AR_PLATFORMIO_PY"
 rm pio_end.txt 1pio_start.txt 2pio_start.txt pio_start.txt
 
 # include dirs
-AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H \"-I{compiler.sdk.path}/include/config\""
+AR_INC="-DESP_PLATFORM -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\" -DHAVE_CONFIG_H -DGCC_NOT_5_2_0=0 -DWITH_POSIX \"-I{compiler.sdk.path}/include/config\""
 echo "    CPPPATH=[" >> "$AR_PLATFORMIO_PY" && echo "       join(FRAMEWORK_DIR, \"tools\", \"sdk\", \"include\", \"config\")," >> "$AR_PLATFORMIO_PY"
 while [ "$1" != "" ]; do
 	cpath=$1

tools/push-to-arduino.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+source ./tools/config.sh
+
+if [ -x $GITHUB_TOKEN ]; then
+	echo "ERROR: GITHUB_TOKEN was not defined"
+	exit 1
+fi
+
+if ! [ -d "$AR_COMPS/arduino" ]; then
+	echo "ERROR: Target arduino folder does not exist!"
+	exit 1
+fi
+
+#
+# UPDATE FILES
+#
+
+if [ $AR_HAS_COMMIT == "0" ]; then
+	cd $AR_COMPS/arduino
+
+	# make changes to the files
+	echo "Patching files in branch '$AR_NEW_BRANCH_NAME'..."
+	rm -rf $AR_COMPS/arduino/tools/sdk
+	cp -Rf $AR_SDK $AR_COMPS/arduino/tools/sdk
+	cp -f $AR_ESPTOOL_PY $AR_COMPS/arduino/tools/esptool.py
+	cp -f $AR_GEN_PART_PY $AR_COMPS/arduino/tools/gen_esp32part.py
+	cp -f $AR_PLATFORMIO_PY $AR_COMPS/arduino/tools/platformio-build.py
+	cp -f $AR_PLATFORM_TXT $AR_COMPS/arduino/platform.txt
+
+	# did any of the files change?
+	if [ -n "$(git status --porcelain)" ]; then
+		echo "Pushing changes to branch '$AR_NEW_BRANCH_NAME'..."
+	    git add . && git commit --message "$AR_NEW_COMMIT_MESSAGE" && git push -u origin $AR_NEW_BRANCH_NAME
+		if [ $? -ne 0 ]; then
+		    echo "ERROR: Pushing to branch '$AR_NEW_BRANCH_NAME' failed"
+			exit 1
+		fi
+	else
+	    echo "No changes in branch '$AR_NEW_BRANCH_NAME'"
+	    if [ $AR_HAS_BRANCH == "0" ]; then
+	    	echo "Delete created branch '$AR_NEW_BRANCH_NAME'"
+	    	git branch -d $AR_NEW_BRANCH_NAME
+	    fi
+	    exit 0
+	fi
+fi
+
+#
+# CREATE PULL REQUEST
+#
+
+if [ "$AR_HAS_PR" == "0" ]; then
+	pr_created=`git_create_pr "$AR_NEW_BRANCH_NAME" "$AR_NEW_PR_TITLE"`
+	if [ $pr_created == "0" ]; then
+		echo "ERROR: Failed to create PR '$AR_NEW_PR_TITLE': "`echo "$git_create_pr_res" | jq -r '.message'`": "`echo "$git_create_pr_res" | jq -r '.errors[].message'`
+		exit 1
+	fi
+fi
+exit 0