Updates and enhances scripts

Author: HamzaHajeir

.gitignore

@@ -22,3 +22,6 @@ version.txt
 dependencies.lock
 managed_components/
 target/
+core_version.h
+package.json
+release-info.txt

build.sh

@@ -20,17 +20,18 @@ export IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS=1
 
 CCACHE_ENABLE=1
 
-TARGET="all"
+export TARGET="all"
 BUILD_TYPE="all"
 BUILD_DEBUG="default"
 SKIP_ENV=0
 COPY_OUT=0
-ARCHIVE_OUT=0
+ARCHIVE_OUT=1
 if [ -z $DEPLOY_OUT ]; then
     DEPLOY_OUT=0
 fi
 
 function print_help() {
+
     echo "Usage: build.sh [-s] [-n] [-A <arduino_branch>] [-I <idf_branch>] [-D <debug_level>] [-i <idf_commit>] [-c <path>] [-t <target>] [-b <build|menuconfig|reconfigure|idf-libs|copy-bootloader|mem-variant>] [config ...]"
     echo "       -s     Skip installing/updating of ESP-IDF and all components"
     echo "       -n     Disable ccache"
@@ -47,17 +48,17 @@ function print_help() {
     exit 1
 }
 
-while getopts ":A:I:i:c:t:b:D:sde" opt; do
+while getopts ":A:I:i:c:t:b:D:sdne" opt; do
     case ${opt} in
         s )
             SKIP_ENV=1
             ;;
-        n )
-            CCACHE_ENABLE=0
-            ;;
         d )
             DEPLOY_OUT=1
             ;;
+        n )
+            CCACHE_ENABLE=0
+            ;;
         e )
             ARCHIVE_OUT=1
             ;;
@@ -112,6 +113,8 @@ echo "TARGET(s): ${TARGET[@]}"
 
 mkdir -p dist
 
+rm -rf dependencies.lock
+
 if [ $SKIP_ENV -eq 0 ]; then
     echo "* Installing/Updating ESP-IDF and all components..."
     # update components from git
@@ -131,6 +134,10 @@ else
     source ./tools/config.sh
 fi
 
+if [ -f "$AR_MANAGED_COMPS/espressif__esp-sr/.component_hash" ]; then
+    rm -rf $AR_MANAGED_COMPS/espressif__esp-sr/.component_hash
+fi
+
 if [ "$BUILD_TYPE" != "all" ]; then
     if [ "$TARGET" = "all" ]; then
         echo "ERROR: You need to specify target for non-default builds"
@@ -175,9 +182,20 @@ if [ "$BUILD_TYPE" != "all" ]; then
     exit 0
 fi
 
+git submodule update --init --recursive
+
 rm -rf build sdkconfig out
 mkdir -p "$AR_TOOLS/esp32-arduino-libs"
 
+# Add release-info
+rm -rf release-info.txt
+IDF_Commit_short=$(git -C "$IDF_PATH" rev-parse --short HEAD || echo "")
+AR_Commit_short=$(git -C "$AR_COMPS/arduino" rev-parse --short HEAD || echo "")
+echo "Framework built from
+- $IDF_REPO branch [$IDF_BRANCH](https://github.com/$IDF_REPO/tree/$IDF_BRANCH) commit [$IDF_Commit_short](https://github.com/$IDF_REPO/commits/$IDF_BRANCH/#:~:text=$IDF_Commit_short)
+- $AR_REPO branch [$AR_BRANCH](https://github.com/$AR_REPO/tree/$AR_BRANCH) commit [$AR_Commit_short](https://github.com/$AR_REPO/commits/$AR_BRANCH/#:~:text=$AR_Commit_short)
+- Arduino lib builder branch: $GIT_BRANCH" >> release-info.txt
+
 #targets_count=`jq -c '.targets[] | length' configs/builds.json`
 for target_json in `jq -c '.targets[]' configs/builds.json`; do
     target=$(echo "$target_json" | jq -c '.target' | tr -d '"')
@@ -221,6 +239,9 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do
         idf_libs_configs="$idf_libs_configs;configs/defconfig.$defconf"
     done
 
+    if [ -f "$AR_MANAGED_COMPS/espressif__esp-sr/.component_hash" ]; then
+        rm -rf $AR_MANAGED_COMPS/espressif__esp-sr/.component_hash
+    fi
     echo "* Build IDF-Libs: $idf_libs_configs"
     rm -rf build sdkconfig
     idf.py -DIDF_TARGET="$target" -DSDKCONFIG_DEFAULTS="$idf_libs_configs" idf-libs
@@ -246,6 +267,10 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do
             bootloader_configs="$bootloader_configs;configs/defconfig.$defconf";
         done
 
+        if [ -f "$AR_MANAGED_COMPS/espressif__esp-sr/.component_hash" ]; then
+            rm -rf $AR_MANAGED_COMPS/espressif__esp-sr/.component_hash
+        fi
+
         echo "* Build BootLoader: $bootloader_configs"
         rm -rf build sdkconfig
         idf.py -DIDF_TARGET="$target" -DSDKCONFIG_DEFAULTS="$bootloader_configs" copy-bootloader
@@ -259,6 +284,10 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do
             mem_configs="$mem_configs;configs/defconfig.$defconf";
         done
 
+        if [ -f "$AR_MANAGED_COMPS/espressif__esp-sr/.component_hash" ]; then
+            rm -rf $AR_MANAGED_COMPS/espressif__esp-sr/.component_hash
+        fi
+
         echo "* Build Memory Variant: $mem_configs"
         rm -rf build sdkconfig
         idf.py -DIDF_TARGET="$target" -DSDKCONFIG_DEFAULTS="$mem_configs" mem-variant
@@ -304,6 +333,7 @@ if [ "$BUILD_TYPE" = "all" ]; then
     python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
+export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
 
 # Generate pioarduino manifest file
 if [ "$BUILD_TYPE" = "all" ]; then
@@ -316,12 +346,29 @@ if [ "$BUILD_TYPE" = "all" ]; then
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
+AR_VERSION=$(jq -c '.version' "$AR_COMPS/arduino/package.json" | tr -d '"')
+AR_VERSION_UNDERSCORE=`echo "$AR_VERSION" | tr . _`
+
+# Generate PlatformIO framework manifest file
+rm -rf "$AR_ROOT/package.json"
+
 # copy everything to arduino-esp32 installation
 if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then
     echo "* Copying to Arduino..."
     ./tools/copy-to-arduino.sh
     if [ $? -ne 0 ]; then exit 1; fi
 fi
+AR_VERSION=$(jq -c '.version' "$AR_COMPS/arduino/package.json" | tr -d '"')
+AR_VERSION_UNDERSCORE=`echo "$AR_VERSION" | tr . _`
+
+
+
+# Generate core_version.h
+rm -rf "$AR_ROOT/core_version.h"
+echo "#define ARDUINO_ESP32_GIT_VER 0x$AR_Commit_short
+#define ARDUINO_ESP32_GIT_DESC $AR_VERSION
+#define ARDUINO_ESP32_RELEASE_$AR_VERSION_UNDERSCORE
+#define ARDUINO_ESP32_RELEASE \"$AR_VERSION_UNDERSCORE\"" >> "$AR_ROOT/core_version.h"
 
 # push changes to esp32-arduino-libs and create pull request into arduino-esp32
 if [ $DEPLOY_OUT -eq 1 ]; then

tools/archive-build.sh

@@ -1,12 +1,70 @@
 #!/bin/bash
 
-IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD || echo "")
-IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD || git -C "$IDF_PATH" tag --points-at HEAD || echo "")
 idf_version_string=${IDF_BRANCH//\//_}"-$IDF_COMMIT"
 
-archive_path="dist/arduino-esp32-libs-$1-$idf_version_string.tar.gz"
+archive_path="dist/arduino-esp32-libs-$TARGET-$idf_version_string.tar.gz"
+build_archive_path="dist/arduino-esp32-build-$TARGET-$idf_version_string.tar.gz"
+pio_archive_path="dist/framework-arduinoespressif32-$TARGET-$idf_version_string.tar.gz"
+pio_zip_archive_path="dist/framework-arduinoespressif32-$TARGET-$idf_version_string.zip"
+libs_archive_path="dist/framework-arduinoespressif32-libs-$TARGET-$idf_version_string"
+libs_zip_archive_path="$libs_archive_path.zip"
+libs_tar_archive_path="$libs_archive_path.tar.gz"
 
-mkdir -p dist && rm -rf "$archive_path"
-if [ -d "out" ]; then
-	cd out && tar zcf "../$archive_path" * && cd ..
-fi
+mkdir -p dist && rm -rf "$archive_path" "$build_archive_path"
+
+cd out
+echo "Creating PlatformIO Tasmota framework-arduinoespressif32"
+mkdir -p arduino-esp32/cores/esp32
+mkdir -p arduino-esp32/tools/partitions
+cp -rf ../components/arduino/tools arduino-esp32
+cp -rf ../components/arduino/cores arduino-esp32
+cp -rf ../components/arduino/libraries arduino-esp32
+cp -rf ../components/arduino/variants arduino-esp32
+cp -f ../components/arduino/CMa* arduino-esp32
+cp -f ../components/arduino/idf* arduino-esp32
+cp -f ../components/arduino/Kco* arduino-esp32
+cp -f ../components/arduino/pac* arduino-esp32
+# rm -rf arduino-esp32/docs
+# rm -rf arduino-esp32/tests
+# rm -rf arduino-esp32/idf_component_examples
+# rm -rf arduino-esp32/libraries/RainMaker
+# rm -rf arduino-esp32/libraries/Insights
+# rm -rf arduino-esp32/libraries/ESP_I2S
+# rm -rf arduino-esp32/libraries/SPIFFS
+# rm -rf arduino-esp32/libraries/BLE
+# rm -rf arduino-esp32/libraries/SimpleBLE
+# rm -rf arduino-esp32/libraries/BluetoothSerial
+# rm -rf arduino-esp32/libraries/WiFiProv
+# rm -rf arduino-esp32/libraries/WiFiClientSecure
+# rm -rf arduino-esp32/libraries/NetworkClientSecure
+# rm -rf arduino-esp32/libraries/ESP_SR
+# rm -rf arduino-esp32/libraries/ESP_NOW
+# rm -rf arduino-esp32/libraries/TFLiteMicro
+# rm -rf arduino-esp32/libraries/ESP32
+rm -rf arduino-esp32/package
+rm -rf arduino-esp32/tools/esp32-arduino-libs
+rm -rf arduino-esp32/tools/*.exe
+rm -rf arduino-esp32/tools/esptool.py
+rm -rf arduino-esp32/tools/get.py
+rm -rf arduino-esp32/tools/ide-debug
+rm -rf arduino-esp32/tools/gen_insights_package.py
+rm -rf arduino-esp32/platform.txt
+rm -rf arduino-esp32/programmers.txt
+rm -rf arduino-esp32/boards.txt
+rm -rf arduino-esp32/package.json
+rm -rf arduino-esp32/*.md
+cp -Rf tools/esp32-arduino-libs arduino-esp32/tools/
+cp ../package.json arduino-esp32/package.json
+cp ../core_version.h arduino-esp32/cores/esp32/core_version.h
+mv arduino-esp32/ framework-arduinoespressif32/
+cd framework-arduinoespressif32/libraries
+rm -rf **/examples
+cd ../tools/esp32-arduino-libs
+#rm -rf **/flags
+cd ../../../
+# If the framework is needed as tar.gz uncomment next line
+# tar --exclude=.* -zcf ../$pio_archive_path framework-arduinoespressif32/
+mv framework-arduinoespressif32/tools/esp32-arduino-libs framework-arduinoespressif32-libs
+
+7z a -mx=9 -tzip -xr'!.*' ../$pio_zip_archive_path framework-arduinoespressif32/
+7z a -mx=9 -tzip -xr'!.*' ../$libs_zip_archive_path framework-arduinoespressif32-libs/

tools/config.sh

@@ -6,9 +6,10 @@ if [ -z $IDF_PATH ]; then
 fi
 
 if [ -z $IDF_BRANCH ]; then
-    IDF_BRANCH="release/v5.3"
+    export IDF_BRANCH="release/v5.3"
 fi
 
+# Arduino branch to use
 if [ -z $AR_PR_TARGET_BRANCH ]; then
     AR_PR_TARGET_BRANCH="master"
 fi
@@ -27,6 +28,12 @@ fi
 # Owner of the target ESP32 Arduino repository
 AR_USER="${GITHUB_REPOSITORY_OWNER:-espressif}"
 
+# IDF commit to use
+#IDF_COMMIT=""
+
+# Arduino commit to use
+#AR_COMMIT=""
+
 # The full name of the repository
 AR_REPO="$AR_USER/arduino-esp32"
 IDF_REPO="$AR_USER/esp-idf"
@@ -52,9 +59,21 @@ AR_SDK="$AR_TOOLS/esp32-arduino-libs/$IDF_TARGET"
 PIOARDUINO_SDK="FRAMEWORK_SDK_DIR, \"$IDF_TARGET\""
 TOOLS_JSON_OUT="$AR_TOOLS/esp32-arduino-libs"
 
+if [ "$IDF_COMMIT" ]; then
+	export IDF_COMMIT
+fi
+
 if [ -d "$IDF_PATH" ]; then
     export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
-    export IDF_BRANCH=$(git -C "$IDF_PATH" symbolic-ref --short HEAD || git -C "$IDF_PATH" tag --points-at HEAD)
+fi
+
+echo "Using IDF branch $IDF_BRANCH"
+echo "Using IDF commit $IDF_COMMIT"
+
+if [ "$AR_COMMIT" ]; then
+    echo "Using Arduino commit $AR_COMMIT"
+else
+    export AR_COMMIT=$(git -C "$AR_COMPS/arduino" rev-parse --short HEAD || echo "")
 fi
 
 function get_os(){
@@ -84,6 +103,7 @@ function get_os(){
 AR_OS=`get_os`
 
 export SED="sed"
+export AWK="awk"
 export SSTAT="stat -c %s"
 
 if [[ "$AR_OS" == "macos" ]]; then
@@ -96,6 +116,7 @@ if [[ "$AR_OS" == "macos" ]]; then
         exit 1
     fi
     export SED="gsed"
+    export AWK="gawk"
     export SSTAT="stat -f %z"
 fi
 

tools/gen_pioarduino_manifest.py

@@ -3,6 +3,7 @@
 import os
 import re
 import sys
+import datetime
 
 MANIFEST_DATA = {
     "name": "framework-arduinoespressif32-libs",
@@ -11,7 +12,7 @@
     "license": "LGPL-2.1-or-later",
     "repository": {
         "type": "git",
-        "url": "https://github.com/espressif/esp32-arduino-lib-builder",
+        "url": "https://github.com/hamzahajeir/esp32-arduino-lib-builder",
     },
 }
 
@@ -48,8 +49,10 @@ def main(dst_dir, version_string, commit_hash):
         return -1
 
     manifest_file_path = os.path.join(dst_dir, "package.json")
+    build_date = datetime.date.today()
     with open(manifest_file_path, "w", encoding="utf8") as fp:
         MANIFEST_DATA["version"] = f"{converted_version}+sha.{commit_hash}"
+        MANIFEST_DATA["date"] = f"{build_date}"
         json.dump(MANIFEST_DATA, fp, indent=2)
 
     print(

tools/gen_tools_json.py

@@ -35,7 +35,7 @@ def replace_if_xz(system):
     # let's get the checksum file from the release
     release_manifest_url = ""
     # parse the download url to extract all info needed for the checksum file url
-    urlx = re.findall("^https://github.com/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/releases/download/([a-zA-Z0-9_\-.]+)/([a-zA-Z0-9_\-.]+)$", new_url)
+    urlx = re.findall("^https://github.com/([a-zA-Z0-9_-]+)/([a-zA-Z0-9_-]+)/releases/download/([a-zA-Z0-9_\\-.]+)/([a-zA-Z0-9_\-.]+)$", new_url)
     if urlx and len(urlx) > 0:
         (owner, proj, version, filename) = urlx[0]
         release_manifest_url = "https://github.com/%s/%s/releases/download/%s/%s-%s-checksum.sha256" % (owner, proj, version, proj, version)
@@ -59,7 +59,7 @@ def replace_if_xz(system):
             "files": []
         }
         release_manifest_contents = requests.get(release_manifest_url).text
-        x = re.findall("\s([a-zA-Z0-9_\-.]+):\s([0-9]+)\s[a-z]+\\n([a-f0-9]+)\s\*.*", release_manifest_contents)
+        x = re.findall("\s([a-zA-Z0-9_\-.]+):\s([0-9]+)\s[a-z]+\\n([a-f0-9]+)\\s\*.*", release_manifest_contents)
         if x and len(x) > 0:
             for line in x:
                 (filename, size, checksum) = line