From 61d378faf476d70626f2f023a7da8d58919aee9f Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 3 Aug 2019 11:49:43 -0700 Subject: [PATCH 1/5] Move all scripts and documentation to Python3 Python 2 EOL is Jan 1, 2020. Migrate scripts to run under Python 3. Under Windows, we're already running Python 3.7, by dumb luck. The oddness is that the Windows standalone executable for Python 3 is called "python" whereas under UNIX-like OSes it's called "python3" with "python" always referring to the Python 2 executable. The ZIP needs to be updated to include a Python3.exe (copy of Python.exe) so that we can use the same command lines under Linux and Windows, and to preserve my sanity. Fixes #6376 --- doc/eclipse/makefile.init | 2 +- doc/installing.rst | 6 +-- doc/ota_updates/readme.rst | 2 +- .../BearSSL_CertStore/certs-from-mozilla.py | 22 ++++----- package/build_boards_manager_package.sh | 4 +- package/drop_versions.py | 3 +- package/merge_packages.py | 5 +- .../package_esp8266com_index.template.json | 46 +++++++++---------- platform.txt | 22 ++++----- tests/common.sh | 4 +- tests/device/Makefile | 3 +- tests/device/libraries/BSTest/Makefile | 2 +- tests/device/libraries/BSTest/runner.py | 3 +- tests/device/test_BearSSL/make_spiffs.py | 22 ++++----- tools/boards.txt.py | 2 +- tools/build.py | 4 +- tools/elf2bin.py | 2 +- tools/espota.py | 6 +-- tools/get.py | 9 +--- tools/makecorever.py | 2 +- tools/signing.py | 2 +- tools/upload.py | 2 +- 22 files changed, 78 insertions(+), 97 deletions(-) diff --git a/doc/eclipse/makefile.init b/doc/eclipse/makefile.init index 76438d62c8..852bb8aabe 100644 --- a/doc/eclipse/makefile.init +++ b/doc/eclipse/makefile.init @@ -14,7 +14,7 @@ ESP8266_BASE = $(ARDUINO_BASE)/hardware/esp8266com/esp8266 ESP8266_TOOLS = $(ESP8266_BASE)/tools XTENSA_TOOLS_ROOT = $(ESP8266_TOOLS)/xtensa-lx106-elf/bin -PYTHON_BIN = python +PYTHON_BIN = python3 ESPTOOL_PY_BIN = $(ESP8266_TOOLS)/esptool.py ESPOTA_PY_BIN = $(ESP8266_TOOLS)/espota.py ESPTOOL_BIN = $(ESP8266_TOOLS)/esptool/esptool.exe diff --git a/doc/installing.rst b/doc/installing.rst index a2c2b00e28..1c6317d2bd 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -43,7 +43,7 @@ Prerequisites - Arduino 1.6.8 (or newer, current working version is 1.8.5) - git -- Python 2.7 (https://python.org) +- Python 3.x (https://python.org) - terminal, console, or command prompt (depending on your OS) - Internet connection @@ -110,7 +110,7 @@ Instructions - Windows 10 .. code:: bash cd esp8266/tools - python get.py + python3 get.py - Restart Arduino @@ -184,7 +184,7 @@ Instructions - Other OS .. code:: bash cd esp8266/tools - python get.py + python3 get.py - Restart Arduino diff --git a/doc/ota_updates/readme.rst b/doc/ota_updates/readme.rst index bec4fba2e9..45f0079d9b 100644 --- a/doc/ota_updates/readme.rst +++ b/doc/ota_updates/readme.rst @@ -193,7 +193,7 @@ Instructions below show configuration of OTA on NodeMCU 1.0 (ESP-12E Module) boa - esp8266/Arduino platform package 2.0.0 or newer - for instructions follow https://github.com/esp8266/Arduino#installing-with-boards-manager - - Python 2.7 - https://www.python.org/ + - Python 3.x - https://www.python.org/ **Note:** Windows users should select “Add python.exe to Path” (see below – this option is not selected by default). diff --git a/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py b/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py index 13c77ccf52..c583da3812 100755 --- a/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py +++ b/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # This script pulls the list of Mozilla trusted certificate authorities # from the web at the "mozurl" below, parses the file to grab the PEM @@ -11,13 +11,8 @@ import csv import os from subprocess import Popen, PIPE, call -import urllib2 -try: - # for Python 2.x - from StringIO import StringIO -except ImportError: - # for Python 3.x - from io import StringIO +from urllib.request import urlopen +from io import StringIO # Mozilla's URL for the CSV file with included PEM certs mozurl = "https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV" @@ -25,9 +20,10 @@ # Load the manes[] and pems[] array from the URL names = [] pems = [] -response = urllib2.urlopen(mozurl) -csvData = response.read() -csvReader = csv.reader(StringIO(csvData)) +response = urlopen(mozurl) +csvData = response.read().decode('utf-8') +csvFile = StringIO(csvData) +csvReader = csv.reader(csvFile) for row in csvReader: names.append(row[0]+":"+row[1]+":"+row[2]) pems.append(row[30]) @@ -46,10 +42,10 @@ for i in range(0, len(pems)): certName = "data/ca_%03d.der" % (idx); thisPem = pems[i].replace("'", "") - print names[i] + " -> " + certName + print(names[i] + " -> " + certName) ssl = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE) pipe = ssl.stdin - pipe.write(thisPem) + pipe.write(thisPem.encode('utf-8')) pipe.close() ssl.wait() if os.path.exists(certName): diff --git a/package/build_boards_manager_package.sh b/package/build_boards_manager_package.sh index e18ededfc3..8220ad2367 100755 --- a/package/build_boards_manager_package.sh +++ b/package/build_boards_manager_package.sh @@ -74,7 +74,7 @@ fi # handles tool paths differently when package is installed in hardware folder cat $srcdir/platform.txt | \ $SED 's/runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-lx106-elf//g' | \ -$SED 's/runtime.tools.python.path=.*//g' | \ +$SED 's/runtime.tools.python3.path=.*//g' | \ $SED 's/runtime.tools.esptool.path={runtime.platform.path}\/tools\/esptool//g' | \ $SED 's/tools.esptool.path={runtime.platform.path}\/tools\/esptool/tools.esptool.path=\{runtime.tools.esptool.path\}/g' | \ $SED 's/^tools.esptool.cmd=.*//g' | \ @@ -156,7 +156,7 @@ new_json=package_esp8266com_index.json set +e # Merge the old and new, then drop any obsolete package versions -python ../../merge_packages.py $new_json $old_json | python ../../drop_versions.py - tools 1.20.0-26-gb404fb9 >tmp && mv tmp $new_json && rm $old_json +python3 ../../merge_packages.py $new_json $old_json | python3 ../../drop_versions.py - tools 1.20.0-26-gb404fb9 >tmp && mv tmp $new_json && rm $old_json # Verify the JSON file can be read, fail if it's not OK set -e diff --git a/package/drop_versions.py b/package/drop_versions.py index a3aa9b288c..c5659c8fe2 100755 --- a/package/drop_versions.py +++ b/package/drop_versions.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This script drops one or multiple versions of a release # -from __future__ import print_function import json import sys from collections import OrderedDict diff --git a/package/merge_packages.py b/package/merge_packages.py index dba8bb9d63..e3732920c4 100755 --- a/package/merge_packages.py +++ b/package/merge_packages.py @@ -1,10 +1,9 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This script merges two Arduino Board Manager package json files. # Usage: -# python merge_packages.py package_esp8266com_index.json version/new/package_esp8266com_index.json +# python3 merge_packages.py package_esp8266com_index.json version/new/package_esp8266com_index.json # Written by Ivan Grokhotkov, 2015 # -from __future__ import print_function import json import sys diff --git a/package/package_esp8266com_index.template.json b/package/package_esp8266com_index.template.json index 74204a1235..85f396a230 100644 --- a/package/package_esp8266com_index.template.json +++ b/package/package_esp8266com_index.template.json @@ -142,52 +142,52 @@ "systems": [ { "host": "x86_64-mingw32", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/python-3.7.2.post1-embed-win32v2.zip", + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python-3.7.2.post1-embed-win32v2.zip", "archiveFileName": "python-3.7.2.post1-embed-win32v2.zip", "checksum": "SHA-256:26665d2925ee75118bb7d8620e9ee988adc2ca3e660a9f4c06a09a06c94c0c29", "size": "6431781" }, { "host": "i686-mingw32", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/python-3.7.2.post1-embed-win32v2.zip", + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python-3.7.2.post1-embed-win32v2.zip", "archiveFileName": "python-3.7.2.post1-embed-win32v2.zip", "checksum": "SHA-256:26665d2925ee75118bb7d8620e9ee988adc2ca3e660a9f4c06a09a06c94c0c29", "size": "6431781" }, { "host": "aarch64-linux-gnu", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/python-placeholder.tar.gz", - "archiveFileName": "python-placeholder.tar.gz", - "checksum": "SHA-256:3b32fdb0905abf97e923ff968b6a0da8ce85d632b27845d7e2fc759778778785", - "size": "193" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-placeholder.tar.gz", + "archiveFileName": "python3-placeholder.tar.gz", + "checksum": "SHA-256:d8cf9d9d66423d7b90978ebe285a73a6e8611995cd0d5e6273e929a0cf2c9850", + "size": "191" }, { "host": "arm-linux-gnueabihf", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/python-placeholder.tar.gz", - "archiveFileName": "python-placeholder.tar.gz", - "checksum": "SHA-256:3b32fdb0905abf97e923ff968b6a0da8ce85d632b27845d7e2fc759778778785", - "size": "193" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-placeholder.tar.gz", + "archiveFileName": "python3-placeholder.tar.gz", + "checksum": "SHA-256:d8cf9d9d66423d7b90978ebe285a73a6e8611995cd0d5e6273e929a0cf2c9850", + "size": "191" }, { "host": "i686-pc-linux-gnu", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/python-placeholder.tar.gz", - "archiveFileName": "python-placeholder.tar.gz", - "checksum": "SHA-256:3b32fdb0905abf97e923ff968b6a0da8ce85d632b27845d7e2fc759778778785", - "size": "193" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-placeholder.tar.gz", + "archiveFileName": "python3-placeholder.tar.gz", + "checksum": "SHA-256:d8cf9d9d66423d7b90978ebe285a73a6e8611995cd0d5e6273e929a0cf2c9850", + "size": "191" }, { "host": "x86_64-apple-darwin", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/python-placeholder.tar.gz", - "archiveFileName": "python-placeholder.tar.gz", - "checksum": "SHA-256:3b32fdb0905abf97e923ff968b6a0da8ce85d632b27845d7e2fc759778778785", - "size": "193" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-placeholder.tar.gz", + "archiveFileName": "python3-placeholder.tar.gz", + "checksum": "SHA-256:d8cf9d9d66423d7b90978ebe285a73a6e8611995cd0d5e6273e929a0cf2c9850", + "size": "191" }, { "host": "x86_64-pc-linux-gnu", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-3/python-placeholder.tar.gz", - "archiveFileName": "python-placeholder.tar.gz", - "checksum": "SHA-256:3b32fdb0905abf97e923ff968b6a0da8ce85d632b27845d7e2fc759778778785", - "size": "193" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python3-placeholder.tar.gz", + "archiveFileName": "python3-placeholder.tar.gz", + "checksum": "SHA-256:d8cf9d9d66423d7b90978ebe285a73a6e8611995cd0d5e6273e929a0cf2c9850", + "size": "191" } ] }, @@ -359,4 +359,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/platform.txt b/platform.txt index 9ddfe6dab2..5e7ab768b7 100644 --- a/platform.txt +++ b/platform.txt @@ -10,7 +10,7 @@ version=2.6.0-dev # These will be removed by the packager script when doing a JSON release runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}/tools/xtensa-lx106-elf -runtime.tools.python.path={runtime.platform.path}/tools/python +runtime.tools.python3.path={runtime.platform.path}/tools/python3 runtime.tools.esptool.path={runtime.platform.path}/tools/esptool runtime.tools.signing={runtime.platform.path}/tools/signing.py @@ -84,8 +84,8 @@ compiler.elf2hex.extra_flags= ## generate file with git version number ## needs bash, git, and echo -recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python.path}/python" "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h" -recipe.hooks.core.prebuild.pattern="{runtime.tools.python.path}/python" "{runtime.tools.makecorever}" --build_path "{build.path}" --platform_path "{runtime.platform.path}" --version "unix-{version}" +recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.signing}" --mode header --publickey "{build.source.path}/public.key" --out "{build.path}/core/Updater_Signing.h" +recipe.hooks.core.prebuild.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.makecorever}" --build_path "{build.path}" --platform_path "{runtime.platform.path}" --version "unix-{version}" ## Build the app.ld linker file recipe.hooks.linking.prelink.1.pattern="{compiler.path}{compiler.c.cmd}" -CC -E -P {build.vtable_flags} "{runtime.platform.path}/tools/sdk/ld/eagle.app.v6.common.ld.h" -o "{build.path}/local.eagle.app.v6.common.ld" @@ -109,8 +109,8 @@ recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {build.exception_ recipe.objcopy.eep.pattern= ## Create hex -recipe.objcopy.hex.1.pattern="{runtime.tools.python.path}/python" "{runtime.tools.elf2bin}" --eboot "{runtime.tools.eboot}" --app "{build.path}/{build.project_name}.elf" --flash_mode {build.flash_mode} --flash_freq {build.flash_freq} --flash_size {build.flash_size} --path "{runtime.tools.xtensa-lx106-elf-gcc.path}/bin" --out "{build.path}/{build.project_name}.bin" -recipe.objcopy.hex.2.pattern="{runtime.tools.python.path}/python" "{runtime.tools.signing}" --mode sign --privatekey "{build.source.path}/private.key" --bin "{build.path}/{build.project_name}.bin" --out "{build.path}/{build.project_name}.bin.signed" --legacy "{build.path}/{build.project_name}.bin.legacy_sig" +recipe.objcopy.hex.1.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.elf2bin}" --eboot "{runtime.tools.eboot}" --app "{build.path}/{build.project_name}.elf" --flash_mode {build.flash_mode} --flash_freq {build.flash_freq} --flash_size {build.flash_size} --path "{runtime.tools.xtensa-lx106-elf-gcc.path}/bin" --out "{build.path}/{build.project_name}.bin" +recipe.objcopy.hex.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.signing}" --mode sign --privatekey "{build.source.path}/private.key" --bin "{build.path}/{build.project_name}.bin" --out "{build.path}/{build.project_name}.bin.signed" --legacy "{build.path}/{build.project_name}.bin.legacy_sig" ## Save hex recipe.output.tmp_file={build.project_name}.bin @@ -126,12 +126,12 @@ recipe.size.regex.data=^(?:\.data|\.rodata|\.bss)\s+([0-9]+).* tools.esptool.path= # Because the variable expansion doesn't allow one tool to find another, the following lines -# will point to "{runtime.platform.path}/tools/python/python" in GIT and -# "{runtime.tools.python.path}/python" for JSON board manager releases. -#tools.esptool.cmd={runtime.tools.python.path}/python -#tools.esptool.network_cmd={runtime.tools.python.path}/python -tools.esptool.cmd={runtime.platform.path}/tools/python/python -tools.esptool.network_cmd={runtime.platform.path}/tools/python/python +# will point to "{runtime.platform.path}/tools/python3/python3" in GIT and +# "{runtime.tools.python3.path}/python3" for JSON board manager releases. +#tools.esptool.cmd={runtime.tools.python3.path}/python3 +#tools.esptool.network_cmd={runtime.tools.python3.path}/python3 +tools.esptool.cmd={runtime.platform.path}/tools/python3/python3 +tools.esptool.network_cmd={runtime.platform.path}/tools/python3/python3 tools.esptool.upload.protocol=esp tools.esptool.upload.params.verbose=--trace diff --git a/tests/common.sh b/tests/common.sh index ca2c535232..ca99bee7ab 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -65,7 +65,7 @@ function build_sketches() local build_rem=$5 local lwip=$6 mkdir -p $build_dir - local build_cmd="python tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir -n $lwip $build_arg " + local build_cmd="python3 tools/build.py -b generic -v -w all -s 4M1M -v -k --build_cache $cache_dir -p $PWD/$build_dir -n $lwip $build_arg " local sketches=$(find $srcpath -name *.ino | sort) print_size_info >size.log export ARDUINO_IDE_PATH=$arduino @@ -163,7 +163,7 @@ function install_ide() cat esp8266/platform.local.txt echo -e "\n----\n" cd esp8266/tools - python get.py + python3 get.py export PATH="$ide_path:$core_path/tools/xtensa-lx106-elf/bin:$PATH" } diff --git a/tests/device/Makefile b/tests/device/Makefile index 3412bc96dc..b34cb0dbbc 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -4,8 +4,7 @@ TEST_LIST ?= $(wildcard test_*/*.ino) ESP8266_CORE_PATH ?= $(realpath ../..) BUILD_DIR ?= $(PWD)/.build HARDWARE_DIR ?= $(PWD)/.hardware -#PYTHON ?= python3 -PYTHON ?= python +PYTHON ?= python3 ESPTOOL ?= $(PYTHON) $(ESP8266_CORE_PATH)/tools/esptool/esptool.py MKSPIFFS ?= $(ESP8266_CORE_PATH)/tools/mkspiffs/mkspiffs UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB) diff --git a/tests/device/libraries/BSTest/Makefile b/tests/device/libraries/BSTest/Makefile index 2677ad26f3..93c181f5fb 100644 --- a/tests/device/libraries/BSTest/Makefile +++ b/tests/device/libraries/BSTest/Makefile @@ -15,7 +15,7 @@ $(PYTHON_ENV_DIR): . $(PYTHON_ENV_DIR)/bin/activate && pip install -r requirements.txt test: $(TEST_EXECUTABLE) $(PYTHON_ENV_DIR) - . $(PYTHON_ENV_DIR)/bin/activate && python runner.py -e $(TEST_EXECUTABLE) -m test/test.py + . $(PYTHON_ENV_DIR)/bin/activate && $(PYTHON) runner.py -e $(TEST_EXECUTABLE) -m test/test.py $(TEST_EXECUTABLE): test/test.cpp g++ -std=c++11 -Isrc -o $@ test/test.cpp diff --git a/tests/device/libraries/BSTest/runner.py b/tests/device/libraries/BSTest/runner.py index 9778f35ada..21ecb68a7e 100644 --- a/tests/device/libraries/BSTest/runner.py +++ b/tests/device/libraries/BSTest/runner.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -from __future__ import print_function +#!/usr/bin/env python3 import pexpect from pexpect import EOF, TIMEOUT, fdpexpect import sys diff --git a/tests/device/test_BearSSL/make_spiffs.py b/tests/device/test_BearSSL/make_spiffs.py index 13c77ccf52..c583da3812 100755 --- a/tests/device/test_BearSSL/make_spiffs.py +++ b/tests/device/test_BearSSL/make_spiffs.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # This script pulls the list of Mozilla trusted certificate authorities # from the web at the "mozurl" below, parses the file to grab the PEM @@ -11,13 +11,8 @@ import csv import os from subprocess import Popen, PIPE, call -import urllib2 -try: - # for Python 2.x - from StringIO import StringIO -except ImportError: - # for Python 3.x - from io import StringIO +from urllib.request import urlopen +from io import StringIO # Mozilla's URL for the CSV file with included PEM certs mozurl = "https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV" @@ -25,9 +20,10 @@ # Load the manes[] and pems[] array from the URL names = [] pems = [] -response = urllib2.urlopen(mozurl) -csvData = response.read() -csvReader = csv.reader(StringIO(csvData)) +response = urlopen(mozurl) +csvData = response.read().decode('utf-8') +csvFile = StringIO(csvData) +csvReader = csv.reader(csvFile) for row in csvReader: names.append(row[0]+":"+row[1]+":"+row[2]) pems.append(row[30]) @@ -46,10 +42,10 @@ for i in range(0, len(pems)): certName = "data/ca_%03d.der" % (idx); thisPem = pems[i].replace("'", "") - print names[i] + " -> " + certName + print(names[i] + " -> " + certName) ssl = Popen(['openssl','x509','-inform','PEM','-outform','DER','-out', certName], shell = False, stdin = PIPE) pipe = ssl.stdin - pipe.write(thisPem) + pipe.write(thisPem.encode('utf-8')) pipe.close() ssl.wait() if os.path.exists(certName): diff --git a/tools/boards.txt.py b/tools/boards.txt.py index cd50470a92..371d6bbec4 100755 --- a/tools/boards.txt.py +++ b/tools/boards.txt.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # boards.txt python builder for esp8266/Arduino # Copyright (C) 2017 community diff --git a/tools/build.py b/tools/build.py index 828b53408e..80ddcbb23d 100755 --- a/tools/build.py +++ b/tools/build.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # build.py — build a sketch using arduino-builder @@ -20,8 +20,6 @@ # # - -from __future__ import print_function import sys import os import argparse diff --git a/tools/elf2bin.py b/tools/elf2bin.py index 214cd9407a..e661163e2c 100755 --- a/tools/elf2bin.py +++ b/tools/elf2bin.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Generate an Arduino compatible BIN file from bootloader and sketch ELF # Replaces esptool-ck.exe and emulates its behavior. diff --git a/tools/espota.py b/tools/espota.py index 0b62705119..373020a913 100755 --- a/tools/espota.py +++ b/tools/espota.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Original espota.py by Ivan Grokhotkov: # https://gist.github.com/igrr/d35ab8446922179dc58c @@ -8,9 +8,9 @@ # Modified since 2016-01-03 from Matthew O'Gorman (https://githumb.com/mogorman) # # This script will push an OTA update to the ESP -# use it like: python espota.py -i -I -p -P [-a password] -f +# use it like: python3 espota.py -i -I -p -P [-a password] -f # Or to upload SPIFFS image: -# python espota.py -i -I -p -P [-a password] -s -f +# python3 espota.py -i -I -p -P [-a password] -s -f # # Changes # 2015-09-18: diff --git a/tools/get.py b/tools/get.py index a1091c5f08..859d5e782e 100755 --- a/tools/get.py +++ b/tools/get.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # This script will download and extract required tools into the current directory. # Tools list is obtained from package/package_esp8266com_index.template.json file. # Written by Ivan Grokhotkov, 2015. # -from __future__ import print_function import os import shutil import errno @@ -15,11 +14,7 @@ import tarfile import zipfile import re -if sys.version_info[0] == 3: - from urllib.request import urlretrieve -else: - # Not Python 3 - today, it is most likely to be Python 2 - from urllib import urlretrieve +from urllib.request import urlretrieve dist_dir = 'dist/' diff --git a/tools/makecorever.py b/tools/makecorever.py index b8fbda8ef7..ee963f0471 100755 --- a/tools/makecorever.py +++ b/tools/makecorever.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Generate the core_version.h header per-build # diff --git a/tools/signing.py b/tools/signing.py index d3dcc7cc18..8c58d8bbbc 100755 --- a/tools/signing.py +++ b/tools/signing.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # import argparse diff --git a/tools/upload.py b/tools/upload.py index e6725fdf75..1cc911a707 100755 --- a/tools/upload.py +++ b/tools/upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Wrapper for Arduino core / others that can call esptool.py possibly multiple times # Adds pyserial to sys.path automatically based on the path of the current file From b06f44ff9b02d3e41f3b286ae5074d4c5b3bb24b Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 3 Aug 2019 12:08:09 -0700 Subject: [PATCH 2/5] Add new Windows ZIP with python3.exe file --- package/package_esp8266com_index.template.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/package/package_esp8266com_index.template.json b/package/package_esp8266com_index.template.json index 85f396a230..d9c09d5208 100644 --- a/package/package_esp8266com_index.template.json +++ b/package/package_esp8266com_index.template.json @@ -142,17 +142,17 @@ "systems": [ { "host": "x86_64-mingw32", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python-3.7.2.post1-embed-win32v2.zip", - "archiveFileName": "python-3.7.2.post1-embed-win32v2.zip", - "checksum": "SHA-256:26665d2925ee75118bb7d8620e9ee988adc2ca3e660a9f4c06a09a06c94c0c29", - "size": "6431781" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python-3.7.2.post1-embed-win32v2a.zip", + "archiveFileName": "python-3.7.2.post1-embed-win32v2a.zip", + "checksum": "SHA-256:8eb71cd637c42abbeea9f31912bbaca22222405d8624ace8901c801519321d35", + "size": "6481988" }, { "host": "i686-mingw32", - "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python-3.7.2.post1-embed-win32v2.zip", - "archiveFileName": "python-3.7.2.post1-embed-win32v2.zip", - "checksum": "SHA-256:26665d2925ee75118bb7d8620e9ee988adc2ca3e660a9f4c06a09a06c94c0c29", - "size": "6431781" + "url": "https://github.com/earlephilhower/esp-quick-toolchain/releases/download/2.5.0-4/python-3.7.2.post1-embed-win32v2a.zip", + "archiveFileName": "python-3.7.2.post1-embed-win32va2.zip", + "checksum": "SHA-256:8eb71cd637c42abbeea9f31912bbaca22222405d8624ace8901c801519321d35", + "size": "6481988" }, { "host": "aarch64-linux-gnu", From 2f3105e604ee80da10d83b47390ade6e036aa608 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 3 Aug 2019 13:37:02 -0700 Subject: [PATCH 3/5] Sort options in boards.txt generation for repeatability The order of the board opts dict changes depending on the Python version and machine, so sort the options before printing them to get a stable ordering. --- boards.txt | 32 +++++++++---------- .../package_esp8266com_index.template.json | 2 +- tools/boards.txt.py | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/boards.txt b/boards.txt index b14d8df2cd..3adf70873f 100644 --- a/boards.txt +++ b/boards.txt @@ -757,15 +757,15 @@ esp8285.menu.baud.921600.upload.speed=921600 espduino.name=ESPDuino (ESP-13 Module) espduino.build.board=ESP8266_ESP13 espduino.build.variant=ESPDuino -espduino.menu.ResetMethod.v2=ESPduino-V2 -espduino.menu.ResetMethod.v2.upload.resetmethod=nodemcu espduino.menu.ResetMethod.v1=ESPduino-V1 espduino.menu.ResetMethod.v1.upload.resetmethod=ck +espduino.menu.ResetMethod.v2=ESPduino-V2 +espduino.menu.ResetMethod.v2.upload.resetmethod=nodemcu +espduino.menu.UploadTool.espota=OTA +espduino.menu.UploadTool.espota.upload.tool=espota espduino.menu.UploadTool.esptool=Serial espduino.menu.UploadTool.esptool.upload.tool=esptool espduino.menu.UploadTool.esptool.upload.verbose=-vv -espduino.menu.UploadTool.espota=OTA -espduino.menu.UploadTool.espota.upload.tool=espota espduino.upload.tool=esptool espduino.upload.maximum_data_size=81920 espduino.upload.wait_for_upload_port=true @@ -4548,20 +4548,20 @@ wifinfo.build.board=WIFINFO wifinfo.build.variant=wifinfo wifinfo.menu.ESPModule.ESP07192=ESP07 (1M/192K SPIFFS) wifinfo.menu.ESPModule.ESP07192.build.board=ESP8266_ESP07 -wifinfo.menu.ESPModule.ESP07192.build.flash_size=1M wifinfo.menu.ESPModule.ESP07192.build.flash_ld=eagle.flash.1m192.ld -wifinfo.menu.ESPModule.ESP07192.build.spiffs_start=0xCB000 -wifinfo.menu.ESPModule.ESP07192.build.spiffs_end=0xFB000 +wifinfo.menu.ESPModule.ESP07192.build.flash_size=1M wifinfo.menu.ESPModule.ESP07192.build.spiffs_blocksize=4096 +wifinfo.menu.ESPModule.ESP07192.build.spiffs_end=0xFB000 +wifinfo.menu.ESPModule.ESP07192.build.spiffs_start=0xCB000 wifinfo.menu.ESPModule.ESP07192.upload.maximum_size=827376 wifinfo.menu.ESPModule.ESP12=ESP12 (4M/1M SPIFFS) wifinfo.menu.ESPModule.ESP12.build.board=ESP8266_ESP12 -wifinfo.menu.ESPModule.ESP12.build.flash_size=4M wifinfo.menu.ESPModule.ESP12.build.flash_ld=eagle.flash.4m1m.ld -wifinfo.menu.ESPModule.ESP12.build.spiffs_start=0x300000 -wifinfo.menu.ESPModule.ESP12.build.spiffs_end=0x3FB000 +wifinfo.menu.ESPModule.ESP12.build.flash_size=4M wifinfo.menu.ESPModule.ESP12.build.spiffs_blocksize=8192 +wifinfo.menu.ESPModule.ESP12.build.spiffs_end=0x3FB000 wifinfo.menu.ESPModule.ESP12.build.spiffs_pagesize=256 +wifinfo.menu.ESPModule.ESP12.build.spiffs_start=0x300000 wifinfo.menu.ESPModule.ESP12.upload.maximum_size=1044464 wifinfo.upload.tool=esptool wifinfo.upload.maximum_data_size=81920 @@ -4793,16 +4793,16 @@ arduino-esp8266.name=Arduino arduino-esp8266.build.board=ESP8266_ARDUINO arduino-esp8266.menu.BoardModel.primo=Primo arduino-esp8266.menu.BoardModel.primo.build.board=ESP8266_ARDUINO_PRIMO -arduino-esp8266.menu.BoardModel.primo.build.variant=arduino_spi arduino-esp8266.menu.BoardModel.primo.build.extra_flags=-DF_CRYSTAL=40000000 -DESP8266 -arduino-esp8266.menu.BoardModel.unowifideved=Uno WiFi -arduino-esp8266.menu.BoardModel.unowifideved.build.board=ESP8266_ARDUINO_UNOWIFI -arduino-esp8266.menu.BoardModel.unowifideved.build.variant=arduino_uart -arduino-esp8266.menu.BoardModel.unowifideved.build.extra_flags=-DF_CRYSTAL=40000000 -DESP8266 +arduino-esp8266.menu.BoardModel.primo.build.variant=arduino_spi arduino-esp8266.menu.BoardModel.starottodeved=Star OTTO -arduino-esp8266.menu.BoardModel.starottodeved.build.variant=arduino_uart arduino-esp8266.menu.BoardModel.starottodeved.build.board=ESP8266_ARDUINO_STAR_OTTO arduino-esp8266.menu.BoardModel.starottodeved.build.extra_flags=-DF_CRYSTAL=40000000 -DESP8266 +arduino-esp8266.menu.BoardModel.starottodeved.build.variant=arduino_uart +arduino-esp8266.menu.BoardModel.unowifideved=Uno WiFi +arduino-esp8266.menu.BoardModel.unowifideved.build.board=ESP8266_ARDUINO_UNOWIFI +arduino-esp8266.menu.BoardModel.unowifideved.build.extra_flags=-DF_CRYSTAL=40000000 -DESP8266 +arduino-esp8266.menu.BoardModel.unowifideved.build.variant=arduino_uart arduino-esp8266.upload.tool=esptool arduino-esp8266.upload.maximum_data_size=81920 arduino-esp8266.upload.wait_for_upload_port=true diff --git a/package/package_esp8266com_index.template.json b/package/package_esp8266com_index.template.json index d9c09d5208..a1bd0b5c27 100644 --- a/package/package_esp8266com_index.template.json +++ b/package/package_esp8266com_index.template.json @@ -359,4 +359,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/tools/boards.txt.py b/tools/boards.txt.py index 371d6bbec4..68e4d89ab2 100755 --- a/tools/boards.txt.py +++ b/tools/boards.txt.py @@ -1405,7 +1405,7 @@ def all_boards (): # standalone options if 'opts' in board: - for optname in board['opts']: + for optname in sorted(board['opts']): print(id + optname + '=' + board['opts'][optname]) # macros From 93b8bb327efac622532fd4b259e455b215ae0665 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sun, 4 Aug 2019 12:38:42 -0700 Subject: [PATCH 4/5] Re-add Python2 compatibility tweaks Most scripts can run as Python 2 or Python 3 with minimal changes, so re-add (and fix, as necessary) compatibility tweaks to the scripts. --- .../BearSSL_CertStore/certs-from-mozilla.py | 17 +++++++++++++---- package/drop_versions.py | 1 + package/merge_packages.py | 3 ++- tests/device/libraries/BSTest/runner.py | 1 + tests/device/test_BearSSL/make_spiffs.py | 17 +++++++++++++---- tools/boards.txt.py | 1 + tools/build.py | 1 + tools/elf2bin.py | 1 + tools/get.py | 7 ++++++- 9 files changed, 39 insertions(+), 10 deletions(-) diff --git a/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py b/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py index c583da3812..e423bc25b9 100755 --- a/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py +++ b/libraries/ESP8266WiFi/examples/BearSSL_CertStore/certs-from-mozilla.py @@ -7,12 +7,19 @@ # and use them for your outgoing SSL connections. # # Script by Earle F. Philhower, III. Released to the public domain. - +from __future__ import print_function import csv import os +import sys from subprocess import Popen, PIPE, call -from urllib.request import urlopen -from io import StringIO +try: + from urllib.request import urlopen +except: + from urllib2 import urlopen +try: + from StringIO import StringIO +except: + from io import StringIO # Mozilla's URL for the CSV file with included PEM certs mozurl = "https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV" @@ -21,7 +28,9 @@ names = [] pems = [] response = urlopen(mozurl) -csvData = response.read().decode('utf-8') +csvData = response.read() +if sys.version_info[0] > 2: + csvData = csvData.decode('utf-8') csvFile = StringIO(csvData) csvReader = csv.reader(csvFile) for row in csvReader: diff --git a/package/drop_versions.py b/package/drop_versions.py index c5659c8fe2..6acd5a9456 100755 --- a/package/drop_versions.py +++ b/package/drop_versions.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # This script drops one or multiple versions of a release # +from __future__ import print_function import json import sys from collections import OrderedDict diff --git a/package/merge_packages.py b/package/merge_packages.py index e3732920c4..1e0b1fb279 100755 --- a/package/merge_packages.py +++ b/package/merge_packages.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 # This script merges two Arduino Board Manager package json files. # Usage: -# python3 merge_packages.py package_esp8266com_index.json version/new/package_esp8266com_index.json +# python merge_packages.py package_esp8266com_index.json version/new/package_esp8266com_index.json # Written by Ivan Grokhotkov, 2015 # +from __future__ import print_function import json import sys diff --git a/tests/device/libraries/BSTest/runner.py b/tests/device/libraries/BSTest/runner.py index 21ecb68a7e..425ac2a422 100644 --- a/tests/device/libraries/BSTest/runner.py +++ b/tests/device/libraries/BSTest/runner.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from __future__ import print_function import pexpect from pexpect import EOF, TIMEOUT, fdpexpect import sys diff --git a/tests/device/test_BearSSL/make_spiffs.py b/tests/device/test_BearSSL/make_spiffs.py index c583da3812..e423bc25b9 100755 --- a/tests/device/test_BearSSL/make_spiffs.py +++ b/tests/device/test_BearSSL/make_spiffs.py @@ -7,12 +7,19 @@ # and use them for your outgoing SSL connections. # # Script by Earle F. Philhower, III. Released to the public domain. - +from __future__ import print_function import csv import os +import sys from subprocess import Popen, PIPE, call -from urllib.request import urlopen -from io import StringIO +try: + from urllib.request import urlopen +except: + from urllib2 import urlopen +try: + from StringIO import StringIO +except: + from io import StringIO # Mozilla's URL for the CSV file with included PEM certs mozurl = "https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReportPEMCSV" @@ -21,7 +28,9 @@ names = [] pems = [] response = urlopen(mozurl) -csvData = response.read().decode('utf-8') +csvData = response.read() +if sys.version_info[0] > 2: + csvData = csvData.decode('utf-8') csvFile = StringIO(csvData) csvReader = csv.reader(csvFile) for row in csvReader: diff --git a/tools/boards.txt.py b/tools/boards.txt.py index 68e4d89ab2..42e86d7115 100755 --- a/tools/boards.txt.py +++ b/tools/boards.txt.py @@ -32,6 +32,7 @@ # 512K/1M/2M/4M/8M/16M: menus for flash & SPIFFS size # lwip/lwip2 menus for available lwip versions +from __future__ import print_function import os import sys import collections diff --git a/tools/build.py b/tools/build.py index 80ddcbb23d..a0ee7a92d4 100755 --- a/tools/build.py +++ b/tools/build.py @@ -20,6 +20,7 @@ # # +from __future__ import print_function import sys import os import argparse diff --git a/tools/elf2bin.py b/tools/elf2bin.py index e661163e2c..e4423176e0 100755 --- a/tools/elf2bin.py +++ b/tools/elf2bin.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from __future__ import print_function import argparse import re import os diff --git a/tools/get.py b/tools/get.py index 859d5e782e..9a85ae84b4 100755 --- a/tools/get.py +++ b/tools/get.py @@ -3,6 +3,7 @@ # Tools list is obtained from package/package_esp8266com_index.template.json file. # Written by Ivan Grokhotkov, 2015. # +from __future__ import print_function import os import shutil import errno @@ -14,7 +15,11 @@ import tarfile import zipfile import re -from urllib.request import urlretrieve +if sys.version_info[0] == 3: + from urllib.request import urlretrieve +else: + # Not Python 3 - today, it is most likely to be Python 2 + from urllib import urlretrieve dist_dir = 'dist/' From f0a2b0501bf8e7f1f81327414b8d238a84fd4478 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Tue, 20 Aug 2019 08:14:51 -0700 Subject: [PATCH 5/5] Fix merge problem --- boards.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/boards.txt b/boards.txt index d23ee2b12a..de05a9f486 100644 --- a/boards.txt +++ b/boards.txt @@ -770,8 +770,6 @@ espduino.menu.UploadTool.espota.upload.tool=espota espduino.menu.UploadTool.esptool=Serial espduino.menu.UploadTool.esptool.upload.tool=esptool espduino.menu.UploadTool.esptool.upload.verbose=--trace -espduino.menu.UploadTool.espota=OTA -espduino.menu.UploadTool.espota.upload.tool=espota espduino.upload.tool=esptool espduino.upload.maximum_data_size=81920 espduino.upload.wait_for_upload_port=true