From d8c18da0b6aa889ba792c18a861d4277c1d0c6f6 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Thu, 2 May 2019 12:09:15 -0500 Subject: [PATCH 1/5] remove mpy-cross exclusion for '__init__.py' in packaged modules --- circuitpython_build_tools/build.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 49244e2..e976dec 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -163,9 +163,7 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False full_path = os.path.join(library_path, filename) with tempfile.NamedTemporaryFile() as temp_file: _munge_to_temp(full_path, temp_file, library_version) - if (not mpy_cross or - os.stat(full_path).st_size == 0 or - filename.endswith("__init__.py")): + if not mpy_cross or os.stat(full_path).st_size == 0: output_file = os.path.join(output_directory, filename) shutil.copyfile(temp_file.name, output_file) else: From c38f93dcc4b1cbf8b5c63dd812e54dba9de076b8 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Thu, 2 May 2019 13:15:32 -0500 Subject: [PATCH 2/5] restructure package file bundling; uses folder name prefix string --- circuitpython_build_tools/build.py | 12 ++++++++++-- circuitpython_build_tools/scripts/build_bundles.py | 14 +++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index e976dec..717f3a1 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -100,22 +100,30 @@ def _munge_to_temp(original_path, temp_file, library_version): temp_file.write(line.encode("utf-8") + b"\r\n") temp_file.flush() -def library(library_path, output_directory, mpy_cross=None, example_bundle=False): +def library(library_path, output_directory, mpy_cross=None, example_bundle=False, pkg_folder_prefix=None): py_files = [] package_files = [] example_files = [] total_size = 512 for filename in os.listdir(library_path): full_path = os.path.join(library_path, filename) - if os.path.isdir(full_path) and filename not in ["docs"]: + if os.path.isdir(full_path): files = os.listdir(full_path) files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), files) files = map(lambda x: os.path.join(filename, x), files) if filename.startswith("examples"): example_files.extend(files) + #print(" - example files: {}".format(example_files)) else: + if pkg_folder_prefix: + if (not example_bundle and + not filename.startswith(pkg_folder_prefix)): + print("skipped path: {}".format(full_path)) + continue if not example_bundle: package_files.extend(files) + #print(" - package files: {}".format(package_files)) + if (filename.endswith(".py") and filename not in IGNORE_PY and not example_bundle): diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index 057c1ee..ec9080d 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -49,7 +49,8 @@ def add_file(bundle, src_file, zip_name): def build_bundle(libs, bundle_version, output_filename, - build_tools_version="devel", mpy_cross=None, example_bundle=False): + build_tools_version="devel", mpy_cross=None, example_bundle=False, + pkg_folder_prefix=None): build_dir = "build-" + os.path.basename(output_filename) top_folder = os.path.basename(output_filename).replace(".zip", "") build_lib_dir = os.path.join(build_dir, top_folder, "lib") @@ -69,8 +70,8 @@ def build_bundle(libs, bundle_version, output_filename, success = True for library_path in libs: try: - build.library(library_path, build_lib_dir, mpy_cross=mpy_cross, - example_bundle=example_bundle) + build.library(library_path, build_lib_dir, pkg_folder_prefix=pkg_folder_prefix, + mpy_cross=mpy_cross, example_bundle=example_bundle) except ValueError as e: print("build.library failure:", library_path) print(e) @@ -135,7 +136,8 @@ def _find_libraries(current_path, depth): @click.option('--output_directory', default="bundles", help="Output location for the zip files.") @click.option('--library_location', required=True, help="Location of libraries to bundle.") @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.") -def build_bundles(filename_prefix, output_directory, library_location, library_depth): +@click.option('--package_folder_prefix', default=None, required=False, help="Prefix string used to determine package folders to bundle.") +def build_bundles(filename_prefix, output_directory, library_location, library_depth, pkg_folder_prefix): os.makedirs(output_directory, exist_ok=True) bundle_version = build.version_string() @@ -158,6 +160,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d filename_prefix + '-py-{VERSION}.zip'.format( VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, + pkg_folder_prefix=pkg_folder_prefix, build_tools_version=build_tools_version) # Build .mpy bundle(s) @@ -175,7 +178,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d TAG=version["name"], VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross, - build_tools_version=build_tools_version) + build_tools_version=build_tools_version, + pkg_folder_prefix=pkg_folder_prefix) # Build example bundle zip_filename = os.path.join(output_directory, From 92138a56fdf667a634d7f7bfe66b83f4a3dbaace Mon Sep 17 00:00:00 2001 From: sommersoft Date: Thu, 2 May 2019 21:35:26 -0500 Subject: [PATCH 3/5] include package file based on package name prefix --- .../scripts/build_bundles.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index ec9080d..dc5e2f2 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -109,8 +109,8 @@ def build_bundle(libs, bundle_version, output_filename, with zipfile.ZipFile(output_filename, 'w') as bundle: build_metadata = {"build-tools-version": build_tools_version} bundle.comment = json.dumps(build_metadata).encode("utf-8") - if multiple_libs: - total_size += add_file(bundle, "README.txt", os.path.join(top_folder, "README.txt")) + #if multiple_libs: + # total_size += add_file(bundle, "README.txt", os.path.join(top_folder, "README.txt")) for root, dirs, files in os.walk(build_dir): ziproot = root[len(build_dir + "/"):] for filename in files: @@ -187,3 +187,14 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, build_tools_version=build_tools_version, example_bundle=True) + +# git ignore +if __name__ == "__main__": + + build_bundles( + "test", + "/home/sommersoft/Dev/cpy-build-tools/.bundles", + "/home/sommersoft/Dev/circuitpython_libs/Adafruit_CircuitPython_Bundle/libraries", + 2, + "adafruit_" + ) From 732c144b3625abd4af4bba0b01257f9139983c55 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Thu, 2 May 2019 21:40:31 -0500 Subject: [PATCH 4/5] remove local debuggy stuff that snuck in --- .../scripts/build_bundles.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/circuitpython_build_tools/scripts/build_bundles.py b/circuitpython_build_tools/scripts/build_bundles.py index dc5e2f2..ec9080d 100755 --- a/circuitpython_build_tools/scripts/build_bundles.py +++ b/circuitpython_build_tools/scripts/build_bundles.py @@ -109,8 +109,8 @@ def build_bundle(libs, bundle_version, output_filename, with zipfile.ZipFile(output_filename, 'w') as bundle: build_metadata = {"build-tools-version": build_tools_version} bundle.comment = json.dumps(build_metadata).encode("utf-8") - #if multiple_libs: - # total_size += add_file(bundle, "README.txt", os.path.join(top_folder, "README.txt")) + if multiple_libs: + total_size += add_file(bundle, "README.txt", os.path.join(top_folder, "README.txt")) for root, dirs, files in os.walk(build_dir): ziproot = root[len(build_dir + "/"):] for filename in files: @@ -187,14 +187,3 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d VERSION=bundle_version)) build_bundle(libs, bundle_version, zip_filename, build_tools_version=build_tools_version, example_bundle=True) - -# git ignore -if __name__ == "__main__": - - build_bundles( - "test", - "/home/sommersoft/Dev/cpy-build-tools/.bundles", - "/home/sommersoft/Dev/circuitpython_libs/Adafruit_CircuitPython_Bundle/libraries", - 2, - "adafruit_" - ) From f803ba57a37362695d7e99039b8bf4d8ee996c41 Mon Sep 17 00:00:00 2001 From: sommersoft Date: Mon, 6 May 2019 10:44:22 -0500 Subject: [PATCH 5/5] build.py->library: refactor file selection, so it includes subfolders --- circuitpython_build_tools/build.py | 39 ++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/circuitpython_build_tools/build.py b/circuitpython_build_tools/build.py index 717f3a1..6bedd9e 100644 --- a/circuitpython_build_tools/build.py +++ b/circuitpython_build_tools/build.py @@ -108,31 +108,56 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False for filename in os.listdir(library_path): full_path = os.path.join(library_path, filename) if os.path.isdir(full_path): - files = os.listdir(full_path) - files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), files) + path_walk = [names for names in os.walk(full_path)] + #print("- '{}' walk: {}".format(filename, path_walk)) + + # iterate through path_walk, appending each file to + # 'walked_files' while retaining subdirectory structure + walked_files = [] + for path in path_walk: + path_tail_idx = path[0].rfind("/") + 1 + path_tail = path[0][path_tail_idx:] + rel_path = "" + # if this entry is the package top dir, keep it + # empty so we don't double append the dir name + if filename not in path_tail: + rel_path = "{}/".format(path_tail) + + for path_files in path[2]: + walked_files.append("{}{}".format(rel_path, path_files)) + #print(" - expanded file walk: {}".format(walked_files)) + + files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), walked_files) files = map(lambda x: os.path.join(filename, x), files) + if filename.startswith("examples"): example_files.extend(files) - #print(" - example files: {}".format(example_files)) + #print("- example files: {}".format(example_files)) else: if pkg_folder_prefix: if (not example_bundle and not filename.startswith(pkg_folder_prefix)): - print("skipped path: {}".format(full_path)) - continue + #print("skipped path: {}".format(full_path)) + continue if not example_bundle: package_files.extend(files) - #print(" - package files: {}".format(package_files)) + #print("- package files: {} | {}".format(filename, package_files)) if (filename.endswith(".py") and filename not in IGNORE_PY and not example_bundle): - py_files.append(filename) + py_files.append(filename) if len(py_files) > 1: raise ValueError("Multiple top level py files not allowed. Please put them in a package " "or combine them into a single file.") + for fn in example_files: + base_dir = os.path.join(output_directory.replace("/lib", "/"), os.path.dirname(fn)) + if not os.path.isdir(base_dir): + os.makedirs(base_dir) + total_size += 512 + for fn in package_files: base_dir = os.path.join(output_directory, os.path.dirname(fn)) if not os.path.isdir(base_dir):