Include mpy-cross '__init__.py' In Packages; Restructure Package Bundling

circuitpython_build_tools/build.py

@@ -100,31 +100,64 @@ def _munge_to_temp(original_path, temp_file, library_version):
                 temp_file.write(line.encode("utf-8") + b"\r\n")
     temp_file.flush()
 
-def library(library_path, output_directory, mpy_cross=None, example_bundle=False):
+def library(library_path, output_directory, mpy_cross=None, example_bundle=False, pkg_folder_prefix=None):
     py_files = []
     package_files = []
     example_files = []
     total_size = 512
     for filename in os.listdir(library_path):
         full_path = os.path.join(library_path, filename)
-        if os.path.isdir(full_path) and filename not in ["docs"]:
-            files = os.listdir(full_path)
-            files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), files)
+        if os.path.isdir(full_path):
+            path_walk = [names for names in os.walk(full_path)]
+            #print("- '{}' walk: {}".format(filename, path_walk))
+
+            # iterate through path_walk, appending each file to
+            # 'walked_files' while retaining subdirectory structure
+            walked_files = []
+            for path in path_walk:
+                path_tail_idx = path[0].rfind("/") + 1
+                path_tail = path[0][path_tail_idx:]
+                rel_path = ""
+                # if this entry is the package top dir, keep it
+                # empty so we don't double append the dir name
+                if filename not in path_tail:
+                    rel_path = "{}/".format(path_tail)
+
+                for path_files in path[2]:
+                    walked_files.append("{}{}".format(rel_path, path_files))
+            #print(" - expanded file walk: {}".format(walked_files))
+
+            files = filter(lambda x: x.endswith(".py") or x.startswith("font5x8.bin"), walked_files)
             files = map(lambda x: os.path.join(filename, x), files)
+
             if filename.startswith("examples"):
                 example_files.extend(files)
+                #print("- example files: {}".format(example_files))
             else:
+                if pkg_folder_prefix:
+                    if (not example_bundle and
+                        not filename.startswith(pkg_folder_prefix)):
+                            #print("skipped path: {}".format(full_path))
+                            continue
                 if not example_bundle:
                     package_files.extend(files)
+                    #print("- package files: {} | {}".format(filename, package_files))
+
         if (filename.endswith(".py") and
            filename not in IGNORE_PY and
            not example_bundle):
-            py_files.append(filename)
+                py_files.append(filename)
 
     if len(py_files) > 1:
         raise ValueError("Multiple top level py files not allowed. Please put them in a package "
                          "or combine them into a single file.")
 
+    for fn in example_files:
+        base_dir = os.path.join(output_directory.replace("/lib", "/"), os.path.dirname(fn))
+        if not os.path.isdir(base_dir):
+            os.makedirs(base_dir)
+            total_size += 512
+
     for fn in package_files:
         base_dir = os.path.join(output_directory, os.path.dirname(fn))
         if not os.path.isdir(base_dir):
@@ -163,9 +196,7 @@ def library(library_path, output_directory, mpy_cross=None, example_bundle=False
         full_path = os.path.join(library_path, filename)
         with tempfile.NamedTemporaryFile() as temp_file:
             _munge_to_temp(full_path, temp_file, library_version)
-            if (not mpy_cross or
-                    os.stat(full_path).st_size == 0 or
-                    filename.endswith("__init__.py")):
+            if not mpy_cross or os.stat(full_path).st_size == 0:
                 output_file = os.path.join(output_directory, filename)
                 shutil.copyfile(temp_file.name, output_file)
             else:

circuitpython_build_tools/scripts/build_bundles.py

@@ -49,7 +49,8 @@ def add_file(bundle, src_file, zip_name):
 
 
 def build_bundle(libs, bundle_version, output_filename,
-        build_tools_version="devel", mpy_cross=None, example_bundle=False):
+        build_tools_version="devel", mpy_cross=None, example_bundle=False,
+        pkg_folder_prefix=None):
     build_dir = "build-" + os.path.basename(output_filename)
     top_folder = os.path.basename(output_filename).replace(".zip", "")
     build_lib_dir = os.path.join(build_dir, top_folder, "lib")
@@ -69,8 +70,8 @@ def build_bundle(libs, bundle_version, output_filename,
     success = True
     for library_path in libs:
         try:
-            build.library(library_path, build_lib_dir, mpy_cross=mpy_cross,
-                          example_bundle=example_bundle)
+            build.library(library_path, build_lib_dir,  pkg_folder_prefix=pkg_folder_prefix,
+                          mpy_cross=mpy_cross, example_bundle=example_bundle)
         except ValueError as e:
             print("build.library failure:", library_path)
             print(e)
@@ -135,7 +136,8 @@ def _find_libraries(current_path, depth):
 @click.option('--output_directory', default="bundles", help="Output location for the zip files.")
 @click.option('--library_location', required=True, help="Location of libraries to bundle.")
 @click.option('--library_depth', default=0, help="Depth of library folders. This is useful when multiple libraries are bundled together but are initially in separate subfolders.")
-def build_bundles(filename_prefix, output_directory, library_location, library_depth):
+@click.option('--package_folder_prefix', default=None, required=False, help="Prefix string used to determine package folders to bundle.")
+def build_bundles(filename_prefix, output_directory, library_location, library_depth, pkg_folder_prefix):
     os.makedirs(output_directory, exist_ok=True)
 
     bundle_version = build.version_string()
@@ -158,6 +160,7 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
         filename_prefix + '-py-{VERSION}.zip'.format(
             VERSION=bundle_version))
     build_bundle(libs, bundle_version, zip_filename,
+                 pkg_folder_prefix=pkg_folder_prefix,
                  build_tools_version=build_tools_version)
 
     # Build .mpy bundle(s)
@@ -175,7 +178,8 @@ def build_bundles(filename_prefix, output_directory, library_location, library_d
                 TAG=version["name"],
                 VERSION=bundle_version))
         build_bundle(libs, bundle_version, zip_filename, mpy_cross=mpy_cross,
-                     build_tools_version=build_tools_version)
+                     build_tools_version=build_tools_version,
+                     pkg_folder_prefix=pkg_folder_prefix)
 
     # Build example bundle
     zip_filename = os.path.join(output_directory,