From e8eeeb6670ee67ce793371cc470be31f877c5d49 Mon Sep 17 00:00:00 2001 From: Wilko Nienhaus Date: Wed, 13 Oct 2021 21:28:02 +0300 Subject: [PATCH 1/2] add setup.py to build upip installable distribution package for pypi setup.py (setuptools) uses the sdist_upip module (included in this commit) to generate packages in the correct format for use with upip. To generate a distribution package, make sure the VERSION is set as desired in setup.py and then run `python3 setup.py sdist`. The result will be a tar.gz file in the dist/ subdirectory. --- sdist_upip.py | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 30 ++++++++++ 2 files changed, 181 insertions(+) create mode 100644 sdist_upip.py create mode 100644 setup.py diff --git a/sdist_upip.py b/sdist_upip.py new file mode 100644 index 0000000..60c0c3e --- /dev/null +++ b/sdist_upip.py @@ -0,0 +1,151 @@ +# This module is part of Pycopy https://github.com/pfalcon/pycopy +# and pycopy-lib https://github.com/pfalcon/pycopy-lib, projects to +# create a (very) lightweight full-stack Python distribution. +# +# Copyright (c) 2016-2019 Paul Sokolovsky +# Licence: MIT +# +# This module overrides distutils (also compatible with setuptools) "sdist" +# command to perform pre- and post-processing as required for Pycopy's +# upip package manager. +# +# Preprocessing steps: +# * Creation of Python resource module (R.py) from each top-level package's +# resources. +# Postprocessing steps: +# * Removing metadata files not used by upip (this includes setup.py) +# * Recompressing gzip archive with 4K dictionary size so it can be +# installed even on low-heap targets. +# +import sys +import os +import zlib +from subprocess import Popen, PIPE +import glob +import tarfile +import re +import io + +from distutils.filelist import FileList +from setuptools.command.sdist import sdist as _sdist + + +def gzip_4k(inf, fname): + comp = zlib.compressobj(level=9, wbits=16 + 12) + with open(fname + ".out", "wb") as outf: + while 1: + data = inf.read(1024) + if not data: + break + outf.write(comp.compress(data)) + outf.write(comp.flush()) + os.rename(fname, fname + ".orig") + os.rename(fname + ".out", fname) + + +FILTERS = [ + # include, exclude, repeat + (r".+\.egg-info/(PKG-INFO|requires\.txt)", r"setup.py$"), + (r".+\.py$", r"[^/]+$"), + (None, r".+\.egg-info/.+"), +] + + +outbuf = io.BytesIO() + +def filter_tar(name): + fin = tarfile.open(name, "r:gz") + fout = tarfile.open(fileobj=outbuf, mode="w") + for info in fin: +# print(info) + if not "/" in info.name: + continue + fname = info.name.split("/", 1)[1] + include = None + + for inc_re, exc_re in FILTERS: + if include is None and inc_re: + if re.match(inc_re, fname): + include = True + + if include is None and exc_re: + if re.match(exc_re, fname): + include = False + + if include is None: + include = True + + if include: + print("including:", fname) + else: + print("excluding:", fname) + continue + + farch = fin.extractfile(info) + fout.addfile(info, farch) + fout.close() + fin.close() + + +def make_resource_module(manifest_files): + resources = [] + # Any non-python file included in manifest is resource + for fname in manifest_files: + ext = fname.rsplit(".", 1) + if len(ext) > 1: + ext = ext[1] + else: + ext = "" + if ext != "py": + resources.append(fname) + + if resources: + print("creating resource module R.py") + resources.sort() + last_pkg = None + r_file = None + for fname in resources: + try: + pkg, res_name = fname.split("/", 1) + except ValueError: + print("not treating %s as a resource" % fname) + continue + if last_pkg != pkg: + last_pkg = pkg + if r_file: + r_file.write("}\n") + r_file.close() + r_file = open(pkg + "/R.py", "w") + r_file.write("R = {\n") + + with open(fname, "rb") as f: + r_file.write("%r: %r,\n" % (res_name, f.read())) + + if r_file: + r_file.write("}\n") + r_file.close() + + +class sdist(_sdist): + + def run(self): + self.filelist = FileList() + self.get_file_list() + make_resource_module(self.filelist.files) + + r = super().run() + + assert len(self.archive_files) == 1 + print("filtering files and recompressing with 4K dictionary") + filter_tar(self.archive_files[0]) + outbuf.seek(0) + gzip_4k(outbuf, self.archive_files[0]) + + return r + + +# For testing only +if __name__ == "__main__": + filter_tar(sys.argv[1]) + outbuf.seek(0) + gzip_4k(outbuf, sys.argv[1]) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..77cbc9b --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +from pathlib import Path +from setuptools import setup +import sdist_upip + + +HERE = Path(__file__).parent +README = (HERE / 'README.rst').read_text() + +VERSION = "1.0.0" + +setup( + name="micropython-py-esp32-ulp", + version=VERSION, + description="Assembler toolchain for the ESP32 ULP co-processor, written in MicroPython", + long_description=README, + long_description_content_type='text/x-rst', + url="https://github.com/ThomasWaldmann/py-esp32-ulp", + license="MIT", + author="py-esp32-ulp authors", + author_email="tw@waldmann-edv.de", + maintainer="py-esp32-ulp authors", + maintainer_email="tw@waldmann-edv.de", + classifiers=[ + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: Implementation :: MicroPython', + ], + platforms=["esp32"], + cmdclass={"sdist": sdist_upip.sdist}, + packages=["esp32_ulp"], +) From 243eebad0e5b31d0ebccae33c577bf4f9e51eebc Mon Sep 17 00:00:00 2001 From: Wilko Nienhaus Date: Thu, 14 Oct 2021 08:15:40 +0300 Subject: [PATCH 2/2] add additional supported platforms It appears the "platforms" fields it not actually used by any tools but can be used to communicate to users what platforms the package is meant to work on. (https://stackoverflow.com/a/34994245) Including linux and darwin here, because the code runs well on the unix port of MicroPython, which itself runs well on Linux and macOS (darwin). In fact most of the testing had been done using the unix port. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 77cbc9b..00093a7 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: Implementation :: MicroPython', ], - platforms=["esp32"], + platforms=["esp32", "linux", "darwin"], cmdclass={"sdist": sdist_upip.sdist}, packages=["esp32_ulp"], )