Generate TZ.h from zoneinfo

Using tzdata pypi package that ships zoneinfo blobs
(can be overwritten using path args, if needed)

Can't use raw data package from IANA, need it built first

More data than the original, not limited to ZONE/...
Also, allow to override certain names and tz strings independent
of tzdata values (resolve esp8266#9007)

Author: mcspr

package/README.md

@@ -111,8 +111,8 @@ The following points assume work in a direct clone of the repository, and not in
 
    * [platform.txt](https://github.com/esp8266/Arduino/blob/master/platform.txt) and [package.json](https://github.com/esp8266/Arduino/blob/master/package.json): update `version` to the release E.g. `3.0.0`,
 
-   * [cores/esp8266/TZ.h](https://github.com/esp8266/Arduino/blob/master/cores/esp8266/TZ.h): import the latest database with the following shell command:\
-     `$ cd tools; sh TZupdate.sh`
+   * [cores/esp8266/TZ.h](https://github.com/esp8266/Arduino/blob/master/cores/esp8266/TZ.h): import the latest database:\
+     `$ cd tools; pip install -U tzdata; python format_tzdata.py --format header --output ../cores/esp8266/TZ.h`
 
    * Update SSL/TLS certificates and public keys in examples:\
      `$ cd tools; sh certsUpdate.sh`

tools/format_tzdata.py

@@ -0,0 +1,185 @@
+# this script refreshes world timezone definitions in
+#   cores/esp8266/TZ.h
+#
+# use the file output argument or stdout redirect to overwrite the target file
+
+import argparse
+import contextlib
+import datetime
+import mmap
+import os
+import pathlib
+import re
+import sys
+
+from importlib import resources
+
+import tzdata  # https://tzdata.readthedocs.io/en/latest/
+
+
+def tzdata_version():
+    return f"Generated with *{tzdata.IANA_VERSION=} {tzdata.__version__=}*"
+
+
+def known_tz(entry):
+    swaps = {
+        "Europe/Simferopol": "EET-2EEST,M3.5.0/3,M10.5.0/4",
+    }
+
+    return swaps.get(entry, None)
+
+
+def known_alias(entry):
+    swaps = {
+        "Europe/Zaporozhye": "Europe/Zaporizhzhia",
+        "Europe/Uzhgorod": "Europe/Uzhhorod",
+    }
+
+    return swaps.get(entry)
+
+
+def fix_name(name):
+    swaps = [["-", "m"], ["+", "p"], ["/", "_"]]
+
+    for lhs, rhs in swaps:
+        name = name.replace(lhs, rhs)
+
+    return name
+
+
+def fix_value(value):
+    return re.sub(r"<[^>]*>", "UNK", value)
+
+
+def utc_alias(zone):
+    return zone in (
+        "Universal",
+        "UTC",
+        "UCT",
+        "Zulu",
+        "GMT",
+        "GMT+0",
+        "GMT-0",
+        "GMT0",
+        "Greenwich",
+    )
+
+
+def tzdata_resource_from_name(name):
+    pair = name.rsplit("/", 1)
+    if len(pair) == 1:
+        return resources.files("tzdata.zoneinfo") / pair[0]
+
+    return resources.files(f'tzdata.zoneinfo.{pair[0].replace("/", ".")}') / pair[1]
+
+
+def make_zones_list(zones):
+    with open(zones, "r") as f:
+        zones = [zone.strip() for zone in f.readlines()]
+
+    return zones
+
+
+def make_zones(args):
+    out = []
+
+    for zone in make_zones_list(args.zones):
+        target = tzdata_resource_from_name(zone)
+        with tzdata_resource_from_name(zone).open("rb") as f:
+            magic = f.read(4)
+            if magic != b"TZif":
+                continue
+
+            m = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
+            newline = m.rfind(b"\n", 0, len(m) - 1)
+            if newline < 0:
+                continue
+
+            m.seek(newline + 1)
+            tz = m.readline().strip()
+            tz = tz.decode("ascii")
+
+            if alias := known_alias(zone):
+                out.append([alias, tz])
+
+            if replacement := known_tz(zone):
+                out.append([zone, replacement])
+            else:
+                out.append([zone, tz])
+
+    out.sort(key=lambda x: x[0])
+    return out
+
+
+def markdown(zones):
+    utcs = []
+    rows = []
+
+    for name, value in zones:
+        if utc_alias(name):
+            utcs.append(name)
+            continue
+
+        rows.append(f"|{name}|{fix_value(value)}|")
+
+    print("|Name|Value|")
+    print("|---|---|")
+    for name in utcs:
+        print(f"|{name}|UTC0|")
+
+    last = ""
+    for row in rows:
+        prefix, _, _ = row.partition("/")
+        if last != prefix:
+            last = prefix
+            print("|||")
+        print(row)
+    print()
+    print("---")
+    print()
+    print(tzdata_version())
+
+
+def header(zones):
+    print(f"// ! ! ! DO NOT EDIT, AUTOMATICALLY GENERATED ! ! !")
+    print(f"// File created {datetime.datetime.now()}")
+    print(f"// Based on IANA database {tzdata.IANA_VERSION}")
+    print(f"// Re-run <esp8266 arduino core>/tools/{sys.argv[0]} to update")
+    print()
+    print("#pragma once")
+    print()
+    for name, value in zones:
+        print(f'#define\tTZ_{fix_name(name)}\tPSTR("{value}")')
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--output",
+        type=argparse.FileType("w"),
+        default=sys.stdout,
+    )
+    parser.add_argument(
+        "--format",
+        default="header",
+        choices=["markdown", "header"],
+    )
+    parser.add_argument(
+        "--zones",
+        help="Zone names, one per line",
+        default=os.path.join(os.path.dirname(tzdata.__file__), "zones"),
+    )
+    parser.add_argument(
+        "--root",
+        help="Where do we get raw zoneinfo files from",
+        default=os.path.join(os.path.dirname(tzdata.__file__), "zoneinfo"),
+    )
+
+    args = parser.parse_args()
+    zones = make_zones(args)
+
+    with contextlib.redirect_stdout(args.output):
+        if args.format == "markdown":
+            markdown(zones)
+        elif args.format == "header":
+            header(zones)