autoload changed scripts using watchdog; see #74

Author: craigbarratt

custom_components/pyscript/__init__.py

@@ -1,11 +1,16 @@
 """Component to allow running Python scripts."""
 
+import asyncio
 import glob
 import json
 import logging
 import os
+import time
+import traceback
 
 import voluptuous as vol
+from watchdog.events import DirModifiedEvent, FileSystemEventHandler
+import watchdog.observers
 
 from homeassistant.config import async_hass_config_yaml
 from homeassistant.config_entries import SOURCE_IMPORT
@@ -29,6 +34,8 @@
     LOGGER_PATH,
     SERVICE_JUPYTER_KERNEL_START,
     UNSUB_LISTENERS,
+    WATCHDOG_OBSERVER,
+    WATCHDOG_TASK,
 )
 from .eval import AstEval
 from .event import Event
@@ -109,6 +116,87 @@ def start_global_contexts(global_ctx_only=None):
         global_ctx.start()
 
 
+async def watchdog_start(hass, pyscript_folder, reload_scripts_handler):
+    """Start watchdog thread to look for changed files in pyscript_folder."""
+    if WATCHDOG_OBSERVER in hass.data[DOMAIN]:
+        return
+
+    class WatchDogHandler(FileSystemEventHandler):
+        """Class for handling watchdog events."""
+
+        def __init__(self, watchdog_q):
+            self.watchdog_q = watchdog_q
+
+        def process(self, event):
+            """Send watchdog events to main loop task."""
+            _LOGGER.debug("watchdog process(%s)", event)
+            hass.loop.call_soon_threadsafe(self.watchdog_q.put_nowait, event)
+
+        def on_modified(self, event):
+            """File modified."""
+            self.process(event)
+
+        def on_moved(self, event):
+            """File moved."""
+            self.process(event)
+
+        def on_created(self, event):
+            """File created."""
+            self.process(event)
+
+        def on_deleted(self, event):
+            """File deleted."""
+            self.process(event)
+
+    async def task_watchdog(watchdog_q):
+        def check_event(event, do_reload):
+            """Check if event should trigger a reload."""
+            if event.is_directory:
+                # don't reload if it's just a directory modified
+                if isinstance(event, DirModifiedEvent):
+                    return do_reload
+                return True
+            # only reload if it's a script or requirements.txt file
+            if event.src_path.endswith(".py") or event.src_path.endswith("/requirements.txt"):
+                return True
+            return do_reload
+
+        while True:
+            try:
+                #
+                # since some file/dir changes create multiple events, we consume all
+                # events in a small window; first # wait indefinitely for next event
+                #
+                do_reload = check_event(await watchdog_q.get(), False)
+                #
+                # now consume all additional events with 50ms timeout or 500ms elapsed
+                #
+                t_start = time.monotonic()
+                while time.monotonic() - t_start < 0.5:
+                    try:
+                        do_reload = check_event(
+                            await asyncio.wait_for(watchdog_q.get(), timeout=0.05), do_reload
+                        )
+                    except asyncio.TimeoutError:
+                        break
+                if do_reload:
+                    await reload_scripts_handler(None)
+
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                _LOGGER.error("task_watchdog: got exception %s", traceback.format_exc(-1))
+
+    watchdog_q = asyncio.Queue(0)
+    observer = watchdog.observers.Observer()
+    if observer is not None:
+        # don't run watchdog when we are testing (it patches to None)
+        hass.data[DOMAIN][WATCHDOG_OBSERVER] = observer
+        hass.data[DOMAIN][WATCHDOG_TASK] = Function.create_task(task_watchdog(watchdog_q))
+
+        observer.schedule(WatchDogHandler(watchdog_q), pyscript_folder, recursive=True)
+
+
 async def async_setup_entry(hass, config_entry):
     """Initialize the pyscript config entry."""
     if Function.hass:
@@ -148,7 +236,7 @@ async def reload_scripts_handler(call):
 
         await State.get_service_params()
 
-        global_ctx_only = call.data.get("global_ctx", None)
+        global_ctx_only = call.data.get("global_ctx", None) if call else None
 
         await install_requirements(hass, config_entry, pyscript_folder)
         await load_scripts(hass, config_entry.data, global_ctx_only=global_ctx_only)
@@ -210,8 +298,19 @@ async def hass_started(event):
         await State.get_service_params()
         hass.data[DOMAIN][UNSUB_LISTENERS].append(hass.bus.async_listen(EVENT_STATE_CHANGED, state_changed))
         start_global_contexts()
+        if WATCHDOG_OBSERVER in hass.data[DOMAIN]:
+            observer = hass.data[DOMAIN][WATCHDOG_OBSERVER]
+            observer.start()
 
     async def hass_stop(event):
+        if WATCHDOG_OBSERVER in hass.data[DOMAIN]:
+            observer = hass.data[DOMAIN][WATCHDOG_OBSERVER]
+            observer.stop()
+            observer.join()
+            del hass.data[DOMAIN][WATCHDOG_OBSERVER]
+            Function.reaper_cancel(hass.data[DOMAIN][WATCHDOG_TASK])
+            del hass.data[DOMAIN][WATCHDOG_TASK]
+
         _LOGGER.debug("stopping global contexts")
         await unload_scripts(unload_all=True)
         # sync with waiter, and then tell waiter and reaper tasks to exit
@@ -225,6 +324,8 @@ async def hass_stop(event):
     )
     hass.data[DOMAIN][UNSUB_LISTENERS].append(hass.bus.async_listen(EVENT_HOMEASSISTANT_STOP, hass_stop))
 
+    await watchdog_start(hass, pyscript_folder, reload_scripts_handler)
+
     return True
 
 

custom_components/pyscript/const.py

@@ -24,6 +24,9 @@
 REQUIREMENTS_FILE = "requirements.txt"
 REQUIREMENTS_PATHS = ("", "apps/*", "modules/*")
 
+WATCHDOG_OBSERVER = "watch_dog_observer"
+WATCHDOG_TASK = "watch_dog_task"
+
 ALLOWED_IMPORTS = {
     "black",
     "cmath",

custom_components/pyscript/manifest.json

@@ -4,7 +4,7 @@
   "config_flow": true,
   "documentation": "https://github.com/custom-components/pyscript",
   "issue_tracker": "https://github.com/custom-components/pyscript/issues",
-  "requirements": ["croniter==0.3.34"],
+  "requirements": ["croniter==0.3.34", "watchdog==0.8.3"],
   "ssdp": [],
   "zeroconf": [],
   "homekit": {},

tests/test_apps_modules.py

@@ -164,6 +164,8 @@ def glob_side_effect(path, recursive=None):
         "custom_components.pyscript.open", mock_open
     ), patch(
         "homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
     ), patch(

tests/test_config_flow.py

@@ -272,7 +272,9 @@ async def test_options_flow_user_no_change(hass, pyscript_bypass_setup):
 
 async def test_config_entry_reload(hass):
     """Test that config entry reload does not duplicate listeners."""
-    with patch("homeassistant.config.load_yaml_config_file", return_value={}):
+    with patch("homeassistant.config.load_yaml_config_file", return_value={}), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
+    ):
         result = await hass.config_entries.flow.async_init(
             DOMAIN,
             context={"source": SOURCE_USER},

tests/test_decorator_errors.py

@@ -25,6 +25,8 @@ async def setup_script(hass, notify_q, now, source):
         "homeassistant.config.load_yaml_config_file", return_value={}
     ), patch(
         "custom_components.pyscript.open", mock_open(read_data=source)
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
     ), patch(

tests/test_function.py

@@ -135,6 +135,8 @@ def glob_side_effect(path, recursive=None):
         "homeassistant.config.load_yaml_config_file", return_value=config
     ), patch(
         "custom_components.pyscript.install_requirements", return_value=None,
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
     ), patch(

tests/test_init.py

@@ -34,6 +34,8 @@ async def setup_script(hass, notify_q, now, source):
         "custom_components.pyscript.open", mock_open(read_data=source)
     ), patch(
         "homeassistant.config.load_yaml_config_file", return_value={}
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
     ), patch(
@@ -70,7 +72,9 @@ async def test_setup_makedirs_on_no_dir(hass, caplog):
     """Test setup calls os.makedirs when no dir found."""
     with patch("custom_components.pyscript.os.path.isdir", return_value=False), patch(
         "custom_components.pyscript.os.makedirs"
-    ) as makedirs_call, patch("homeassistant.config.load_yaml_config_file", return_value={}):
+    ), patch("custom_components.pyscript.watchdog_start", return_value=None) as makedirs_call, patch(
+        "homeassistant.config.load_yaml_config_file", return_value={}
+    ):
         res = await async_setup_component(hass, "pyscript", {DOMAIN: {}})
 
     assert res

tests/test_jupyter.py

@@ -139,6 +139,8 @@ def glob_side_effect(path, recursive=None):
         "homeassistant.config.load_yaml_config_file", return_value={}
     ), patch(
         "custom_components.pyscript.install_requirements", return_value=None,
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
     ), patch(

tests/test_reload.py

@@ -106,6 +106,8 @@ def glob_side_effect(path, recursive=None):
         "homeassistant.util.yaml.loader.open", mock_open
     ), patch(
         "homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
     ), patch(

tests/test_tasks.py

@@ -118,6 +118,8 @@ def glob_side_effect(path, recursive=None):
         "homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.global_ctx.os.path.getmtime", return_value=1000
     ), patch(

tests/test_unique.py

@@ -25,6 +25,8 @@ async def setup_script(hass, notify_q, now, source):
         "custom_components.pyscript.trigger.dt_now", return_value=now
     ), patch(
         "homeassistant.config.load_yaml_config_file", return_value={}
+    ), patch(
+        "custom_components.pyscript.watchdog_start", return_value=None
     ), patch(
         "custom_components.pyscript.os.path.getmtime", return_value=1000
     ), patch(