Improve Caching Schema (#50)

- Use Django async caching API
- Remove async_lru_cache.
- Fix #46 
- Use aiofile for async file operations.
- Protect against malicious file paths
- Use cache versioning to invalidate old web module files stored in cache

Author: Archmonger

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ["3.8", "3.9", "3.10"]
     steps:
       - uses: actions/checkout@v2
       - uses: nanasess/setup-chromedriver@master

README.md

@@ -106,20 +106,14 @@ You may configure additional options as well:
 # the base URL for all IDOM-releated resources
 IDOM_BASE_URL: str = "_idom/"
 
-# Set cache size limit for loading JS files for IDOM.
-# Only applies when not using Django's caching framework (see below).
-IDOM_WEB_MODULE_LRU_CACHE_SIZE: int | None = None
-
 # Maximum seconds between two reconnection attempts that would cause the client give up.
 # 0 will disable reconnection.
 IDOM_WS_MAX_RECONNECT_DELAY: int = 604800
 
 # Configure a cache for loading JS files
 CACHES = {
-  # Configure a cache for loading JS files for IDOM
-  "idom_web_modules": {"BACKEND": ...},
-  # If the above cache is not configured, then we'll use the "default" instead
-  "default": {"BACKEND": ...},
+  # If "idom" cache is not configured, then we'll use the "default" instead
+  "idom": {"BACKEND": ...},
 }
 ```
 

requirements/pkg-deps.txt

@@ -1,2 +1,3 @@
 channels <4.0.0
 idom >=0.34.0, <0.35.0
+aiofile >=3.0, <4.0

setup.py

@@ -38,7 +38,7 @@ def list2cmdline(cmd_list):
 
 package = {
     "name": name,
-    "python_requires": ">=3.7",
+    "python_requires": ">=3.8",
     "packages": find_packages(str(src_dir)),
     "package_dir": {"": "src"},
     "description": "Control the web with Python",
@@ -52,15 +52,14 @@ def list2cmdline(cmd_list):
     "zip_safe": False,
     "classifiers": [
         "Framework :: Django",
-        "Framework :: Django :: 3.1",
-        "Framework :: Django :: 3.2",
+        "Framework :: Django :: 4.0",
         "Operating System :: OS Independent",
         "Intended Audience :: Developers",
         "Intended Audience :: Science/Research",
         "Topic :: Multimedia :: Graphics",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Environment :: Web Environment",
     ],
 }

src/django_idom/config.py

@@ -1,7 +1,7 @@
 from typing import Dict
 
 from django.conf import settings
-from django.core.cache import DEFAULT_CACHE_ALIAS
+from django.core.cache import DEFAULT_CACHE_ALIAS, caches
 from idom.core.proto import ComponentConstructor
 
 
@@ -12,17 +12,7 @@
 IDOM_WEB_MODULES_URL = IDOM_BASE_URL + "web_module/"
 IDOM_WS_MAX_RECONNECT_DELAY = getattr(settings, "IDOM_WS_MAX_RECONNECT_DELAY", 604800)
 
-_CACHES = getattr(settings, "CACHES", {})
-if _CACHES:
-    if "idom_web_modules" in getattr(settings, "CACHES", {}):
-        IDOM_WEB_MODULE_CACHE = "idom_web_modules"
-    else:
-        IDOM_WEB_MODULE_CACHE = DEFAULT_CACHE_ALIAS
+if "idom" in getattr(settings, "CACHES", {}):
+    IDOM_CACHE = caches["idom"]
 else:
-    IDOM_WEB_MODULE_CACHE = None
-
-
-# the LRU cache size for the route serving IDOM_WEB_MODULES_DIR files
-IDOM_WEB_MODULE_LRU_CACHE_SIZE = getattr(
-    settings, "IDOM_WEB_MODULE_LRU_CACHE_SIZE", None
-)
+    IDOM_CACHE = caches[DEFAULT_CACHE_ALIAS]

src/django_idom/views.py

@@ -1,43 +1,34 @@
-import asyncio
-import functools
 import os
 
-from django.core.cache import caches
+from aiofile import async_open
+from django.core.exceptions import SuspiciousOperation
 from django.http import HttpRequest, HttpResponse
 from idom.config import IDOM_WED_MODULES_DIR
 
-from .config import IDOM_WEB_MODULE_CACHE, IDOM_WEB_MODULE_LRU_CACHE_SIZE
-
-
-if IDOM_WEB_MODULE_CACHE is None:
-
-    def async_lru_cache(*lru_cache_args, **lru_cache_kwargs):
-        def async_lru_cache_decorator(async_function):
-            @functools.lru_cache(*lru_cache_args, **lru_cache_kwargs)
-            def cached_async_function(*args, **kwargs):
-                coroutine = async_function(*args, **kwargs)
-                return asyncio.ensure_future(coroutine)
-
-            return cached_async_function
-
-        return async_lru_cache_decorator
-
-    @async_lru_cache(IDOM_WEB_MODULE_LRU_CACHE_SIZE)
-    async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse:
-        file_path = IDOM_WED_MODULES_DIR.current.joinpath(*file.split("/"))
-        return HttpResponse(file_path.read_text(), content_type="text/javascript")
-
-else:
-    _web_module_cache = caches[IDOM_WEB_MODULE_CACHE]
-
-    async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse:
-        file = IDOM_WED_MODULES_DIR.current.joinpath(*file.split("/")).absolute()
-        last_modified_time = os.stat(file).st_mtime
-        cache_key = f"{file}:{last_modified_time}"
-
-        response = _web_module_cache.get(cache_key)
-        if response is None:
-            response = HttpResponse(file.read_text(), content_type="text/javascript")
-            _web_module_cache.set(cache_key, response, timeout=None)
-
-        return response
+from .config import IDOM_CACHE
+
+
+async def web_modules_file(request: HttpRequest, file: str) -> HttpResponse:
+    """Gets JavaScript required for IDOM modules at runtime. These modules are
+    returned from cache if available."""
+    web_modules_dir = IDOM_WED_MODULES_DIR.current
+    path = web_modules_dir.joinpath(*file.split("/")).absolute()
+
+    # Prevent attempts to walk outside of the web modules dir
+    if str(web_modules_dir) != os.path.commonpath((path, web_modules_dir)):
+        raise SuspiciousOperation(
+            "Attempt to access a directory outside of IDOM_WED_MODULES_DIR."
+        )
+
+    # Fetch the file from cache, if available
+    last_modified_time = os.stat(path).st_mtime
+    cache_key = f"django_idom:web_module:{str(path).lstrip(str(web_modules_dir))}"
+    response = await IDOM_CACHE.aget(cache_key, version=last_modified_time)
+    if response is None:
+        async with async_open(path, "r") as fp:
+            response = HttpResponse(await fp.read(), content_type="text/javascript")
+        await IDOM_CACHE.adelete(cache_key)
+        await IDOM_CACHE.aset(
+            cache_key, response, timeout=None, version=last_modified_time
+        )
+    return response