From 1365c19825075c96bb0cceb428f81a2993b93910 Mon Sep 17 00:00:00 2001 From: Jason Hildebrand Date: Thu, 2 Jan 2020 16:09:03 -0600 Subject: [PATCH 1/3] Refactored plugin loading out of Application, to make it possible to load plugins in non-web situations (scripts, cron jobs, etc.). --- webware/Application.py | 39 +++------------------- webware/MockApplication.py | 67 ++++++++++++++++++++++++++++++++++++++ webware/PlugIn.py | 5 +-- webware/PlugInLoader.py | 60 ++++++++++++++++++++++++++++++++++ webware/__init__.py | 16 +++++++++ 5 files changed, 151 insertions(+), 36 deletions(-) create mode 100644 webware/MockApplication.py create mode 100644 webware/PlugInLoader.py diff --git a/webware/Application.py b/webware/Application.py index 34d2652..57b5882 100644 --- a/webware/Application.py +++ b/webware/Application.py @@ -19,8 +19,6 @@ from time import time, localtime -import pkg_resources - from MiscUtils import NoDefault from MiscUtils.Funcs import asclocaltime from MiscUtils.NamedValueAccess import valueForName @@ -32,9 +30,9 @@ from ExceptionHandler import ExceptionHandler from HTTPRequest import HTTPRequest from HTTPExceptions import HTTPException, HTTPSessionExpired -from PlugIn import PlugIn from Transaction import Transaction from WSGIStreamOut import WSGIStreamOut +from PlugInLoader import PlugInLoader import URLParser @@ -237,6 +235,7 @@ def __init__(self, path=None, settings=None, development=None): if self.setting('UseSessionSweeper'): self.startSessionSweeper() + self._plugInLoader = None self.loadPlugIns() self._wasShutDown = False @@ -1093,19 +1092,7 @@ def loadPlugIn(self, name, module): May return None if loading was unsuccessful (in which case this method prints a message saying so). Used by `loadPlugIns` (note the **s**). """ - try: - plugIn = PlugIn(self, name, module) - willNotLoadReason = plugIn.load() - if willNotLoadReason: - print(f' Plug-in {name} cannot be loaded because:\n' - f' {willNotLoadReason}') - return None - plugIn.install() - except Exception: - print() - print(f'Plug-in {name} raised exception.') - raise - return plugIn + return self._plugInLoader(name, module) def loadPlugIns(self): """Load all plug-ins. @@ -1115,24 +1102,8 @@ def loadPlugIns(self): Application at startup time, just before listening for requests. See the docs in `PlugIn` for more info. """ - plugInNames = set(self.setting('PlugIns')) - plugInNames.add('Webware') - plugIns = [ - (entry_point.name, entry_point.load()) - for entry_point - in pkg_resources.iter_entry_points('webware.plugins') - if entry_point.name in plugInNames - ] - - print('Plug-ins list:', ', '.join( - name for name, _module in plugIns if name != 'Webware')) - - # Now that we have our plug-in list, load them... - for name, module in plugIns: - plugIn = self.loadPlugIn(name, module) - if plugIn: - self._plugIns[name] = plugIn - print() + self._plugInLoader = loader = PlugInLoader(self) + self._plugIns = loader.loadPlugIns(self.setting('PlugIns')) # endregion Plug-in loading diff --git a/webware/MockApplication.py b/webware/MockApplication.py new file mode 100644 index 0000000..7e615d6 --- /dev/null +++ b/webware/MockApplication.py @@ -0,0 +1,67 @@ +import os + +from ConfigurableForServerSidePath import ConfigurableForServerSidePath + + +class MockImportManager(object): + + def recordFile(self, filename, isfile=None): + pass + + +defaultConfig = dict( + CacheDir='Cache', + PlugIns=['MiscUtils', 'WebUtils', 'TaskKit', 'UserKit', 'PSP'], +) + + +class MockApplication(ConfigurableForServerSidePath): + + def __init__(self, path=None, settings=None, development=None): + ConfigurableForServerSidePath.__init__(self) + if path is None: + path = os.getcwd() + self._serverSidePath = os.path.abspath(path) + self._webwarePath = os.path.abspath(os.path.dirname(__file__)) + if development is None: + development = bool(os.environ.get('WEBWARE_DEVELOPMENT')) + self._development = development + + appConfig = self.config() # get and cache the configuration + if settings: + appConfig.update(settings) + self._cacheDir = self.serverSidePath(self.setting('CacheDir') or 'Cache') + from MiscUtils.PropertiesObject import PropertiesObject + props = PropertiesObject(os.path.join( + self._webwarePath, 'Properties.py')) + self._webwareVersion = props['version'] + self._webwareVersionString = props['versionString'] + self._imp = MockImportManager() + for path in (self._cacheDir,): + if path and not os.path.exists(path): + os.makedirs(path) + + def defaultConfig(self): + return defaultConfig + + def configReplacementValues(self): + """Get config values that need to be escaped.""" + return dict( + ServerSidePath=self._serverSidePath, + WebwarePath=self._webwarePath, + Development=self._development) + + def configFilename(self): + return self.serverSidePath('Configs/Application.config') + + def serverSidePath(self, path=None): + if path: + return os.path.normpath( + os.path.join(self._serverSidePath, path)) + return self._serverSidePath + + def hasContext(self, context): + return False + + def addServletFactory(self, factory): + pass diff --git a/webware/PlugIn.py b/webware/PlugIn.py index 9b63931..5fa9ab2 100644 --- a/webware/PlugIn.py +++ b/webware/PlugIn.py @@ -81,13 +81,14 @@ def __init__(self, application, name, module): self._cacheDir = os.path.join(self._app._cacheDir, self._name) self._examplePages = self._examplePagesContext = None - def load(self): + def load(self, verbose=True): """Loads the plug-in into memory, but does not yet install it. Will return None on success, otherwise a message (string) that says why the plug-in could not be loaded. """ - print(f'Loading plug-in: {self._name} at {self._path}') + if verbose: + print(f'Loading plug-in: {self._name} at {self._path}') # Grab the Properties.py self._properties = PropertiesObject( diff --git a/webware/PlugInLoader.py b/webware/PlugInLoader.py new file mode 100644 index 0000000..09c6461 --- /dev/null +++ b/webware/PlugInLoader.py @@ -0,0 +1,60 @@ +import pkg_resources + +from PlugIn import PlugIn + + +class PlugInLoader(object): + + def __init__(self, app): + self.app = app + self._plugIns = {} + + def loadPlugIn(self, name, module, verbose=True): + """Load and return the given plug-in. + + May return None if loading was unsuccessful (in which case this method + prints a message saying so). Used by `loadPlugIns` (note the **s**). + """ + try: + plugIn = PlugIn(self.app, name, module) + willNotLoadReason = plugIn.load(verbose=verbose) + if willNotLoadReason: + print(f' Plug-in {name} cannot be loaded because:\n' + f' {willNotLoadReason}') + return None + plugIn.install() + except Exception: + print() + print(f'Plug-in {name} raised exception.') + raise + return plugIn + + def loadPlugIns(self, plugInNames, verbose=True): + """Load all plug-ins. + + A plug-in allows you to extend the functionality of Webware without + necessarily having to modify its source. Plug-ins are loaded by + Application at startup time, just before listening for requests. + See the docs in `PlugIn` for more info. + """ + plugInNames = set(plugInNames) + plugInNames.add('Webware') + plugIns = [ + (entry_point.name, entry_point.load()) + for entry_point + in pkg_resources.iter_entry_points('webware.plugins') + if entry_point.name in plugInNames + ] + + if verbose: + print('Plug-ins list:', ', '.join( + name for name, _module in plugIns if name != 'Webware')) + + # Now that we have our plug-in list, load them... + for name, module in plugIns: + plugIn = self.loadPlugIn(name, module, verbose=verbose) + if plugIn: + self._plugIns[name] = plugIn + if verbose: + print() + return self._plugIns diff --git a/webware/__init__.py b/webware/__init__.py index f5b3679..73e72a9 100644 --- a/webware/__init__.py +++ b/webware/__init__.py @@ -1 +1,17 @@ """Webware for Python""" + +import sys + + +def add_to_python_path(): + webwarePath = __path__[0] + if webwarePath not in sys.path: + sys.path.insert(0, webwarePath) + + +def load_plugins(path, settings=None, development=None): + from MockApplication import MockApplication + from PlugInLoader import PlugInLoader + app = MockApplication(path, settings, development) + loader = PlugInLoader(app) + loader.loadPlugIns(app.setting('PlugIns'), verbose=False) From 514ccae78e917e5d1d1a420fbfa84def66abad98 Mon Sep 17 00:00:00 2001 From: Jason Hildebrand Date: Thu, 2 Jan 2020 16:31:14 -0600 Subject: [PATCH 2/3] Document how to bootstrap Webware for command-line scripts. --- docs/appdev.rst | 21 +++++++++++++++++++++ webware/MockApplication.py | 4 ++++ 2 files changed, 25 insertions(+) diff --git a/docs/appdev.rst b/docs/appdev.rst index 4be7ec5..120a2c6 100644 --- a/docs/appdev.rst +++ b/docs/appdev.rst @@ -363,6 +363,25 @@ Now run this file in your IDE in debug mode. For instance, in PyCharm, right-cli Some IDEs like PyCharm can also debug remote processes. This could be useful to debug a test or production server. +Bootstrap Webware from Command line +----------------------------------- + +You may be in a situation where you want to execute some part of your Webware app from the command line, for example to implement a cron job or +maintenance script. In these situations you probably don't want to instantiate a full-fledged `Application` -- some of the downsides are that doing so +would cause standard output and standard error to be redirected to the log file, and that it sets up the session sweeper, task manager, etc. +But you may still need access to plugins such as MiscUtils, MiddleKit, which you may not be able to import directly. + +Here is a lightweight approach which allows you to bootstrap Webware and plugins:: + + import webware + webware.add_to_python_path() + webware.load_plugins('/your/app/directory') + + # now plugins are available... + import MiscUtils + import MiddleKit + + How do I Develop an App? ------------------------ @@ -381,3 +400,5 @@ The answer to that question might not seem clear after being deluged with all th * With this additional knowledge, create more sophisticated pages. * If you need to secure your pages using a login screen, you'll want to look at the SecurePage, LoginPage, and SecureCountVisits examples in ``Examples``. You'll need to modify them to suit your particular needs. + + diff --git a/webware/MockApplication.py b/webware/MockApplication.py index 7e615d6..f98a570 100644 --- a/webware/MockApplication.py +++ b/webware/MockApplication.py @@ -16,6 +16,10 @@ def recordFile(self, filename, isfile=None): class MockApplication(ConfigurableForServerSidePath): + """ + A minimal implementation which is compatible with Application + and which is sufficient to load plugins. + """ def __init__(self, path=None, settings=None, development=None): ConfigurableForServerSidePath.__init__(self) From ddaba9ea95e8c0bcc6eeb61a86b5724d83115ece Mon Sep 17 00:00:00 2001 From: Jason Hildebrand Date: Tue, 7 Jan 2020 10:33:40 -0600 Subject: [PATCH 3/3] Added documentation on debugging using pdb. --- docs/appdev.rst | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/appdev.rst b/docs/appdev.rst index 120a2c6..2a19d98 100644 --- a/docs/appdev.rst +++ b/docs/appdev.rst @@ -310,7 +310,16 @@ When creating the Application instance, it takes a ``development`` flag as argum print ~~~~~ -The most common technique is the infamous ``print`` statement which has been replaced with a ``print()`` function in Python 3. The results of ``print()`` calls go to the console where the WSGI server was started (not to the HTML page as would happen with CGI). In production mode, you can specify an ``AppLogFilename`` in ``Application.config``, which will cause the standard output and error to be redirected to this file. +The most common technique is the infamous ``print`` statement which has been replaced with a ``print()`` function in Python 3. The results of ``print()`` calls go to the console where the WSGI server was started (not to the HTML page as would happen with CGI). If you specify ``AppLogFilename`` in ``Application.config``, this will cause the standard output and error to be redirected to this file. + +For convenient debugging, we recommend you use a clause like this in your ``Application.config`` file:: + + if Development: + AppLogFilename = None + else: + AppLogFilename = 'Logs/Application.log' + +This will prevent standard output and error from being redirected to the log file in development mode, which makes it easier to find debugging output, and also makes it possible to use ```pdb``` (see below). Prefixing the debugging output with a special tag (such as ``>>``) is useful because it stands out on the console and you can search for the tag in source code to remove the print statements after they are no longer useful. For example:: @@ -331,7 +340,7 @@ While this is totally useful during development, giving away too much internal i Reloading the Development Server ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When a servlet's source code changes, it is reloaded. However, ancestor classes of servlets, library modules and configuration files are not. You may wish to enable the auto-reloading feature when running the development server, by adding the ``-r`` or ``--reload`` option to the ``webware seve command`` in order to mitigate this problem. +When a servlet's source code changes, it is reloaded. However, ancestor classes of servlets, library modules and configuration files are not. You may wish to enable the auto-reloading feature when running the development server, by adding the ``-r`` or ``--reload`` option to the ``webware serve command`` in order to mitigate this problem. In any case, when having problems, consider restarting the development server (or the WSGI server you are running in production). @@ -345,6 +354,19 @@ Assertions are used to ensure that the internal conditions of the application ar assert shoppingCart.total() >= 0, \ f'shopping cart total is {shoppingCart.total()}' +Debugging using PDB +~~~~~~~~~~~~~~~~~~~ +To use python's built-in debugger ```pdb```, see the tip above about setting ```AppLogFilename``` for convenient debugging. + +To have Webware automatically put you into pdb when an exception occurs, set this in your ``Application.config`` file:: + + EnterDebuggerOnException = Development + +A quick and easy way to debug a particular section of code is to add these lines at that point in the code:: + + import pdb + pdb.set_trace() + Debugging in an IDE ~~~~~~~~~~~~~~~~~~~