Skip to content

Commit 907fb49

Browse files
mrmasterplanandialbrecht
authored andcommitted
change singleton behavior
1 parent fbf9a57 commit 907fb49

File tree

4 files changed

+40
-24
lines changed

4 files changed

+40
-24
lines changed

docs/source/extending.rst

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ a keyword to the lexer:
4545
from sqlparse.lexer import Lexer
4646
4747
# get the lexer singleton object to configure it
48-
lex = Lexer()
48+
lex = Lexer.get_default_instance()
4949
5050
# Clear the default configurations.
5151
# After this call, reg-exps and keyword dictionaries need to be loaded

sqlparse/lexer.py

Lines changed: 34 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
"""SQL Lexer"""
99
import re
10+
1011
# This code is based on the SqlLexer in pygments.
1112
# http://pygments.org/
1213
# It's separated from the rest of pygments to increase performance
@@ -18,21 +19,39 @@
1819
from sqlparse.utils import consume
1920

2021

21-
class _LexerSingletonMetaclass(type):
22-
_lexer_instance = None
23-
24-
def __call__(cls, *args, **kwargs):
25-
if _LexerSingletonMetaclass._lexer_instance is None:
26-
_LexerSingletonMetaclass._lexer_instance = super(
27-
_LexerSingletonMetaclass, cls
28-
).__call__(*args, **kwargs)
29-
return _LexerSingletonMetaclass._lexer_instance
30-
31-
32-
class Lexer(metaclass=_LexerSingletonMetaclass):
22+
class Lexer:
3323
"""The Lexer supports configurable syntax.
3424
To add support for additional keywords, use the `add_keywords` method."""
3525

26+
_default_intance = None
27+
28+
# Development notes:
29+
# - This class is prepared to be able to support additional SQL dialects
30+
# in the future by adding additional functions that take the place of
31+
# the function default_initialization()
32+
# - The lexer class uses an explicit singleton behavior with the
33+
# instance-getter method get_default_instance(). This mechanism has
34+
# the advantage that the call signature of the entry-points to the
35+
# sqlparse library are not affected. Also, usage of sqlparse in third
36+
# party code does not need to be adapted. On the other hand, singleton
37+
# behavior is not thread safe, and the current implementation does not
38+
# easily allow for multiple SQL dialects to be parsed in the same
39+
# process. Such behavior can be supported in the future by passing a
40+
# suitably initialized lexer object as an additional parameter to the
41+
# entry-point functions (such as `parse`). Code will need to be written
42+
# to pass down and utilize such an object. The current implementation
43+
# is prepared to support this thread safe approach without the
44+
# default_instance part needing to change interface.
45+
46+
@classmethod
47+
def get_default_instance(cls):
48+
"""Returns the lexer instance used internally
49+
by the sqlparse core functions."""
50+
if cls._default_intance is None:
51+
cls._default_intance = cls()
52+
cls._default_intance.default_initialization()
53+
return cls._default_intance
54+
3655
def default_initialization(self):
3756
"""Initialize the lexer with default dictionaries.
3857
Useful if you need to revert custom syntax settings."""
@@ -45,13 +64,10 @@ def default_initialization(self):
4564
self.add_keywords(keywords.KEYWORDS_MSACCESS)
4665
self.add_keywords(keywords.KEYWORDS)
4766

48-
def __init__(self):
49-
self.default_initialization()
50-
5167
def clear(self):
5268
"""Clear all syntax configurations.
5369
Useful if you want to load a reduced set of syntax configurations.
54-
After this call, reg-exps and keyword dictionaries need to be loaded
70+
After this call, regexps and keyword dictionaries need to be loaded
5571
to make the lexer functional again."""
5672
self._SQL_REGEX = []
5773
self._keywords = []
@@ -73,7 +89,7 @@ def is_keyword(self, value):
7389
"""Checks for a keyword.
7490
7591
If the given value is in one of the KEYWORDS_* dictionary
76-
it's considered a keyword. Otherwise tokens.Name is returned.
92+
it's considered a keyword. Otherwise, tokens.Name is returned.
7793
"""
7894
val = value.upper()
7995
for kwdict in self._keywords:
@@ -136,4 +152,4 @@ def tokenize(sql, encoding=None):
136152
Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream
137153
of ``(token type, value)`` items.
138154
"""
139-
return Lexer().get_tokens(sql, encoding)
155+
return Lexer.get_default_instance().get_tokens(sql, encoding)

tests/test_keywords.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,5 @@ class TestSQLREGEX:
99
'1.', '-1.',
1010
'.1', '-.1'])
1111
def test_float_numbers(self, number):
12-
ttype = next(tt for action, tt in Lexer()._SQL_REGEX if action(number))
12+
ttype = next(tt for action, tt in Lexer.get_default_instance()._SQL_REGEX if action(number))
1313
assert tokens.Number.Float == ttype

tests/test_parse.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -509,7 +509,7 @@ def test_configurable_keywords():
509509
(sqlparse.tokens.Punctuation, ";"),
510510
]
511511

512-
Lexer().add_keywords(
512+
Lexer.get_default_instance().add_keywords(
513513
{
514514
"BACON": sqlparse.tokens.Name.Builtin,
515515
"SPAM": sqlparse.tokens.Keyword,
@@ -520,7 +520,7 @@ def test_configurable_keywords():
520520
tokens = sqlparse.parse(sql)[0]
521521

522522
# reset the syntax for later tests.
523-
Lexer().default_initialization()
523+
Lexer.get_default_instance().default_initialization()
524524

525525
assert list(
526526
(t.ttype, t.value)
@@ -539,7 +539,7 @@ def test_configurable_keywords():
539539

540540

541541
def test_configurable_regex():
542-
lex = Lexer()
542+
lex = Lexer.get_default_instance()
543543
lex.clear()
544544

545545
my_regex = (r"ZORDER\s+BY\b", sqlparse.tokens.Keyword)
@@ -559,7 +559,7 @@ def test_configurable_regex():
559559
tokens = sqlparse.parse("select * from foo zorder by bar;")[0]
560560

561561
# reset the syntax for later tests.
562-
Lexer().default_initialization()
562+
Lexer.get_default_instance().default_initialization()
563563

564564
assert list(
565565
(t.ttype, t.value)

0 commit comments

Comments
 (0)