Skip to content

Commit 261e21a

Browse files
CitoCito
committed
Adapt FieldStorage to newer Python versions
The FieldStorage class now takes a separator as an additional parameter in recent Python versions. We need to take this into consideration, and also change the splitting of query strings in POST requests so that this is consistent with the behavior in GET requests.
1 parent bdedaf8 commit 261e21a

File tree

2 files changed

+107
-34
lines changed

2 files changed

+107
-34
lines changed

‎webware/WebUtils/FieldStorage.py

Lines changed: 52 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,13 @@ class FieldStorage(cgi.FieldStorage):
2222
versions which append values from the query string to values sent via POST
2323
for parameters with the same name. With other words, our FieldStorage class
2424
overrides the query string parameters with the parameters sent via POST.
25-
26-
As recommended by W3C in section B.2.2 of the HTML 4.01 specification,
27-
we also support use of ';' in place of '&' as separator in query strings.
2825
"""
2926

3027
def __init__(self, fp=None, headers=None, outerboundary=b'',
3128
environ=None, keep_blank_values=False,
3229
strict_parsing=False, limit=None,
33-
encoding='utf-8', errors='replace', max_num_fields=None):
30+
encoding='utf-8', errors='replace',
31+
max_num_fields=None, separator='&'):
3432
if environ is None:
3533
environ = os.environ
3634
method = environ.get('REQUEST_METHOD', 'GET').upper()
@@ -48,20 +46,29 @@ def __init__(self, fp=None, headers=None, outerboundary=b'',
4846
if 'CONTENT_LENGTH' in environ:
4947
headers = {'content-type': content_type,
5048
'content-length': '-1'}
51-
if max_num_fields is None:
52-
# max_num_fields is only supported since Python 3.6.7 and 3.7.2
53-
super().__init__(
54-
fp, headers=headers, outerboundary=outerboundary,
55-
environ=environ, keep_blank_values=keep_blank_values,
56-
strict_parsing=strict_parsing, limit=limit,
57-
encoding=encoding, errors=errors)
49+
if separator == '&':
50+
# separator is only supported since Python 3.6.13
51+
if max_num_fields is None:
52+
# max_num_fields is only supported since Python 3.6.7
53+
super().__init__(
54+
fp, headers=headers, outerboundary=outerboundary,
55+
environ=environ, keep_blank_values=keep_blank_values,
56+
strict_parsing=strict_parsing, limit=limit,
57+
encoding=encoding, errors=errors)
58+
else:
59+
super().__init__(
60+
fp, headers=headers, outerboundary=outerboundary,
61+
environ=environ, keep_blank_values=keep_blank_values,
62+
strict_parsing=strict_parsing, limit=limit,
63+
encoding=encoding, errors=errors,
64+
max_num_fields=max_num_fields)
5865
else:
5966
super().__init__(
6067
fp, headers=headers, outerboundary=outerboundary,
6168
environ=environ, keep_blank_values=keep_blank_values,
6269
strict_parsing=strict_parsing, limit=limit,
6370
encoding=encoding, errors=errors,
64-
max_num_fields=max_num_fields)
71+
max_num_fields=max_num_fields, separator=separator)
6572
finally:
6673
if qs_on_post:
6774
environ['QUERY_STRING'] = qs_on_post
@@ -71,17 +78,39 @@ def __init__(self, fp=None, headers=None, outerboundary=b'',
7178
def add_qs(self, qs):
7279
"""Add all non-existing parameters from the given query string."""
7380
values = defaultdict(list)
74-
for name_values in qs.split('&'):
75-
for name_value in name_values.split(';'):
76-
nv = name_value.split('=', 2)
77-
if len(nv) != 2:
78-
if self.strict_parsing:
79-
raise ValueError(f'bad query field: {name_value!r}')
80-
continue
81-
name = parse.unquote(nv[0].replace('+', ' '))
82-
value = parse.unquote(nv[1].replace('+', ' '))
83-
if len(value) or self.keep_blank_values:
84-
values[name].append(value)
81+
# split the query string in the same way as the current Python does it
82+
try:
83+
max_num_fields = self.max_num_fields
84+
except AttributeError:
85+
max_num_fields = None
86+
try:
87+
separator = self.separator
88+
except AttributeError:
89+
# splitting algorithm before Python 3.6.13
90+
if max_num_fields is not None:
91+
num_fields = 1 + qs.count('&') + qs.count(';')
92+
if max_num_fields < num_fields:
93+
raise ValueError('Max number of fields exceeded')
94+
pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
95+
else:
96+
if not separator or not isinstance(separator, (str, bytes)):
97+
return # invalid separator, do nothing in this case
98+
if max_num_fields is not None:
99+
num_fields = 1 + qs.count(separator)
100+
if max_num_fields < num_fields:
101+
raise ValueError('Max number of fields exceeded')
102+
# new splitting algorithm that only supports one separator
103+
pairs = qs.split(separator)
104+
for name_value in pairs:
105+
nv = name_value.split('=', 1)
106+
if len(nv) != 2:
107+
if self.strict_parsing:
108+
raise ValueError(f'bad query field: {name_value!r}')
109+
continue
110+
name = parse.unquote(nv[0].replace('+', ' '))
111+
value = parse.unquote(nv[1].replace('+', ' '))
112+
if len(value) or self.keep_blank_values:
113+
values[name].append(value)
85114
if self.list is None:
86115
# This makes sure self.keys() are available, even
87116
# when valid POST data wasn't encountered.

‎webware/WebUtils/Tests/TestFieldStorage.py

Lines changed: 55 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def testGetRequest(self):
1919

2020
def testPostRequestWithQuery(self):
2121
fs = FieldStorage(fp=BytesIO(), environ=dict(
22-
REQUEST_METHOD='GET', QUERY_STRING='a=1&b=2&b=3&c=3'))
22+
REQUEST_METHOD='POST', QUERY_STRING='a=1&b=2&b=3&c=3'))
2323
self.assertEqual(fs.getfirst('a'), '1')
2424
self.assertEqual(fs.getfirst('b'), '2')
2525
self.assertEqual(fs.getfirst('c'), '3')
@@ -53,22 +53,66 @@ def testPostRequestOverrides(self):
5353
self.assertEqual(fs.getlist('e'), ['5', '6'])
5454
self.assertEqual(fs.getlist('f'), ['6'])
5555

56+
def testPostRequestWithTooManyFields(self):
57+
fs = FieldStorage(fp=BytesIO(), environ=dict(
58+
REQUEST_METHOD='POST', QUERY_STRING='a=1&a=2&a=3&a=4'),
59+
max_num_fields=4)
60+
self.assertEqual(fs.getlist('a'), ['1', '2', '3', '4'])
61+
if hasattr(fs, 'max_num_fields'): # only test if this is supported
62+
self.assertRaises(
63+
ValueError, FieldStorage,
64+
fp=BytesIO(), environ=dict(
65+
REQUEST_METHOD='POST', QUERY_STRING='a=1&a=2&a=3&a=4'),
66+
max_num_fields=3)
67+
5668
def testPostRequestWithQueryWithSemicolon1(self):
5769
fs = FieldStorage(fp=BytesIO(), environ=dict(
58-
REQUEST_METHOD='GET', QUERY_STRING='a=1&b=2;b=3&c=3'))
70+
REQUEST_METHOD='POST', QUERY_STRING='a=1&b=2;b=3&c=3'))
5971
self.assertEqual(fs.getfirst('a'), '1')
60-
self.assertEqual(fs.getfirst('b'), '2')
6172
self.assertEqual(fs.getfirst('c'), '3')
6273
self.assertEqual(fs.getlist('a'), ['1'])
63-
self.assertEqual(fs.getlist('b'), ['2', '3'])
6474
self.assertEqual(fs.getlist('c'), ['3'])
75+
separator = getattr(fs, 'separator', None)
76+
if separator: # new Python version, splits only &
77+
self.assertEqual(fs.getfirst('b'), '2;b=3')
78+
self.assertEqual(fs.getlist('b'), ['2;b=3'])
79+
fs = FieldStorage(fp=BytesIO(), environ=dict(
80+
REQUEST_METHOD='POST', QUERY_STRING='a=1&b=2&b=3&c=3'),
81+
separator='&')
82+
self.assertEqual(fs.getfirst('a'), '1')
83+
self.assertEqual(fs.getfirst('b'), '2')
84+
self.assertEqual(fs.getfirst('c'), '3')
85+
self.assertEqual(fs.getlist('a'), ['1'])
86+
self.assertEqual(fs.getlist('b'), ['2', '3'])
87+
self.assertEqual(fs.getlist('c'), ['3'])
88+
else: # old Python version, splits ; and &
89+
self.assertEqual(fs.getfirst('b'), '2')
90+
self.assertEqual(fs.getlist('b'), ['2', '3'])
6591

6692
def testPostRequestWithQueryWithSemicolon2(self):
6793
fs = FieldStorage(fp=BytesIO(), environ=dict(
68-
REQUEST_METHOD='GET', QUERY_STRING='a=1;b=2&b=3;c=3'))
69-
self.assertEqual(fs.getfirst('a'), '1')
70-
self.assertEqual(fs.getfirst('b'), '2')
71-
self.assertEqual(fs.getfirst('c'), '3')
72-
self.assertEqual(fs.getlist('a'), ['1'])
73-
self.assertEqual(fs.getlist('b'), ['2', '3'])
74-
self.assertEqual(fs.getlist('c'), ['3'])
94+
REQUEST_METHOD='POST', QUERY_STRING='a=1;b=2&b=3;c=3'))
95+
separator = getattr(fs, 'separator', None)
96+
if separator: # new Python version, splits only &
97+
self.assertEqual(fs.getfirst('a'), '1;b=2')
98+
self.assertEqual(fs.getfirst('b'), '3;c=3')
99+
self.assertIsNone(fs.getfirst('c'))
100+
self.assertEqual(fs.getlist('a'), ['1;b=2'])
101+
self.assertEqual(fs.getlist('b'), ['3;c=3'])
102+
self.assertEqual(fs.getlist('c'), [])
103+
fs = FieldStorage(fp=BytesIO(), environ=dict(
104+
REQUEST_METHOD='POST', QUERY_STRING='a=1;b=2;b=3;c=3'),
105+
separator=';')
106+
self.assertEqual(fs.getfirst('a'), '1')
107+
self.assertEqual(fs.getfirst('b'), '2')
108+
self.assertEqual(fs.getfirst('c'), '3')
109+
self.assertEqual(fs.getlist('a'), ['1'])
110+
self.assertEqual(fs.getlist('b'), ['2', '3'])
111+
self.assertEqual(fs.getlist('c'), ['3'])
112+
else: # old Python version, splits ; and &
113+
self.assertEqual(fs.getfirst('a'), '1')
114+
self.assertEqual(fs.getfirst('b'), '2')
115+
self.assertEqual(fs.getfirst('c'), '3')
116+
self.assertEqual(fs.getlist('a'), ['1'])
117+
self.assertEqual(fs.getlist('b'), ['2', '3'])
118+
self.assertEqual(fs.getlist('c'), ['3'])

0 commit comments

Comments
 (0)