Skip to content

Commit 93733db

Browse files
elpranselprans
committed
Move compilation logic to setup.py
Use a custom build_ext command to cythonize and patch the protocol source. This makes it possible to build asyncpg from Cython source with regular setup.py commands. Cython dependency is still optional.
1 parent 6fe46fd commit 93733db

File tree

8 files changed

+175
-103
lines changed

8 files changed

+175
-103
lines changed

‎Makefile

Lines changed: 9 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
.PHONY: compile debug test clean check-env all
1+
.PHONY: compile debug test clean all
22

33

44
PYTHON ?= python
@@ -15,25 +15,16 @@ clean:
1515
find . -name '__pycache__' | xargs rm -rf
1616

1717

18-
check-env:
19-
$(PYTHON) -c "import cython; (cython.__version__ < '0.24') and exit(1)"
18+
compile:
19+
$(PYTHON) setup.py build_ext --inplace --cython-always
2020

2121

22-
compile: check-env clean
23-
echo "DEF DEBUG = 0" > asyncpg/protocol/__debug.pxi
24-
$(PYTHON) -m cython asyncpg/protocol/protocol.pyx
25-
rm asyncpg/protocol/__debug.pxi
26-
@echo "$$CYTHON_BUILD_PATCH_SCRIPT" | $(PYTHON)
27-
$(PYTHON) setup.py build_ext --inplace
28-
29-
30-
debug: check-env clean
31-
echo "DEF DEBUG = 1" > asyncpg/protocol/__debug.pxi
32-
$(PYTHON) -m cython -a -X linetrace=True asyncpg/protocol/protocol.pyx
33-
rm asyncpg/protocol/__debug.pxi
34-
@echo "$$CYTHON_BUILD_PATCH_SCRIPT" | $(PYTHON)
35-
CFLAGS="${CFLAGS} -DCYTHON_TRACE=1 -DCYTHON_TRACE_NOGIL=1" \
36-
$(PYTHON) setup.py build_ext --inplace --debug
22+
debug:
23+
$(PYTHON) setup.py build_ext --inplace --debug \
24+
--cython-always \
25+
--cython-annotate \
26+
--cython-directives linetrace=True \
27+
--define ASYNCPG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL
3728

3829

3930
test:
@@ -52,72 +43,3 @@ release: clean compile test
5243

5344
htmldocs:
5445
$(MAKE) -C docs html
55-
56-
57-
# Script to patch Cython 'async def' coroutines to have a 'tp_iter' slot,
58-
# which makes them compatible with 'yield from' without the
59-
# `asyncio.coroutine` decorator.
60-
define CYTHON_BUILD_PATCH_SCRIPT
61-
import re
62-
63-
with open('asyncpg/protocol/protocol.c', 'rt') as f:
64-
src = f.read()
65-
66-
src = re.sub(
67-
r'''
68-
\s* offsetof\(__pyx_CoroutineObject,\s*gi_weakreflist\),
69-
\s* 0,
70-
\s* 0,
71-
\s* __pyx_Coroutine_methods,
72-
\s* __pyx_Coroutine_memberlist,
73-
\s* __pyx_Coroutine_getsets,
74-
''',
75-
76-
r'''
77-
offsetof(__pyx_CoroutineObject, gi_weakreflist),
78-
__Pyx_Coroutine_await, /* tp_iter */
79-
0,
80-
__pyx_Coroutine_methods,
81-
__pyx_Coroutine_memberlist,
82-
__pyx_Coroutine_getsets,
83-
''',
84-
85-
src, flags=re.X)
86-
87-
# Fix a segfault in Cython.
88-
src = re.sub(
89-
r'''
90-
\s* __Pyx_Coroutine_get_qualname\(__pyx_CoroutineObject\s+\*self\)
91-
\s* {
92-
\s* Py_INCREF\(self->gi_qualname\);
93-
''',
94-
95-
r'''
96-
__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self)
97-
{
98-
if (self->gi_qualname == NULL) { return __pyx_empty_unicode; }
99-
Py_INCREF(self->gi_qualname);
100-
''',
101-
102-
src, flags=re.X)
103-
104-
src = re.sub(
105-
r'''
106-
\s* __Pyx_Coroutine_get_name\(__pyx_CoroutineObject\s+\*self\)
107-
\s* {
108-
\s* Py_INCREF\(self->gi_name\);
109-
''',
110-
111-
r'''
112-
__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self)
113-
{
114-
if (self->gi_name == NULL) { return __pyx_empty_unicode; }
115-
Py_INCREF(self->gi_name);
116-
''',
117-
118-
src, flags=re.X)
119-
120-
with open('asyncpg/protocol/protocol.c', 'wt') as f:
121-
f.write(src)
122-
endef
123-
export CYTHON_BUILD_PATCH_SCRIPT

‎asyncpg/protocol/buffer.pyx

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ cdef class ReadBuffer:
288288
self._pos0 = 0
289289
self._len0 = len(self._buf0)
290290

291-
IF DEBUG:
291+
if ASYNCPG_DEBUG:
292292
if self._len0 < 1:
293293
raise RuntimeError(
294294
'debug: second buffer of ReadBuffer is empty')
@@ -300,7 +300,7 @@ cdef class ReadBuffer:
300300
cdef:
301301
char * result
302302

303-
IF DEBUG:
303+
if ASYNCPG_DEBUG:
304304
if nbytes > self._length:
305305
return NULL
306306

@@ -361,7 +361,7 @@ cdef class ReadBuffer:
361361
cdef inline read_byte(self):
362362
cdef char* first_byte
363363

364-
IF DEBUG:
364+
if ASYNCPG_DEBUG:
365365
if not self._buf0:
366366
raise RuntimeError(
367367
'debug: first buffer of ReadBuffer is empty')
@@ -537,12 +537,12 @@ cdef class ReadBuffer:
537537
raise BufferError('no message to discard')
538538

539539
if self._current_message_len_unread:
540-
IF DEBUG:
540+
if ASYNCPG_DEBUG:
541541
mtype = chr(self._current_message_type)
542542

543543
discarded = self.consume_message()
544544

545-
IF DEBUG:
545+
if ASYNCPG_DEBUG:
546546
print('!!! discarding message {!r} unread data: {!r}'.format(
547547
mtype,
548548
(<Memory>discarded).as_bytes()))

‎asyncpg/protocol/coreproto.pyx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ cdef class CoreProtocol:
270270
object row
271271
Memory mem
272272

273-
IF DEBUG:
273+
if ASYNCPG_DEBUG:
274274
if buf.get_message_type() != b'D':
275275
raise RuntimeError(
276276
'_parse_data_msgs: first message is not "D"')

‎asyncpg/protocol/debug.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#ifndef ASYNCPG_DEBUG
2+
#define ASYNCPG_DEBUG 0
3+
#endif

‎asyncpg/protocol/debug.pxd

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
cdef extern from "debug.h":
2+
3+
cdef int ASYNCPG_DEBUG

‎asyncpg/protocol/protocol.pxd

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010

1111
from libc.stdint cimport int16_t, int32_t, uint16_t, uint32_t, int64_t, uint64_t
1212

13-
include "__debug.pxi"
13+
from .debug cimport ASYNCPG_DEBUG
14+
1415
include "consts.pxi"
1516
include "pgtypes.pxi"
1617

‎asyncpg/protocol/protocol.pyx

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@ from asyncpg import exceptions as apg_exc
3434

3535
from asyncpg.protocol cimport hton
3636

37-
38-
include "__debug.pxi"
3937
include "consts.pxi"
4038
include "pgtypes.pxi"
4139

@@ -337,7 +335,7 @@ cdef class BaseProtocol(CoreProtocol):
337335
waiter.set_result(True)
338336

339337
cdef _on_result__prepare(self, object waiter):
340-
IF DEBUG:
338+
if ASYNCPG_DEBUG:
341339
if self.statement is None:
342340
raise RuntimeError(
343341
'_on_result__prepare: statement is None')
@@ -367,7 +365,7 @@ cdef class BaseProtocol(CoreProtocol):
367365
waiter.set_result(self.result_status_msg.decode(self.encoding))
368366

369367
cdef _decode_row(self, const char* buf, int32_t buf_len):
370-
IF DEBUG:
368+
if ASYNCPG_DEBUG:
371369
if self.statement is None:
372370
raise RuntimeError(
373371
'_decode_row: statement is None')
@@ -378,7 +376,7 @@ cdef class BaseProtocol(CoreProtocol):
378376
waiter = self.waiter
379377
self.waiter = None
380378

381-
IF DEBUG:
379+
if ASYNCPG_DEBUG:
382380
if waiter is None:
383381
raise RuntimeError('_on_result: waiter is None')
384382

‎setup.py

Lines changed: 149 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,157 @@
55
# the Apache 2.0 License: http://www.apache.org/licenses/LICENSE-2.0
66

77

8-
import setuptools
8+
import os
9+
import os.path
10+
import re
911
import sys
1012

13+
import setuptools
14+
from setuptools.command import build_ext as _build_ext
15+
1116

1217
if sys.version_info < (3, 5):
1318
raise RuntimeError('asyncpg requires Python 3.5 or greater')
1419

1520

21+
CFLAGS = ['-O2']
22+
LDFLAGS = []
23+
24+
25+
class build_ext(_build_ext.build_ext):
26+
user_options = _build_ext.build_ext.user_options + [
27+
('cython-always', None,
28+
'run cythonize() even if .c files are present'),
29+
('cython-annotate', None,
30+
'Produce a colorized HTML version of the Cython source.'),
31+
]
32+
33+
def initialize_options(self):
34+
super(build_ext, self).initialize_options()
35+
self.cython_always = False
36+
self.cython_annotate = None
37+
38+
def finalize_options(self):
39+
need_cythonize = self.cython_always
40+
cfiles = {}
41+
42+
for extension in self.distribution.ext_modules:
43+
for i, sfile in enumerate(extension.sources):
44+
if sfile.endswith('.pyx'):
45+
prefix, ext = os.path.splitext(sfile)
46+
cfile = prefix + '.c'
47+
48+
if os.path.exists(cfile) and not self.cython_always:
49+
extension.sources[i] = cfile
50+
else:
51+
if os.path.exists(cfile):
52+
cfiles[cfile] = os.path.getmtime(cfile)
53+
else:
54+
cfiles[cfile] = 0
55+
need_cythonize = True
56+
57+
if need_cythonize:
58+
try:
59+
import Cython
60+
except ImportError:
61+
raise RuntimeError(
62+
'please install Cython to compile asyncpg from source')
63+
64+
if Cython.__version__ < '0.24':
65+
raise RuntimeError(
66+
'asyncpg requires Cython version 0.24 or greater')
67+
68+
from Cython.Build import cythonize
69+
70+
directives = {}
71+
if self.cython_directives:
72+
for directive in self.cython_directives.split(','):
73+
k, _, v = directive.partition('=')
74+
if v.lower() == 'false':
75+
v = False
76+
if v.lower() == 'true':
77+
v = True
78+
79+
directives[k] = v
80+
81+
self.distribution.ext_modules[:] = cythonize(
82+
self.distribution.ext_modules,
83+
compiler_directives=directives,
84+
annotate=self.cython_annotate)
85+
86+
for cfile, timestamp in cfiles.items():
87+
if os.path.getmtime(cfile) != timestamp:
88+
# The file was recompiled, patch
89+
self._patch_cfile(cfile)
90+
91+
super(build_ext, self).finalize_options()
92+
93+
def _patch_cfile(self, cfile):
94+
# Script to patch Cython 'async def' coroutines to have a 'tp_iter'
95+
# slot, which makes them compatible with 'yield from' without the
96+
# `asyncio.coroutine` decorator.
97+
98+
with open(cfile, 'rt') as f:
99+
src = f.read()
100+
101+
src = re.sub(
102+
r'''
103+
\s* offsetof\(__pyx_CoroutineObject,\s*gi_weakreflist\),
104+
\s* 0,
105+
\s* 0,
106+
\s* __pyx_Coroutine_methods,
107+
\s* __pyx_Coroutine_memberlist,
108+
\s* __pyx_Coroutine_getsets,
109+
''',
110+
111+
r'''
112+
offsetof(__pyx_CoroutineObject, gi_weakreflist),
113+
__Pyx_Coroutine_await, /* tp_iter */
114+
0,
115+
__pyx_Coroutine_methods,
116+
__pyx_Coroutine_memberlist,
117+
__pyx_Coroutine_getsets,
118+
''',
119+
120+
src, flags=re.X)
121+
122+
# Fix a segfault in Cython.
123+
src = re.sub(
124+
r'''
125+
\s* __Pyx_Coroutine_get_qualname\(__pyx_CoroutineObject\s+\*self\)
126+
\s* {
127+
\s* Py_INCREF\(self->gi_qualname\);
128+
''',
129+
130+
r'''
131+
__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self)
132+
{
133+
if (self->gi_qualname == NULL) { return __pyx_empty_unicode; }
134+
Py_INCREF(self->gi_qualname);
135+
''',
136+
137+
src, flags=re.X)
138+
139+
src = re.sub(
140+
r'''
141+
\s* __Pyx_Coroutine_get_name\(__pyx_CoroutineObject\s+\*self\)
142+
\s* {
143+
\s* Py_INCREF\(self->gi_name\);
144+
''',
145+
146+
r'''
147+
__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self)
148+
{
149+
if (self->gi_name == NULL) { return __pyx_empty_unicode; }
150+
Py_INCREF(self->gi_name);
151+
''',
152+
153+
src, flags=re.X)
154+
155+
with open(cfile, 'wt') as f:
156+
f.write(src)
157+
158+
16159
setuptools.setup(
17160
name='asyncpg',
18161
version='0.5.4',
@@ -38,7 +181,9 @@
38181
setuptools.Extension(
39182
"asyncpg.protocol.protocol",
40183
["asyncpg/protocol/record/recordobj.c",
41-
"asyncpg/protocol/protocol.c"],
42-
extra_compile_args=['-O2'])
43-
]
184+
"asyncpg/protocol/protocol.pyx"],
185+
extra_compile_args=CFLAGS,
186+
extra_link_args=LDFLAGS)
187+
],
188+
cmdclass={'build_ext': build_ext},
44189
)

0 commit comments

Comments
 (0)