Skip to content

Commit 9e4a471

Browse files
fangererfangerer
committed
[GR-18413] Build and ship C API.
PullRequest: graalpython/675
2 parents b44db7b + 6ac8fd9 commit 9e4a471

File tree

20 files changed

+273
-381
lines changed

20 files changed

+273
-381
lines changed

‎ci.jsonnet

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{ overlay: "00167393535527db3d7235da288740b6c7f54241" }
1+
{ overlay: "41b5d34c082c57db236d3a620b1dc1d367537fac" }

‎doc/CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ To run the JUnit tests, you can use this command:
129129
To run a subset of the tests, you can use the following. Again, you can use `-d`
130130
to attach with a Java debugger.
131131

132-
mx [-d] unittest JAVA-TEST-CLASSNAME
132+
mx [-d] punittest JAVA-TEST-CLASSNAME
133133

134134
To run the Python standard library tests, you can use the following:
135135

‎graalpython/com.oracle.graal.python.cext/include/Python.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
#define PYPY_VERSION 0
5151
#define PYPY_VERSION_NUM 0
5252

53-
#include <truffle.h>
5453
#include <stdio.h>
5554
#include <string.h>
5655
#include <errno.h>

‎graalpython/com.oracle.graal.python.cext/setup.py

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@
4242
import os
4343
import shutil
4444
import site
45-
import tempfile
4645
import logging
4746
from distutils.core import setup, Extension
4847
from distutils.sysconfig import get_config_var, get_config_vars
@@ -57,6 +56,11 @@
5756
darwin_native = sys.platform == "darwin" and sys.graal_python_platform_id == "native"
5857
so_ext = get_config_var("EXT_SUFFIX")
5958

59+
60+
if darwin_native:
61+
get_config_vars()["LDSHARED"] = get_config_vars()['LDSHARED_LINUX'] + " -L" + sys.graal_python_capi_home + " -lpython." + get_config_vars()["SOABI"] + " -rpath '@loader_path/../'"
62+
63+
6064
def system(cmd, msg=""):
6165
logger.debug("Running command: " + cmd)
6266
status = os.system(cmd)
@@ -83,6 +87,7 @@ def __init__(self, lib_name, package_spec, url):
8387
self.url = url
8488

8589
def download(self):
90+
import tempfile
8691
package_pattern = os.environ.get("GINSTALL_PACKAGE_PATTERN", None)
8792
package_version_pattern = os.environ.get("GINSTALL_PACKAGE_VERSION_PATTERN", None)
8893
tempdir = tempfile.mkdtemp()
@@ -173,9 +178,10 @@ def install(self, build_dir=None):
173178
return self.lib_install_dir
174179

175180
def conftest(self):
181+
import tempfile
176182
src = b'''#include <bzlib.h>
177183
#include <stdio.h>
178-
184+
179185
int main(void) {
180186
printf("%.200s", BZ2_bzlibVersion());
181187
return 0;
@@ -260,12 +266,13 @@ def __call__(self):
260266

261267

262268
builtin_exts = (
263-
NativeBuiltinModule("_bz2", deps=[Bzip2Depedency("bz2", "bzip2==1.0.8", "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz")]),
264269
NativeBuiltinModule("_cpython_sre"),
265270
NativeBuiltinModule("_cpython_unicodedata"),
266271
NativeBuiltinModule("_memoryview"),
267272
NativeBuiltinModule("_mmap"),
268273
NativeBuiltinModule("_struct"),
274+
# the above modules are more core, we need them first to deal with later, more complex modules with dependencies
275+
NativeBuiltinModule("_bz2", deps=[Bzip2Depedency("bz2", "bzip2==1.0.8", "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz")]),
269276
)
270277

271278

@@ -304,8 +311,8 @@ def build_libpython(capi_home):
304311
ext_modules=[module],
305312
)
306313

307-
def build_builtin_exts(capi_module_home):
308-
args = [verbosity, 'build', 'install_lib', '-f', '--install-dir=%s' % capi_module_home, "clean"]
314+
def build_builtin_exts(capi_home):
315+
args = [verbosity, 'build', 'install_lib', '-f', '--install-dir=%s/modules' % capi_home, "clean"]
309316
for ext in builtin_exts:
310317
distutil_ext = ext()
311318
res = setup(
@@ -319,21 +326,10 @@ def build_builtin_exts(capi_module_home):
319326
logger.debug("Successfully built and installed module %s", ext.name)
320327

321328

322-
def build(capi_home, capi_module_home):
329+
def build(capi_home):
323330
build_libpython(capi_home)
324-
build_builtin_exts(capi_module_home)
325-
326-
327-
def clean(capi_home, capi_module_home):
328-
libpython_path = os.path.join(capi_home, libpython_name + so_ext)
329-
if os.path.exists(libpython_path):
330-
logger.info("Cleaning %s", libpython_path)
331-
os.unlink(libpython_path)
332-
if os.path.exists(capi_module_home):
333-
for f in [os.path.join(capi_module_home, f) for f in os.listdir(capi_module_home) if f.endswith(so_ext)]:
334-
logger.info("Cleaning %s", f)
335-
os.unlink(f)
331+
build_builtin_exts(capi_home)
336332

337333

338334
if __name__ == "__main__":
339-
build()
335+
build(sys.argv[1])

‎graalpython/com.oracle.graal.python.cext/src/capi.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
#include <polyglot.h>
4848

4949
#include "Python.h"
50+
#include <truffle.h>
5051

5152
#define SRC_CS "utf-8"
5253

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

Lines changed: 4 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ public static void main(String[] args) {
7171
private boolean quietFlag = false;
7272
private boolean noUserSite = false;
7373
private boolean noSite = false;
74-
private boolean ensureCapi = false;
7574
private boolean stdinIsInteractive = System.console() != null;
7675
private boolean runLLI = false;
7776
private boolean unbufferedIO = false;
@@ -152,13 +151,6 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
152151
case "--show-version":
153152
versionAction = VersionAction.PrintAndContinue;
154153
break;
155-
case "-ensure-capi":
156-
if (wantsExperimental) {
157-
ensureCapi = true;
158-
} else {
159-
unrecognized.add(arg);
160-
}
161-
break;
162154
case "-debug-java":
163155
if (wantsExperimental) {
164156
if (!isAOT()) {
@@ -235,9 +227,10 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
235227
arguments.add(i + 1 + j, "-" + optionChar);
236228
}
237229
} else {
238-
if (arg.startsWith("--llvm.")) {
239-
addRelaunchArg(arg);
240-
} else if (arg.startsWith("--python.CAPI")) {
230+
if (arg.startsWith("--llvm.") ||
231+
arg.startsWith("--python.CoreHome") ||
232+
arg.startsWith("--python.StdLibHome") ||
233+
arg.startsWith("--python.CAPI")) {
241234
addRelaunchArg(arg);
242235
}
243236
// possibly a polyglot argument
@@ -395,9 +388,6 @@ protected void launch(Builder contextBuilder) {
395388
print("Type \"help\", \"copyright\", \"credits\" or \"license\" for more information.");
396389
}
397390
}
398-
if (ensureCapi) {
399-
evalInternal(context, "import build_capi; build_capi.ensure_capi([" + (quietFlag ? "'-q'" : "") + "])\n");
400-
}
401391
if (!noSite) {
402392
evalInternal(context, "import site\n");
403393
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java

Lines changed: 3 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@
5353
import com.oracle.graal.python.PythonLanguage;
5454
import com.oracle.graal.python.builtins.Builtin;
5555
import com.oracle.graal.python.builtins.CoreFunctions;
56-
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
5756
import com.oracle.graal.python.builtins.PythonBuiltins;
5857
import com.oracle.graal.python.builtins.modules.PythonCextBuiltins.CheckFunctionResultNode;
5958
import com.oracle.graal.python.builtins.objects.PNone;
@@ -89,7 +88,6 @@
8988
import com.oracle.graal.python.runtime.exception.PythonErrorType;
9089
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
9190
import com.oracle.truffle.api.CallTarget;
92-
import com.oracle.truffle.api.CompilerAsserts;
9391
import com.oracle.truffle.api.CompilerDirectives;
9492
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
9593
import com.oracle.truffle.api.RootCallTarget;
@@ -250,9 +248,7 @@ private Object loadDynamicModuleWithSpec(String name, String path, InteropLibrar
250248
}
251249
}
252250

253-
private static final String CAPI_NOT_BUILT_HINT = "Did you forget to build the C API using 'graalpython -m build_capi'?";
254-
private static final String CAPI_LOAD_ERROR = "Could not load C API from %s.\n" + CAPI_NOT_BUILT_HINT;
255-
private static final String CAPI_LOCATE_ERROR = "Could not locate C API library '%s' in 'sys.path'\n" + CAPI_NOT_BUILT_HINT;
251+
private static final String CAPI_LOAD_ERROR = "Could not load C API from %s.\n";
256252

257253
@TruffleBoundary
258254
private void ensureCapiWasLoaded() {
@@ -262,8 +258,8 @@ private void ensureCapiWasLoaded() {
262258
CompilerDirectives.transferToInterpreterAndInvalidate();
263259

264260
String libPythonName = "libpython" + ExtensionSuffixesNode.getSoAbi(context);
265-
String libPythonPath = getCapiHome(context, env, libPythonName);
266-
TruffleFile capiFile = env.getInternalTruffleFile(libPythonPath);
261+
TruffleFile homePath = env.getInternalTruffleFile(context.getCAPIHome());
262+
TruffleFile capiFile = homePath.resolve(libPythonName);
267263
Object capi = null;
268264
try {
269265
SourceBuilder capiSrcBuilder = Source.newBuilder(LLVM_LANGUAGE, capiFile);
@@ -291,42 +287,6 @@ private void ensureCapiWasLoaded() {
291287
}
292288
}
293289

294-
private String getCapiHome(PythonContext context, Env env, String libPythonName) {
295-
CompilerAsserts.neverPartOfCompilation();
296-
// look for file 'libPythonName' in the path
297-
ReadAttributeFromObjectNode readNode = ReadAttributeFromObjectNode.getUncached();
298-
PythonModule sysModule = context.getCore().lookupBuiltinModule("sys");
299-
String path = tryPathEntry(env, libPythonName, readNode.execute(sysModule, "graal_python_capi_home"));
300-
if (path != null) {
301-
return path;
302-
}
303-
PythonLanguage.getLogger().severe(() -> String.format(CAPI_LOCATE_ERROR, libPythonName));
304-
throw raise(PythonErrorType.ImportError, CAPI_LOCATE_ERROR, libPythonName);
305-
}
306-
307-
private String tryPathEntry(Env env, String libPythonName, Object entry) {
308-
String path = String.join(env.getFileNameSeparator(), asString(entry), libPythonName);
309-
PythonLanguage.getLogger().log(Level.FINER, "Looking for " + libPythonName + " in " + path);
310-
try {
311-
if (env.getPublicTruffleFile(path).exists()) {
312-
PythonLanguage.getLogger().log(Level.FINE, "Found " + libPythonName + " in " + path);
313-
return path;
314-
}
315-
} catch (SecurityException e) {
316-
// ignore
317-
}
318-
return null;
319-
}
320-
321-
private String asString(Object object) {
322-
if (object instanceof String) {
323-
return (String) object;
324-
} else if (object instanceof PString) {
325-
return ((PString) object).getValue();
326-
}
327-
throw raise(PythonBuiltinClassType.TypeError, "path entry '%s' is not of type 'str'", object);
328-
}
329-
330290
private SetItemNode getSetItemNode() {
331291
if (setItemNode == null) {
332292
CompilerDirectives.transferToInterpreterAndInvalidate();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java

Lines changed: 26 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -185,12 +185,17 @@ public void postInitialize(PythonCore core) {
185185
sys.setAttribute(name, base_prefix);
186186
}
187187

188+
String coreHome = context.getCoreHome();
189+
String stdlibHome = context.getStdlibHome();
190+
String capiHome = context.getCAPIHome();
191+
188192
sys.setAttribute("executable", PythonOptions.getOption(context, PythonOptions.Executable));
189193
sys.setAttribute("graal_python_jython_emulation_enabled", PythonOptions.getOption(context, PythonOptions.EmulateJython));
190194
sys.setAttribute("graal_python_host_import_enabled", context.getEnv().isHostLookupAllowed());
191195
sys.setAttribute("graal_python_home", context.getLanguage().getHome());
192-
sys.setAttribute("graal_python_core_home", context.getCoreHome());
193-
sys.setAttribute("graal_python_stdlib_home", context.getStdlibHome());
196+
sys.setAttribute("graal_python_core_home", coreHome);
197+
sys.setAttribute("graal_python_stdlib_home", stdlibHome);
198+
sys.setAttribute("graal_python_capi_home", capiHome);
194199
sys.setAttribute("__flags__", core.factory().createTuple(new Object[]{
195200
false, // bytes_warning
196201
!PythonOptions.getFlag(context, PythonOptions.PythonOptimizeFlag), // debug
@@ -214,33 +219,39 @@ public void postInitialize(PythonCore core) {
214219

215220
LanguageInfo llvmInfo = env.getInternalLanguages().get(LLVM_LANGUAGE);
216221
Toolchain toolchain = env.lookup(llvmInfo, Toolchain.class);
217-
String capiSrc = context.getCAPIHome();
218-
String capiHome = getCapiHome(env, sys);
222+
223+
boolean isIsolated = PythonOptions.getOption(context, PythonOptions.IsolateFlag);
224+
boolean capiSeparate = !capiHome.equals(coreHome);
219225

220226
Object[] path;
221227
int pathIdx = 0;
222-
boolean doIsolate = PythonOptions.getOption(context, PythonOptions.IsolateFlag);
223-
int defaultPaths = doIsolate ? 3 : 4;
228+
int defaultPathsLen = 2;
229+
if (!isIsolated) {
230+
defaultPathsLen++;
231+
}
232+
if (capiSeparate) {
233+
defaultPathsLen++;
234+
}
224235
if (option.length() > 0) {
225236
String[] split = option.split(context.getEnv().getPathSeparator());
226-
path = new Object[split.length + defaultPaths];
237+
path = new Object[split.length + defaultPathsLen];
227238
System.arraycopy(split, 0, path, 0, split.length);
228239
pathIdx = split.length;
229240
} else {
230-
path = new Object[defaultPaths];
241+
path = new Object[defaultPathsLen];
231242
}
232-
if (!doIsolate) {
243+
if (!isIsolated) {
233244
path[pathIdx++] = getScriptPath(env, args);
234245
}
235-
path[pathIdx++] = context.getStdlibHome();
236-
path[pathIdx++] = context.getCoreHome() + env.getFileNameSeparator() + "modules";
237-
path[pathIdx++] = capiHome;
246+
path[pathIdx++] = stdlibHome;
247+
path[pathIdx++] = coreHome + env.getFileNameSeparator() + "modules";
248+
if (capiSeparate) {
249+
// include our native modules on the path
250+
path[pathIdx++] = capiHome + env.getFileNameSeparator() + "modules";
251+
}
238252
PList sysPaths = core.factory().createList(path);
239253
sys.setAttribute("path", sysPaths);
240-
sys.setAttribute("graal_python_cext_src", capiSrc);
241254
sys.setAttribute("graal_python_platform_id", toolchain.getIdentifier());
242-
sys.setAttribute("graal_python_capi_home", capiHome);
243-
sys.setAttribute("graal_python_capi_module_home", capiHome);
244255
}
245256

246257
private static String getScriptPath(Env env, String[] args) {
@@ -266,33 +277,6 @@ private static String getScriptPath(Env env, String[] args) {
266277
return scriptPath;
267278
}
268279

269-
private static String getCapiHome(Env env, PythonModule sys) {
270-
String pythonSubdir = "python" + PythonLanguage.MAJOR + "." + PythonLanguage.MINOR;
271-
return String.join(env.getFileNameSeparator(), getCapiUserBase(env, sys), "lib", pythonSubdir, "capi");
272-
}
273-
274-
// similar to 'sysconfig._getuserbase()'
275-
private static String getCapiUserBase(Env env, PythonModule sys) {
276-
String customUserBase = env.getEnvironment().get("PYTHONUSERBASE");
277-
if (customUserBase != null) {
278-
return customUserBase;
279-
}
280-
281-
Object osName = sys.getAttribute("platform");
282-
if (FRAMEWORK != PNone.NONE && PLATFORM_DARWIN.equals(osName)) {
283-
String _framework = FRAMEWORK.toString();
284-
return String.join(env.getFileNameSeparator(), System.getProperty("user.home"), "Library", _framework, PythonLanguage.MAJOR + "." + PythonLanguage.MINOR);
285-
} else if (PLATFORM_WIN32.equals(osName)) {
286-
// first try to get 'APPDATA'
287-
String appdata = System.getenv("APPDATA");
288-
if (appdata != null) {
289-
return appdata + env.getFileNameSeparator() + "Python";
290-
}
291-
return System.getProperty("user.home") + env.getFileNameSeparator() + "Python";
292-
}
293-
return System.getProperty("user.home") + env.getFileNameSeparator() + ".local";
294-
}
295-
296280
static String getPythonArch() {
297281
String arch = System.getProperty("os.arch", "");
298282
if (arch.equals("amd64")) {

0 commit comments

Comments
 (0)