adds corresponding python module

Author: Christopher Doris

juliapy/.gitignore

@@ -0,0 +1 @@
+__pycache__

juliapy/julia/__init__.py

@@ -0,0 +1,63 @@
+_CONFIG = dict()
+
+def _init_():
+    import os, os.path, sys, ctypes as c, types, shutil, subprocess
+    libpath = os.environ.get('JULIAPY_LIB')
+    if libpath is None:
+        exepath = os.environ.get('JULIAPY_EXE')
+        if exepath is None:
+            exepath = shutil.which('julia')
+            if exepath is None:
+                raise Exception('Cannot find Julia. Ensure it is in your PATH, or set JULIAPY_EXE to its path.')
+        else:
+            if not os.path.isfile(exepath):
+                raise Exception('JULIAPY_EXE=%s does not exist' % repr(exepath))
+        _CONFIG['exepath'] = exepath
+        libpath = subprocess.run([exepath, '-e', 'import Libdl; print(abspath(Libdl.dlpath("libjulia")))'], stdout=(subprocess.PIPE)).stdout.decode('utf8')
+    else:
+        if not os.path.isfile(libpath):
+            raise Exception('JULIAPY_LIB=%s does not exist' % repr(libpath))
+    _CONFIG['libpath'] = libpath
+    try:
+        d = os.getcwd()
+        os.chdir(os.path.dirname(libpath))
+        lib = c.CDLL(libpath)
+    finally:
+        os.chdir(d)
+
+    _CONFIG['lib'] = lib
+    lib.jl_init__threading.argtypes = []
+    lib.jl_init__threading.restype = None
+    lib.jl_init__threading()
+    lib.jl_eval_string.argtypes = [c.c_char_p]
+    lib.jl_eval_string.restype = c.c_void_p
+    res = lib.jl_eval_string(
+        '''
+        ENV["PYTHONJL_LIBPTR"] = "{}"
+        import Python
+        Python.with_gil() do
+            Python.pyimport("sys").modules["julia"].Main = Python.pyjl(Main)
+        end
+        '''.format(c.pythonapi._handle).encode('utf8'))
+    if res is None:
+        raise Exception('Python.jl did not start properly. Ensure that the Python package is installed in Julia.')
+
+    class Wrapper(types.ModuleType):
+
+        def __getattr__(self, k):
+            return getattr(self.Main, k)
+
+        def __dir__(self):
+            return super().__dir__() + self.Main.__dir__()
+
+    sys.modules['julia'].__class__ = Wrapper
+
+_init_()
+del _init_
+
+Core = Main.Core
+Base = Main.Base
+Python = Main.Python
+
+def _import(*names):
+    Main.eval(Base.Meta.parse('import ' + ', '.join(names)))

juliapy/setup.py

@@ -0,0 +1,19 @@
+import setuptools
+
+setuptools.setup(
+    name="julia",
+    version="0.1.0",
+    author="Christopher Rowley",
+    # author_email="author@example.com",
+    description="Julia-Python interoperability",
+    # long_description=long_description,
+    # long_description_content_type="text/markdown",
+    # url="https://github.com/pypa/sampleproject",
+    packages=["julia"],
+    classifiers=[
+        "Programming Language :: Python :: 3",
+        "License :: OSI Approved :: MIT License",
+        "Operating System :: OS Independent",
+    ],
+    python_requires='>=3.3',
+)

src/PyBuffer.jl

@@ -26,9 +26,9 @@ mutable struct PyBuffer
         b = new(info)
         finalizer(b) do b
             if CONFIG.isinitialized
-                s = gil_on()
-                err = C.PyBuffer_Release(pointer(b.info))
-                s || gil_off()
+                err = with_gil() do
+                    C.PyBuffer_Release(pointer(b.info))
+                end
                 check(err)
             end
         end

src/PyObjectArray.jl

@@ -4,11 +4,11 @@ mutable struct PyObjectArray{N} <: AbstractArray{PyObject, N}
         x = new{N}(fill(CPyPtr(C_NULL), dims))
         finalizer(x) do x
             if CONFIG.isinitialized
-                s = gil_on()
-                for ptr in x.ptrs
-                    C.Py_DecRef(ptr)
+                with_gil() do
+                    for ptr in x.ptrs
+                        C.Py_DecRef(ptr)
+                    end
                 end
-                s || gil_off()
             end
         end
     end

src/Python.jl

@@ -27,10 +27,8 @@ include("utils.jl")
     pyprogname_w :: Vector{Cwchar_t} = []
     "True if this is stackless Python."
     isstackless :: Bool = false
-    "True if the Python library was already loaded."
-    preloaded :: Bool = false
-    "True if the Python interpreter was already initialized."
-    preinitialized :: Bool = false
+    """True if Julia is embedded into Python (indicated by ENV["PYTHONJL_LIBPTR"] being set)."""
+    isembedded :: Bool = false
     "True if the Python interpreter is currently initialized."
     isinitialized :: Bool = false
     "The running Python version."
@@ -47,8 +45,6 @@ include("utils.jl")
     sysautolasttraceback :: Bool = true
     "True if the Python input hook is currently running."
     inputhookrunning :: Bool = false
-    "If NULL, we have the GIL. If non-NULL, this is the saved thread state."
-    gilstate :: Ptr{Cvoid} = C_NULL
 end
 Base.show(io::IO, ::MIME"text/plain", c::Config) =
     for k in fieldnames(Config)

src/builtins.jl

@@ -55,6 +55,6 @@ for n in [
 ]
     j = Symbol(:py, lowercase(string(n)))
     p = QuoteNode(Symbol(:PyExc_, n))
-    @eval const $j = pylazyobject(() -> pyborrowedobject(unsafe_load(Ptr{CPyPtr}(C.@pyglobal($p)))))
+    @eval const $j = pylazyobject(() -> pyborrowedobject(unsafe_load(Ptr{CPyPtr}(C.pyglobal($p)))))
     @eval export $j
 end

src/convert.jl

@@ -62,18 +62,22 @@ Base.convert(::Type{Any}, o::PyObject) = o
 ### SPECIAL CONVERSIONS
 
 @generated _eltype(o) = try eltype(o); catch; missing; end
+_eltype(o::Type) = missing
 
 @generated _keytype(o) = try keytype(o); catch; missing; end
 _keytype(o::Base.RefValue) = Tuple{}
 _keytype(o::NamedTuple) = Union{Symbol,Int}
 _keytype(o::Tuple) = Int
+_keytype(o::Type) = missing
 
 @generated _valtype(o, k...) = try valtype(o); catch; missing; end
 _valtype(o::NamedTuple, k::Int) = fieldtype(typeof(o), k)
 _valtype(o::NamedTuple, k::Symbol) = fieldtype(typeof(o), k)
 _valtype(o::Tuple, k::Int) = fieldtype(typeof(o), k)
+_valtype(o::Type) = missing
 
 hasmultiindex(o) = true
+hasmultiindex(o::Type) = true
 hasmultiindex(o::AbstractDict) = false
 hasmultiindex(o::NamedTuple) = false
 hasmultiindex(o::Tuple) = false

src/cpython.jl

@@ -5,6 +5,8 @@ module CPython
     using Base: @kwdef
     using UnsafePointers: UnsafePtr
 
+    @enum PyGILState_STATE::Cint PyGILState_LOCKED=0 PyGILState_UNLOCKED=1
+
     const Py_LT = Cint(0)
     const Py_LE = Cint(1)
     const Py_EQ = Cint(2)
@@ -301,9 +303,7 @@ module CPython
 
     const PyTypePtr = Ptr{PyTypeObject}
 
-    macro pyglobal(name)
-        :(CONFIG.preloaded ? cglobal($(esc(name))) : dlsym(CONFIG.libptr, $(esc(name))))
-    end
+    pyglobal(name) = dlsym(CONFIG.libptr, name)
 
     macro cdef(name, rettype, argtypes)
         name isa QuoteNode && name.value isa Symbol || error("name must be a symbol, got $name")
@@ -320,7 +320,7 @@ module CPython
             function $jname($(args...))
                 ptr = $refname[]
                 if ptr == C_NULL
-                    ptr = $refname[] = @pyglobal($name)
+                    ptr = $refname[] = pyglobal($name)
                 end
                 ccall(ptr, $rettype, $argtypes, $(args...))
             end
@@ -344,6 +344,9 @@ module CPython
     @cdef :PyEval_SaveThread Ptr{Cvoid} ()
     @cdef :PyEval_RestoreThread Cvoid (Ptr{Cvoid},)
 
+    @cdef :PyGILState_Ensure PyGILState_STATE ()
+    @cdef :PyGILState_Release Cvoid (PyGILState_STATE,)
+
     @cdef :PyImport_ImportModule PyPtr (Cstring,)
     @cdef :PyImport_Import PyPtr (PyPtr,)
 
@@ -480,7 +483,7 @@ module CPython
     function PyObject_GetBuffer(o, b, flags)
         p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
         if p == C_NULL || p.get[] == C_NULL
-            PyErr_SetString(unsafe_load(Ptr{PyPtr}(@pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'")
+            PyErr_SetString(unsafe_load(Ptr{PyPtr}(pyglobal(:PyExc_TypeError))), "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'")
             return Cint(-1)
         end
         ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
@@ -500,7 +503,7 @@ module CPython
     end
 
     function PyOS_RunInputHook()
-        hook = unsafe_load(Ptr{Ptr{Cvoid}}(@pyglobal(:PyOS_InputHook)))
+        hook = unsafe_load(Ptr{Ptr{Cvoid}}(pyglobal(:PyOS_InputHook)))
         hook == C_NULL || ccall(hook, Cint, ())
         nothing
     end

src/gil.jl

@@ -1,35 +1,12 @@
-"""
-    gil_is_on()
-
-True if we own the GIL. If so, we can safely interact with Python.
-"""
-gil_is_on() = CONFIG.gilstate == C_NULL
-
-"""
-    gil_on()
-
-Ensure the GIL is on, so that we can safely interact with Python. Return true if the GIL was already on.
-"""
-gil_on() =
-    if gil_is_on()
-        true
+function with_gil(f)
+    if CONFIG.isembedded
+        g = C.PyGILState_Ensure()
+        try
+            f()
+        finally
+            C.PyGILState_Release(g)
+        end
     else
-        C.PyEval_RestoreThread(CONFIG.gilstate)
-        CONFIG.gilstate = C_NULL
-        false
-    end
-
-"""
-    gil_off()
-
-Ensure the GIL is off. Return true if the GIL was on.
-
-After calling this, you must not call any Python functions until `gil_on()` is called. (It is OK if the garbage collector frees Python objects with the GIL off: the finalizer will temporarily acquire it.)
-"""
-gil_off() =
-    if gil_is_on()
-        CONFIG.gilstate = C.PyEval_SaveThread()
-        true
-    else
-        false
+        f()
     end
+end