julia arrays now support buffer and array interface

Author: Christopher Doris

src/PyObject.jl

@@ -62,6 +62,7 @@ Base.show(io::IO, o::PyObject) = begin
 end
 
 function Base.show(io::IO, ::MIME"text/plain", o::PyObject)
+    prefix = get(io, :typeinfo, Any) != typeof(o)
     h, w = displaysize(io)
     h -= 3
     x = try
@@ -73,26 +74,29 @@ function Base.show(io::IO, ::MIME"text/plain", o::PyObject)
         if '\n' ∈ x
             # multiple lines
             # each one is truncated to one screen width
-            print(io, "py:")
-            h -= 1
+            if prefix
+                print(io, "py:")
+                h -= 1
+            end
             xs = split(x, '\n')
-            printlines(xs) =
-                for x in xs
+            printlines(xs, nl=true) =
+                for (i,x) in enumerate(xs)
+                    (nl || i>1) && print(io, '\n')
                     if length(x) ≤ w
-                        print(io, '\n', x)
+                        print(io, x)
                     else
-                        print(io, '\n', x[1:nextind(x, 0, w-1)], '…')
+                        print(io, x[1:nextind(x, 0, w-1)], '…')
                     end
                 end
             if length(xs) ≤ h
                 # all lines fit on screen
-                printlines(xs)
+                printlines(xs, prefix)
             else
                 # too many lines, skip the middle ones
                 h -= 1
                 h2 = cld(h, 2)
                 h3 = (length(xs)+1)-(h-h2)
-                printlines(xs[1:h2])
+                printlines(xs[1:h2], prefix)
                 linelen = min(
                     checkbounds(Bool, xs, h2) ? length(xs[h2]) : 0,
                     checkbounds(Bool, xs, h3) ? length(xs[h3]) : 0,
@@ -103,14 +107,16 @@ function Base.show(io::IO, ::MIME"text/plain", o::PyObject)
                 print(io, "\n", pad > 0 ? " "^pad : "", msg)
                 printlines(xs[h3:end])
             end
-        elseif length(x) ≤ w-4
+        elseif length(x) ≤ (prefix ? w-4 : w)
             # one short line
-            print(io, "py: ", x)
+            print(io, prefix ? "py: " : "", x)
             return
         else
             # one long line
-            println(io, "py:")
-            h -= 1
+            if prefix
+                println(io, "py:")
+                h -= 1
+            end
             a = h * w
             if length(x) ≤ a
                 # whole string fits on screen
@@ -127,7 +133,7 @@ function Base.show(io::IO, ::MIME"text/plain", o::PyObject)
             end
         end
     else
-        print(io, "py: ", x)
+        print(io, prefix ? "py: " : "", x)
     end
 end
 

src/old/PyObjectArray.jl renamed to src/PyObjectArray.jl

@@ -1,3 +1,11 @@
+"""
+    PyObjectArray(undef, dims...)
+    PyObjectArray(array)
+
+An array of `PyObject`s which supports the Python buffer protocol.
+
+Internally, the objects are stored as an array of pointers.
+"""
 mutable struct PyObjectArray{N} <: AbstractArray{PyObject, N}
     ptrs :: Array{CPyPtr, N}
     function PyObjectArray{N}(::UndefInitializer, dims::NTuple{N,Integer}) where {N}
@@ -34,10 +42,11 @@ function Base.getindex(x::PyObjectArray, i::Integer...)
 end
 
 function Base.setindex!(x::PyObjectArray, _v, i::Integer...)
-    v = convert(PyObject, _v)
+    vo = C.PyObject_From(_v)
+    isnull(vo) && pythrow()
     C.Py_DecRef(x.ptrs[i...])
-    pyincref!(v)
-    x.ptrs[i...] = pyptr(v)
+    C.Py_IncRef(vo)
+    x.ptrs[i...] = vo
     x
 end
 
@@ -46,3 +55,15 @@ function Base.deleteat!(x::PyObjectArray, i::Integer)
     deleteat!(x.ptrs, i)
     x
 end
+
+C._pyjlarray_get_buffer(o, buf, flags, x::PyObjectArray) =
+    C.pyjl_get_buffer_impl(o, buf, flags, pointer(x.ptrs), sizeof(CPyPtr), length(x), ndims(x), "O", size(x), strides(x.ptrs), true)
+
+C._pyjlarray_array_interface(x::PyObjectArray) =
+    C.PyDict_From(Dict(
+        "shape" => size(x),
+        "typestr" => "|O",
+        "data" => (UInt(pointer(x.ptrs)), false),
+        "strides" => strides(x.ptrs) .* sizeof(CPyPtr),
+        "version" => 3,
+    ))

src/Python.jl

@@ -94,6 +94,7 @@ include("PySet.jl")
 include("PyIterable.jl")
 include("PyIO.jl")
 include("PyBuffer.jl")
+include("PyObjectArray.jl")
 include("PyPandasDataFrame.jl")
 
 include("julia.jl")

src/cpython/arg.jl

@@ -68,42 +68,51 @@ end
 struct NODEFAULT end
 
 PyArg_Find(args::PyPtr, kwargs::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}) =
-    if i !== nothing && !isnull(args) && 0 ≤ i ≤ PyTuple_Size(args)
+    if i !== nothing && !isnull(args) && 0 ≤ i < PyTuple_Size(args)
         return PyTuple_GetItem(args, i)
     elseif k !== nothing && !isnull(kwargs) && (ro = PyDict_GetItemString(kwargs, k)) != PyPtr()
         return ro
     else
         return PyPtr()
     end
 
-PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr=PyPtr(), i::Union{Int,Nothing}=nothing, k::Union{String,Nothing}=nothing, d::Union{T,NODEFAULT}=NODEFAULT()) where {T} = begin
+"""
+    PyArg_GetArg(T, funcname, [args, i], [kwargs, k], [default])
+
+Attempt to find and convert the specified argument to a `T`. Return 0 on success, in which case the result can be retrieved with `takeresult(T)`. Return -1 on failure, with a Python error set.
+
+- `funcname::String` is the name of the function this is used in, for constructing error messages.
+- `args::PyPtr` is a tuple of arguments, and `i::Int` an index.
+- `kwargs::PyPtr` is a dict of keyword arguments, and `k::String` a key.
+- `default` specifies a default value if the argument is not found. NOTE: It need not be a `T`, so when retrieving the result use `takeresult(Union{T,typeof(default)})`.
+"""
+PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, kwargs::PyPtr, k::Union{String,Nothing}, d=NODEFAULT()) where {T} = begin
     ro = PyArg_Find(args, kwargs, i, k)
     if isnull(ro)
-        if k !== nothing
+        if d !== NODEFAULT()
+            putresult(d)
+            return 0
+        elseif k !== nothing
             PyErr_SetString(PyExc_TypeError(), "$name() did not get required argument '$k'")
-        elseif i !== nothing
+            return -1
+        elseif i !== nothing && i ≥ 0
             PyErr_SetString(PyExc_TypeError(), "$name() takes at least $(i+1) arguments (got $(isnull(args) ? 0 : PyTuple_Size(args)))")
+            return -1
         else
             error("impossible to satisfy this argument")
         end
-        return -1
     end
     r = PyObject_TryConvert(ro, T)
     if r == -1
         return -1
     elseif r == 0
-        if d === NODEFAULT()
-            PyErr_SetString(PyExc_TypeError(), "Argument $(k !== nothing ? "'$k'" : i !== nothing ? "$i" : error("impossible")) to $name() must be convertible to a Julia '$T'")
-            return -1
-        else
-            putresult(d)
-            return 0
-        end
+        PyErr_SetString(PyExc_TypeError(), "Argument $(k !== nothing ? "'$k'" : i !== nothing ? "$i" : error("impossible")) to $name() must be convertible to a Julia '$T'")
+        return -1
     else
         return 0
     end
 end
-PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Union{Int,Nothing}, k::Union{String,Nothing}=nothing, d::Union{T,NODEFAULT}=NODEFAULT()) where {T} =
-    PyArg_GetArg(T, name, args, PyPtr(), i, k, d)
-PyArg_GetArg(::Type{T}, name::String, args::PyPtr, kwargs::PyPtr, k::Union{String,Nothing}, d::Union{T, NODEFAULT}=NODEFAULT()) where {T} =
-    PyArg_GetArg(T, name, args, kwargs, nothing, k, d)
+PyArg_GetArg(::Type{T}, name::String, args::PyPtr, i::Int, d=NODEFAULT()) where {T} =
+    PyArg_GetArg(T, name, args, i, PyPtr(), nothing, d)
+PyArg_GetArg(::Type{T}, name::String, kwargs::PyPtr, k::String, d=NODEFAULT()) where {T} =
+    PyArg_GetArg(T, name, PyPtr(), nothing, kwargs, k, d)

src/cpython/juliaany.jl

@@ -41,7 +41,9 @@ PyJuliaValue_From(x) = PyJuliaAnyValue_New(x)
 
 pyjlany_repr(xo::PyPtr) = try
     x = PyJuliaValue_GetValue(xo)
-    s = "<jl $(repr(x))>"
+    # s = "<jl $(repr(x))>"
+    s = sprint((io,x)->show(io,MIME"text/plain"(),x), x, context=:limit=>true)
+    s = string("jl:", '\n' in s ? '\n' : ' ', s)
     PyUnicode_From(s)
 catch err
     PyErr_SetJuliaError(err)