[GR-34608] Fixes for urllib3

PullRequest: graalpython/2129

Author: msimacek

graalpython/com.oracle.graal.python.cext/include/pyhash.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -48,6 +48,7 @@ extern "C" {
 #endif
 
 PyAPI_FUNC(Py_hash_t) _Py_HashDouble(double);
+PyAPI_FUNC(Py_hash_t) _Py_HashPointer(void*);
 PyAPI_FUNC(Py_hash_t) _Py_HashBytes(const void*, Py_ssize_t);
 
 extern long _PyHASH_INF;

graalpython/com.oracle.graal.python.cext/src/descrobject.c

@@ -62,7 +62,8 @@ PyObject* PyDictProxy_New(PyObject *mapping) {
     return (PyObject*) UPCALL_CEXT_O(_jls_PyDictProxy_New, native_to_java(mapping));
 }
 
-typedef PyObject* (*PyDescr_NewClassMethod_fun_t)(void* name,
+typedef PyObject* (*PyDescr_NewClassMethod_fun_t)(PyMethodDef* methodDef,
+                                                    void* name,
                                                     const char* doc,
                                                     int flags,
                                                     int wrapper,
@@ -71,7 +72,8 @@ typedef PyObject* (*PyDescr_NewClassMethod_fun_t)(void* name,
 UPCALL_TYPED_ID(PyDescr_NewClassMethod, PyDescr_NewClassMethod_fun_t);
 PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) {
     int flags = method->ml_flags;
-    return _jls_PyDescr_NewClassMethod(polyglot_from_string(method->ml_name, SRC_CS),
+    return _jls_PyDescr_NewClassMethod(method,
+                    polyglot_from_string(method->ml_name, SRC_CS),
                     method->ml_doc,
                     flags,
                     get_method_flags_wrapper(flags),

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -242,6 +242,11 @@ int PyDict_Merge(PyObject *a, PyObject *b, int override) {
     return UPCALL_CEXT_I(_jls_PyDict_Merge, native_to_java(a), native_to_java(b), override);
 }
 
+UPCALL_ID(PyDict_Keys);
+PyObject * PyDict_Keys(PyObject *dict) {
+    return UPCALL_CEXT_O(_jls_PyDict_Keys, native_to_java(dict));
+}
+
 UPCALL_ID(PyDict_Values);
 PyObject * PyDict_Values(PyObject *dict) {
     return UPCALL_CEXT_O(_jls_PyDict_Values, native_to_java(dict));

graalpython/com.oracle.graal.python.cext/src/errors.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -169,9 +169,13 @@ PyObject* PyErr_Format(PyObject* exception, const char* fmt, ...) {
     return NULL;
 }
 
-UPCALL_ID(PyErr_WriteUnraisable);
 void PyErr_WriteUnraisable(PyObject *obj) {
-    UPCALL_CEXT_VOID(_jls_PyErr_WriteUnraisable, native_to_java(obj));
+    _PyErr_WriteUnraisableMsg(NULL, obj);
+}
+
+UPCALL_ID(_PyErr_WriteUnraisableMsg);
+void _PyErr_WriteUnraisableMsg(const char *err_msg_str, PyObject *obj) {
+    UPCALL_CEXT_VOID(_jls__PyErr_WriteUnraisableMsg, err_msg_str != NULL ? polyglot_from_string(err_msg_str, SRC_CS) : NULL, native_to_java(obj));
 }
 
 UPCALL_ID(PyErr_Display);

graalpython/com.oracle.graal.python.cext/src/methodobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -44,7 +44,8 @@ typedef PyObject *(*PyCFunction)(PyObject *, PyObject *);
 
 PyTypeObject PyCFunction_Type = PY_TRUFFLE_TYPE_WITH_VECTORCALL("builtin_function_or_method", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | _Py_TPFLAGS_HAVE_VECTORCALL, sizeof(PyCFunctionObject), offsetof(PyCFunctionObject, vectorcall));
 
-typedef PyObject* (*PyCFunction_NewEx_fun_t)(void* name, 
+typedef PyObject* (*PyCFunction_NewEx_fun_t)(PyMethodDef* methodDef,
+                                                    void* name,
                                                     void* methObj, 
                                                     int flags, 
                                                     int wrapper,
@@ -53,7 +54,8 @@ typedef PyObject* (*PyCFunction_NewEx_fun_t)(void* name,
                                                     const char* doc);
 UPCALL_TYPED_ID(PyCFunction_NewEx, PyCFunction_NewEx_fun_t);
 PyObject* PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) {
-    return _jls_PyCFunction_NewEx(polyglot_from_string(ml->ml_name, SRC_CS),
+    return _jls_PyCFunction_NewEx(ml,
+                                               polyglot_from_string(ml->ml_name, SRC_CS),
                                                function_pointer_to_java(ml->ml_meth),
                                                ml->ml_flags,
                                                get_method_flags_wrapper(ml->ml_flags),

graalpython/com.oracle.graal.python.cext/src/moduleobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -57,24 +57,21 @@ PyModuleDef_Init(struct PyModuleDef* def)
     return (PyObject*)def;
 }
 
-//                                  cls         dict     name    cfunc  flags  sig  doc
-typedef int (*AddFunction_fun_t)(PyObject *, PyObject *, void *, void *, int , int, char *);
-UPCALL_TYPED_ID(AddFunction, AddFunction_fun_t);
+//                                       method_def     module      name          cfunc   flags sig  doc
+typedef int (*AddFunctionToModule_fun_t)(PyMethodDef *, PyObject *, const char *, void *, int , int, char *);
+UPCALL_TYPED_ID(AddFunctionToModule, AddFunctionToModule_fun_t);
 int PyModule_AddFunctions(PyObject* mod, PyMethodDef* methods) {
     if (!methods) {
         return -1;
     }
-    int idx = 0;
-    PyMethodDef def = methods[idx];
-    while (def.ml_name != NULL) {
-        _jls_AddFunction(native_to_java(mod),
-					   NULL,
-                       polyglot_from_string(def.ml_name, SRC_CS),
-                       function_pointer_to_java(def.ml_meth),
-                       def.ml_flags,
-                       get_method_flags_wrapper(def.ml_flags),
-					   (def.ml_doc ? def.ml_doc : ""));
-        def = methods[++idx];
+    for (PyMethodDef* def = methods; def->ml_name != NULL; def++) {
+        _jls_AddFunctionToModule(def,
+                       native_to_java(mod),
+                       polyglot_from_string(def->ml_name, SRC_CS),
+                       function_pointer_to_java(def->ml_meth),
+                       def->ml_flags,
+                       get_method_flags_wrapper(def->ml_flags),
+					   def->ml_doc);
     }
     return 0;
 }

graalpython/com.oracle.graal.python.cext/src/pyhash.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -59,3 +59,16 @@ void initialize_hashes() {
 Py_hash_t _Py_HashBytes(const void *src, Py_ssize_t len) {
     return UPCALL_L(PY_BUILTIN, polyglot_from_string("hash", SRC_CS), polyglot_from_string(src, "ascii"));
 }
+
+/* taken from CPython */
+Py_hash_t _Py_HashPointer(void *p) {
+    Py_hash_t x;
+    size_t y = (size_t)p;
+    /* bottom 3 or 4 bits are likely to be 0; rotate y by 4 to avoid
+       excessive hash collisions for dicts and sets */
+    y = (y >> 4) | (y << (8 * SIZEOF_VOID_P - 4));
+    x = (Py_hash_t)y;
+    if (x == -1)
+        x = -2;
+    return x;
+}

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -392,18 +392,18 @@ int add_getset(PyTypeObject* cls, PyObject* type_dict, char* name, getter getter
                         closure);
 }
 
-//                                  cls             dict     name    cfunc  flags  sig  doc
-typedef int (*AddFunction_fun_t)(PyTypeObject *, PyObject *, void *, void *, int , int, char *);
-UPCALL_TYPED_ID(AddFunction, AddFunction_fun_t);
-static void add_method_or_slot(PyTypeObject* cls, PyObject* type_dict, char* name, void* result_conversion, void* meth, int flags, int signature, char* doc) {
-	    void *resolved_meth = function_pointer_to_java(meth);
-        _jls_AddFunction(cls,
+//                                     method_def     cls             dict        name          cfunc   flags sig  doc
+typedef int (*AddFunctionToType_fun_t)(PyMethodDef *, PyTypeObject *, PyObject *, const char *, void *, int , int, char *);
+UPCALL_TYPED_ID(AddFunctionToType, AddFunctionToType_fun_t);
+static void add_method(PyTypeObject* cls, PyObject* type_dict, PyMethodDef* def) {
+        _jls_AddFunctionToType(def,
+                       cls,
                        native_to_java(type_dict),
-                       polyglot_from_string(name, SRC_CS),
-                       native_pointer_to_java(result_conversion != NULL ? pytruffle_decorate_function(resolved_meth, result_conversion) : resolved_meth),
-                       flags,
-                       (signature != 0 ? signature : get_method_flags_wrapper(flags)),
-                       doc);
+                       polyglot_from_string(def->ml_name, SRC_CS),
+                       function_pointer_to_java(def->ml_meth),
+                       def->ml_flags,
+                       get_method_flags_wrapper(def->ml_flags),
+                       def->ml_doc);
 }
 
 typedef int (*add_slot_fun_t)(PyTypeObject *, PyObject *, void *, void *, int , int, char *);
@@ -440,11 +440,6 @@ int PyType_Ready(PyTypeObject* cls) {
 	} while(0)
 
 #define ADD_IF_MISSING(attr, def) if (!(attr)) { attr = def; }
-#define ADD_METHOD(m) ADD_METHOD_OR_SLOT(m.ml_name, NULL, m.ml_meth, m.ml_flags, NULL, m.ml_doc)
-#define ADD_METHOD_OR_SLOT(__name__, __res_conv__, __meth__, __flags__, __signature__, __doc__) \
-	if (__meth__) { \
-            add_method_or_slot(cls, dict, (__name__), (__res_conv__), (__meth__), (__flags__), (__signature__), (__doc__)); \
-	}
 #define ADD_SLOT_CONV(__name__, __meth__, __flags__, __signature__) add_slot(cls, dict, (__name__), (__meth__), (__flags__), (__signature__), NULL)
 
     Py_ssize_t n;
@@ -520,13 +515,9 @@ int PyType_Ready(PyTypeObject* cls) {
         cls->tp_dict = dict;
     }
 
-    PyMethodDef* methods = cls->tp_methods;
-    if (methods) {
-        int idx = 0;
-        PyMethodDef def = methods[idx];
-        while (def.ml_name != NULL) {
-            ADD_METHOD(def);
-            def = methods[++idx];
+    if (cls->tp_methods) {
+        for (PyMethodDef* def = cls->tp_methods; def->ml_name != NULL; def++) {
+            add_method(cls, dict, def);
         }
     }
 
@@ -760,9 +751,7 @@ int PyType_Ready(PyTypeObject* cls) {
     return 0;
 
 #undef ADD_IF_MISSING
-#undef ADD_METHOD
 #undef ADD_SLOT
-#undef ADD_METHOD_OR_SLOT
 }
 
 MUST_INLINE static int valid_identifier(PyObject *s) {

graalpython/com.oracle.graal.python.cext/zlib/zlib.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -1108,13 +1108,6 @@ int zlib_Decompress_flush(zlib_stream *zst, ssize_t length) {
             error_occurred(zst, INFLATE_END_ERROR);
             return err;
         }
-    } else if (err != Z_OK && err != Z_BUF_ERROR) {
-        /* We will only get Z_BUF_ERROR if the output buffer was full
-           but there wasn't more output when we tried again, so it is
-           not an error condition.
-        */
-        error_occurred(zst, INFLATE_FLUSH_ERROR);
-        return err;
     }
     if (zst->output->size) {
         zst->output_size = zst->zst.next_out - zst->output->buf;

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_dict.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -48,11 +48,18 @@ def _reference_get_item(args):
         return d.get(args[1])
     except Exception:
         return None
-    
+
+
+def _reference_keys(args):
+    d = args[0]
+    return list(d.keys())
+
+
 def _reference_values(args):
     d = args[0]
     return list(d.values())
-    
+
+
 def _reference_pop(args):
     d = args[0]
     if(len(args) == 2):
@@ -505,6 +512,15 @@ def compile_module(self, name):
         arguments=["PyObject* a", "PyObject* b", "int override"],
         cmpfunc=unhandled_error_compare
     )
+
+    # PyDict_Keys
+    test_PyDict_Keys = CPyExtFunction(
+        _reference_keys,
+        lambda: (({},), ({'a': "hello"},)),
+        resultspec="O",
+        argspec="O",
+        cmpfunc=unhandled_error_compare
+    )
 
     # PyDict_Values
     test_PyDict_Values = CPyExtFunction(

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_err.py

@@ -367,6 +367,7 @@ def compile_module(self, name):
         resultspec="O",
         argspec='Osn',
         arguments=["PyObject* category", "char* msg", "Py_ssize_t level"],
+        stderr_validator=lambda args, stderr: "UserWarning: custom warning" in stderr,
         cmpfunc=unhandled_error_compare
     )
 
@@ -389,17 +390,36 @@ def compile_module(self, name):
             (None,),
             ("hello",),
         ),
-        resultspec="O",
         argspec='O',
         arguments=["PyObject* obj"],
         code="""void wrap_PyErr_WriteUnraisable(PyObject* object) {
-            PyErr_SetString(PyExc_RuntimeError, "unraisable exception");
+            PyErr_SetString(PyExc_RuntimeError, "unraisable_exception");
             if (object == Py_None)
                 object = NULL;
             PyErr_WriteUnraisable(object);
         }""",
         callfunction="wrap_PyErr_WriteUnraisable",
-        stderr_validator=lambda args, stderr: 'unraisable exception' in stderr,
+        stderr_validator=lambda args, stderr: "RuntimeError: unraisable_exception" in stderr,
+        cmpfunc=unhandled_error_compare
+    )
+
+    test_PyErr_WriteUnraisableMsg = CPyExtFunctionVoid(
+        lambda args: None,
+        lambda: (
+            (None,),
+            ("hello",),
+        ),
+        code="""void wrap_PyErr_WriteUnraisableMsg(PyObject* object) {
+                PyErr_SetString(PyExc_RuntimeError, "unraisable_exception");
+                if (object == Py_None)
+                    object = NULL;
+                _PyErr_WriteUnraisableMsg("in my function", object);
+             }
+             """,
+        argspec='O',
+        arguments=["PyObject* obj"],
+        callfunction="wrap_PyErr_WriteUnraisableMsg",
+        stderr_validator=lambda args, stderr: "RuntimeError: unraisable_exception" in stderr and "Exception ignored in my function:" in stderr,
         cmpfunc=unhandled_error_compare
     )