[GR-48686] Implement os.chown and os.getgroups

PullRequest: graalpython/2984

Author: msimacek

docs/contributor/MISSING.md

@@ -9,8 +9,6 @@ This is just a snapshot as of 2023-09-14.
  * **_dbm**:  Use from C
  * **_gdbm**:  Use from C
  * **_tkinter**: Should be used from C
- * **audioop**:  Should be useable from C
- * **nis**:  We should just use the C module
  * **syslog**:  Access to syslog. We should probably just use this from the C module.
 
 #### These are not strictly needed for now
@@ -24,10 +22,14 @@ This is just a snapshot as of 2023-09-14.
 #### These we probably won't support
  * **_opcode**:  We won't have it, our opcodes are different from CPython.
  * **_symtable**:  Interface for the compilers internal symboltable.
- * **ossaudiodev**:  Not needed, it's for Linux OSS audio
 
 #### These we should re-implement
  * **_uuid**: Can be implemented ourselves, is just 1 function
  * **grp**:  UNIX group file access. Should be added to our POSIX APIs
  * **spwd**:  UNIX shadow password file access. Should be added to our POSIX APIs
  * **parser**:  We need to implement this for our parser
+
+#### Deprecated in CPython, won't implement
+ * **audioop**:  Scheduled for removal in 3.13
+ * **ossaudiodev**:  Scheduled for removal in 3.13
+ * **nis**:  Scheduled for removal in 3.13

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_concurrent_futures.txt

@@ -46,7 +46,6 @@
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnFailingInitializerTest.test_initializer
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnInitializerTest.test_initializer
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnProcessPoolExecutorTest.test_idle_process_reuse_multiple
-*graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnProcessPoolExecutorTest.test_idle_process_reuse_one
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnProcessPoolExecutorTest.test_killed_child
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnProcessPoolExecutorTest.test_map
 *graalpython.lib-python.3.test.test_concurrent_futures.ProcessPoolSpawnProcessPoolExecutorTest.test_map_chunksize

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_importlib.txt

@@ -837,7 +837,6 @@
 *test.test_importlib.test_lazy.LazyLoaderTests.test_new_attr
 *test.test_importlib.test_locks.Frozen_DeadlockAvoidanceTests.test_deadlock
 *test.test_importlib.test_locks.Frozen_DeadlockAvoidanceTests.test_no_deadlock
-*test.test_importlib.test_locks.Frozen_LifetimeTests.test_all_locks
 *test.test_importlib.test_locks.Frozen_LifetimeTests.test_lock_lifetime
 *test.test_importlib.test_locks.Frozen_ModuleLockAsRLockTests.test_acquire_contended
 *test.test_importlib.test_locks.Frozen_ModuleLockAsRLockTests.test_acquire_destroy
@@ -851,7 +850,6 @@
 *test.test_importlib.test_locks.Frozen_ModuleLockAsRLockTests.test_weakref_exists
 *test.test_importlib.test_locks.Source_DeadlockAvoidanceTests.test_deadlock
 *test.test_importlib.test_locks.Source_DeadlockAvoidanceTests.test_no_deadlock
-*test.test_importlib.test_locks.Source_LifetimeTests.test_all_locks
 *test.test_importlib.test_locks.Source_LifetimeTests.test_lock_lifetime
 *test.test_importlib.test_locks.Source_ModuleLockAsRLockTests.test_acquire_contended
 *test.test_importlib.test_locks.Source_ModuleLockAsRLockTests.test_acquire_destroy

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java

@@ -109,7 +109,6 @@
 import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider;
 import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
 import com.oracle.graal.python.nodes.util.CastToJavaLongLossyNode;
-import com.oracle.graal.python.nodes.util.CastToJavaUnsignedLongNode;
 import com.oracle.graal.python.nodes.util.CastToTruffleStringNode;
 import com.oracle.graal.python.runtime.GilNode;
 import com.oracle.graal.python.runtime.PosixConstants;
@@ -126,6 +125,7 @@
 import com.oracle.graal.python.runtime.exception.PythonExitException;
 import com.oracle.graal.python.runtime.object.PythonObjectFactory;
 import com.oracle.graal.python.runtime.sequence.PSequence;
+import com.oracle.graal.python.runtime.sequence.storage.LongSequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.ObjectSequenceStorage;
 import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
 import com.oracle.graal.python.util.OverflowException;
@@ -720,6 +720,24 @@ Object setsid(VirtualFrame frame,
         }
     }
 
+    @Builtin(name = "getgroups")
+    @GenerateNodeFactory
+    abstract static class GetGroupsNode extends PythonBuiltinNode {
+        @Specialization
+        Object getgroups(VirtualFrame frame,
+                        @Bind("this") Node inliningTarget,
+                        @CachedLibrary("getPosixSupport()") PosixSupportLibrary posixLib,
+                        @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode,
+                        @Cached PythonObjectFactory factory) {
+            try {
+                long[] groups = posixLib.getgroups(getPosixSupport());
+                return factory.createList(new LongSequenceStorage(groups));
+            } catch (PosixException e) {
+                throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e);
+            }
+        }
+    }
+
     @Builtin(name = "openpty")
     @GenerateNodeFactory
     public abstract static class OpenPtyNode extends PythonBuiltinNode {
@@ -2212,6 +2230,134 @@ PNone chmodFollow(VirtualFrame frame, PosixFd fd, int mode, int dirFd, @Suppress
         }
     }
 
+    @Builtin(name = "fchown", parameterNames = {"fd", "uid", "gid"})
+    @ArgumentClinic(name = "fd", conversion = ClinicConversion.Int)
+    @ArgumentClinic(name = "uid", conversionClass = UidConversionNode.class)
+    @ArgumentClinic(name = "gid", conversionClass = GidConversionNode.class)
+    @GenerateNodeFactory
+    abstract static class FChownNode extends PythonTernaryClinicBuiltinNode {
+        @Specialization
+        Object chown(VirtualFrame frame, int fd, long uid, long gid,
+                        @Bind("this") Node inliningTarget,
+                        @Cached SysModuleBuiltins.AuditNode auditNode,
+                        @CachedLibrary(limit = "1") PosixSupportLibrary posixLib,
+                        @Cached GilNode gil,
+                        @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) {
+            auditNode.audit(inliningTarget, "os.chown", fd, uid, gid, -1);
+            try {
+                gil.release(true);
+                try {
+                    posixLib.fchown(getPosixSupport(), fd, uid, gid);
+                } finally {
+                    gil.acquire();
+                }
+            } catch (PosixException e) {
+                throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, fd);
+            }
+            return PNone.NONE;
+        }
+
+        @Override
+        protected ArgumentClinicProvider getArgumentClinic() {
+            return PosixModuleBuiltinsClinicProviders.FChownNodeClinicProviderGen.INSTANCE;
+        }
+    }
+
+    @Builtin(name = "lchown", parameterNames = {"path", "uid", "gid"})
+    @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"})
+    @ArgumentClinic(name = "uid", conversionClass = UidConversionNode.class)
+    @ArgumentClinic(name = "gid", conversionClass = GidConversionNode.class)
+    @GenerateNodeFactory
+    abstract static class LChownNode extends PythonTernaryClinicBuiltinNode {
+        @Specialization
+        Object chown(VirtualFrame frame, PosixPath path, long uid, long gid,
+                        @Bind("this") Node inliningTarget,
+                        @Cached SysModuleBuiltins.AuditNode auditNode,
+                        @CachedLibrary(limit = "1") PosixSupportLibrary posixLib,
+                        @Cached GilNode gil,
+                        @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) {
+            auditNode.audit(inliningTarget, "os.chown", path.originalObject, uid, gid, -1);
+            try {
+                gil.release(true);
+                try {
+                    posixLib.fchownat(getPosixSupport(), AT_FDCWD.value, path.value, uid, gid, false);
+                } finally {
+                    gil.acquire();
+                }
+            } catch (PosixException e) {
+                throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path.originalObject);
+            }
+            return PNone.NONE;
+        }
+
+        @Override
+        protected ArgumentClinicProvider getArgumentClinic() {
+            return PosixModuleBuiltinsClinicProviders.LChownNodeClinicProviderGen.INSTANCE;
+        }
+    }
+
+    @Builtin(name = "chown", minNumOfPositionalArgs = 3, parameterNames = {"path", "uid", "gid"}, keywordOnlyNames = {"dir_fd", "follow_symlinks"})
+    @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "true"})
+    @ArgumentClinic(name = "uid", conversionClass = UidConversionNode.class)
+    @ArgumentClinic(name = "gid", conversionClass = GidConversionNode.class)
+    @ArgumentClinic(name = "dir_fd", conversionClass = DirFdConversionNode.class)
+    @ArgumentClinic(name = "follow_symlinks", conversion = ClinicConversion.Boolean, defaultValue = "true")
+    @GenerateNodeFactory
+    abstract static class ChownNode extends PythonClinicBuiltinNode {
+        @Specialization
+        Object chown(VirtualFrame frame, PosixPath path, long uid, long gid, int dirFd, boolean followSymlinks,
+                        @Bind("this") Node inliningTarget,
+                        @Shared @Cached SysModuleBuiltins.AuditNode auditNode,
+                        @Shared @CachedLibrary(limit = "1") PosixSupportLibrary posixLib,
+                        @Shared @Cached GilNode gil,
+                        @Shared @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) {
+            auditNode.audit(inliningTarget, "os.chown", path.originalObject, uid, gid, dirFd != AT_FDCWD.value ? dirFd : -1);
+            try {
+                gil.release(true);
+                try {
+                    posixLib.fchownat(getPosixSupport(), dirFd, path.value, uid, gid, followSymlinks);
+                } finally {
+                    gil.acquire();
+                }
+            } catch (PosixException e) {
+                throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, path.originalObject);
+            }
+            return PNone.NONE;
+        }
+
+        @Specialization
+        Object chown(VirtualFrame frame, PosixFd fd, long uid, long gid, int dirFd, boolean followSymlinks,
+                        @Bind("this") Node inliningTarget,
+                        @Shared @Cached SysModuleBuiltins.AuditNode auditNode,
+                        @Shared @CachedLibrary(limit = "1") PosixSupportLibrary posixLib,
+                        @Shared @Cached GilNode gil,
+                        @Shared @Cached PConstructAndRaiseNode.Lazy constructAndRaiseNode) {
+            if (dirFd != AT_FDCWD.value) {
+                throw raise(ValueError, ErrorMessages.CANT_SPECIFY_BOTH_DIR_FD_AND_FD);
+            }
+            if (followSymlinks) {
+                throw raise(ValueError, ErrorMessages.CANNOT_USE_FD_AND_FOLLOW_SYMLINKS_TOGETHER, "chown");
+            }
+            auditNode.audit(inliningTarget, "os.chown", fd.originalObject, uid, gid, -1);
+            try {
+                gil.release(true);
+                try {
+                    posixLib.fchown(getPosixSupport(), fd.fd, uid, gid);
+                } finally {
+                    gil.acquire();
+                }
+            } catch (PosixException e) {
+                throw constructAndRaiseNode.get(inliningTarget).raiseOSErrorFromPosixException(frame, e, fd.originalObject);
+            }
+            return PNone.NONE;
+        }
+
+        @Override
+        protected ArgumentClinicProvider getArgumentClinic() {
+            return PosixModuleBuiltinsClinicProviders.ChownNodeClinicProviderGen.INSTANCE;
+        }
+    }
+
     @Builtin(name = "readlink", minNumOfPositionalArgs = 1, parameterNames = {"path"}, varArgsMarker = true, keywordOnlyNames = {"dir_fd"}, doc = "readlink(path, *, dir_fd=None) -> path\n" +
                     "\nReturn a string representing the path to which the symbolic link points.\n")
     @ArgumentClinic(name = "path", conversionClass = PathConversionNode.class, args = {"false", "false"})
@@ -3240,10 +3386,11 @@ public static PidtConversionNode create() {
         }
     }
 
-    /**
-     * Emulates CPython's {@code _Py_Uid_Converter()}. Always returns an {@code long}.
-     */
-    public abstract static class UidConversionNode extends ArgumentCastNodeWithRaise {
+    @GenerateCached(false)
+    public abstract static class AbstractIdConversionNode extends ArgumentCastNodeWithRaise {
+
+        private static final long MAX_UINT32 = (1L << 32) - 1;
+
         public abstract long executeLong(VirtualFrame frame, Object value);
 
         @Specialization
@@ -3261,39 +3408,69 @@ long doLong(long value) {
         long doGeneric(VirtualFrame frame, Object value,
                         @Bind("this") Node inliningTarget,
                         @Cached PyNumberIndexNode pyNumberIndexNode,
-                        @Cached PyLongAsLongAndOverflowNode asLongAndOverflowNode,
-                        @Cached CastToJavaUnsignedLongNode asUnsignedLong) {
+                        @Cached PyLongAsLongNode asLongNode) {
             Object index;
             try {
                 index = pyNumberIndexNode.execute(frame, inliningTarget, value);
             } catch (PException ex) {
-                throw raise(TypeError, ErrorMessages.UID_SHOULD_BE_INTEGER_NOT_P, value);
-            }
-            try {
-                return checkValue(asLongAndOverflowNode.execute(frame, inliningTarget, index));
-            } catch (OverflowException e) {
-                // fall through
+                throw raise(TypeError, ErrorMessages.S_SHOULD_BE_INTEGER_NOT_P, getIdName(), value);
             }
-            return asUnsignedLong.execute(inliningTarget, index);
-            // We have no means to distinguish overflow/underflow so we just let any OverflowError
-            // from asUnsignedLong fall through. It will not have the same message as CPython, but
-            // still correct type.
+            /*
+             * We have no means to distinguish overflow/underflow, so we just let any OverflowError
+             * from asLongNode fall through. It will not have the same message as CPython, but still
+             * correct type.
+             */
+            return checkValue(asLongNode.execute(frame, inliningTarget, index));
         }
 
         private long checkValue(long value) {
+            // Note that -1 is intentionally allowed
             if (value < -1) {
-                throw raise(OverflowError, ErrorMessages.UID_IS_LESS_THAN_MINIMUM);
+                throw raise(OverflowError, ErrorMessages.S_IS_LESS_THAN_MINIMUM, getIdName());
+            } else if (value > MAX_UINT32) {
+                /* uid_t is uint32_t on Linux */
+                throw raise(OverflowError, ErrorMessages.S_IS_GREATER_THAN_MAXIUMUM, getIdName());
             }
             return value;
         }
 
+        protected abstract String getIdName();
+    }
+
+    /**
+     * Emulates CPython's {@code _Py_Uid_Converter()}. Always returns a {@code long}.
+     */
+    public abstract static class UidConversionNode extends AbstractIdConversionNode {
+
+        @Override
+        protected String getIdName() {
+            return "uid";
+        }
+
         @ClinicConverterFactory(shortCircuitPrimitive = {PrimitiveType.Int, PrimitiveType.Long})
         @NeverDefault
         public static UidConversionNode create() {
             return PosixModuleBuiltinsFactory.UidConversionNodeGen.create();
         }
     }
 
+    /**
+     * Emulates CPython's {@code _Py_Gid_Converter()}. Always returns a {@code long}.
+     */
+    public abstract static class GidConversionNode extends AbstractIdConversionNode {
+
+        @Override
+        protected String getIdName() {
+            return "gid";
+        }
+
+        @ClinicConverterFactory(shortCircuitPrimitive = {PrimitiveType.Int, PrimitiveType.Long})
+        @NeverDefault
+        public static GidConversionNode create() {
+            return PosixModuleBuiltinsFactory.GidConversionNodeGen.create();
+        }
+    }
+
     /**
      * Represents the result of {@code path_t} conversion. Similar to CPython's {@code path_t}
      * structure, but only contains the results of the conversion, not the conversion parameters.

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/lib/PyLongAsLongNode.java

@@ -58,8 +58,7 @@
 
 /**
  * Equivalent of CPython's {@code PyLong_AsLong}. Converts an object into a Java long using it's
- * {@code __index__} or (deprecated) {@code __int__} method. Raises {@code OverflowError} on
- * overflow.
+ * {@code __index__} method. Raises {@code OverflowError} on overflow.
  */
 @GenerateUncached
 @GenerateInline(inlineByDefault = true)

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/ErrorMessages.java

@@ -198,6 +198,7 @@ public abstract class ErrorMessages {
     public static final TruffleString CANNOT_SPECIFY_BOTH_COMMA_AND_UNDERSCORE = tsLiteral("Cannot specify both ',' and '_'.");
     public static final TruffleString CANNOT_SPECIFY_C_WITH_C = tsLiteral("Cannot specify '%c' with '%c'.");
     public static final TruffleString CANNOT_USE_FD_AND_FOLLOW_SYMLINKS_TOGETHER = tsLiteral("%s: cannot use fd and follow_symlinks together");
+    public static final TruffleString CANT_SPECIFY_BOTH_DIR_FD_AND_FD = tsLiteral("%s: can't specify both dir_fd and fd");
     public static final TruffleString CANNOT_CONVERT_FLOAT_INFINITY_TO_INTEGER = tsLiteral("cannot convert float infinity to integer");
     public static final TruffleString CANNOT_CONVERT_FLOAT_NAN_TO_INTEGER = tsLiteral("cannot convert float NaN to integer");
     public static final TruffleString CANT_CAPTURE_NAME_UNDERSCORE_IN_PATTERNS = tsLiteral("can't capture name '_' in patterns");
@@ -859,8 +860,9 @@ public abstract class ErrorMessages {
     public static final TruffleString LINE_BUFFERING_ISNT_SUPPORTED = tsLiteral("line buffering (buffering=1) isn't supported in binary mode, the default buffer size will be used");
     public static final TruffleString UNCLOSED_FILE = tsLiteral("unclosed file %r");
     public static final TruffleString MAXIMUM_RECURSION_DEPTH_EXCEEDED = tsLiteral("maximum recursion depth exceeded");
-    public static final TruffleString UID_IS_LESS_THAN_MINIMUM = tsLiteral("uid is less than minimum");
-    public static final TruffleString UID_SHOULD_BE_INTEGER_NOT_P = tsLiteral("uid should be integer, not %p");
+    public static final TruffleString S_IS_LESS_THAN_MINIMUM = tsLiteral("%s is less than minimum");
+    public static final TruffleString S_IS_GREATER_THAN_MAXIUMUM = tsLiteral("%s is greater than maxiumum");
+    public static final TruffleString S_SHOULD_BE_INTEGER_NOT_P = tsLiteral("%s should be integer, not %p");
     public static final TruffleString RANGE_OBJ_IDX_OUT_OF_RANGE = tsLiteral("range object index out of range");
     public static final TruffleString NUMBER_OF_BITS_MUST_BE_NON_NEGATIVE = tsLiteral("number of bits must be non-negative");
     public static final TruffleString TIMEOUT_MUST_BE_NON_NEG_NUM = tsLiteral("'timeout' must be a non-negative number");