ARROW-12068: [Python] Stop using distutils

LICENSE.txt

@@ -2209,3 +2209,12 @@ The files in cpp/src/arrow/vendored/fast_float/ contain code from
 https://github.com/lemire/fast_float
 
 which is made available under the Apache License 2.0.
+
+--------------------------------------------------------------------------------
+
+The file python/pyarrow/vendored/version.py contains code from
+
+https://github.com/pypa/packaging/
+
+which is made available under both the Apache license v2.0 and the
+BSD 2-clause license.

dev/release/rat_exclude_files.txt

@@ -136,6 +136,7 @@ python/MANIFEST.in
 python/manylinux1/.dockerignore
 python/pyarrow/includes/__init__.pxd
 python/pyarrow/tests/__init__.py
+python/pyarrow/vendored/*
 python/requirements*.txt
 pax_global_header
 MANIFEST.in

dev/tasks/conda-recipes/azure.clean.yml

@@ -13,7 +13,7 @@ jobs:
     displayName: Clone arrow
 
   - script: |
-      conda install -y -c conda-forge pandas anaconda-client
+      conda install -y -c conda-forge pandas anaconda-client packaging
     displayName: Install requirements
 
   - script: |

dev/tasks/conda-recipes/clean.py

@@ -1,4 +1,3 @@
-from distutils.version import LooseVersion
 from subprocess import check_output, check_call
 from typing import List
 
@@ -7,6 +6,8 @@
 import pandas as pd
 import sys
 
+from packaging.version import Version
+
 
 VERSIONS_TO_KEEP = 5
 PACKAGES = [
@@ -44,7 +45,7 @@ def packages_to_delete(package_name: str, platform: str) -> List[str]:
         env=env,
     )
     pkgs = pd.DataFrame(json.loads(pkgs_json)[package_name])
-    pkgs["version"] = pkgs["version"].map(LooseVersion)
+    pkgs["version"] = pkgs["version"].map(Version)
     pkgs["py_version"] = pkgs["build"].str.slice(0, 4)
 
     to_delete = []

python/examples/plasma/sorting/setup.py

@@ -16,7 +16,7 @@
 # under the License.
 
 import numpy as np
-from distutils.core import setup
+from setuptools import setup
 from Cython.Build import cythonize
 
 setup(

python/pyarrow/feather.py

@@ -22,10 +22,11 @@
 from pyarrow.lib import (Codec, FeatherError, Table,  # noqa
                          concat_tables, schema)
 import pyarrow.lib as ext
+from pyarrow.vendored.version import Version
 
 
 def _check_pandas_version():
-    if _pandas_api.loose_version < '0.17.0':
+    if _pandas_api.loose_version < Version('0.17.0'):
         raise ImportError("feather requires pandas >= 0.17.0")
 
 

python/pyarrow/pandas-shim.pxi

@@ -53,12 +53,13 @@ cdef class _PandasAPIShim(object):
             else:
                 return
 
+        from pyarrow.vendored.version import Version
+
         self._pd = pd
         self._version = pd.__version__
-        from distutils.version import LooseVersion
-        self._loose_version = LooseVersion(pd.__version__)
+        self._loose_version = Version(pd.__version__)
 
-        if self._loose_version < LooseVersion('0.23.0'):
+        if self._loose_version < Version('0.23.0'):
             self._have_pandas = False
             if raise_:
                 raise ImportError(
@@ -82,7 +83,7 @@ cdef class _PandasAPIShim(object):
             self._series, self._index, self._categorical_type,
             self._extension_array)
         self._extension_dtype = pd.api.extensions.ExtensionDtype
-        if self._loose_version >= LooseVersion('0.24.0'):
+        if self._loose_version >= Version('0.24.0'):
             self._is_extension_array_dtype = \
                 pd.api.types.is_extension_array_dtype
         else:
@@ -92,12 +93,12 @@ cdef class _PandasAPIShim(object):
         self._datetimetz_type = pd.api.types.DatetimeTZDtype
         self._have_pandas = True
 
-        if self._loose_version > LooseVersion('0.25'):
+        if self._loose_version > Version('0.25'):
             self.has_sparse = False
         else:
             self.has_sparse = True
 
-        self._pd024 = self._loose_version >= LooseVersion('0.24')
+        self._pd024 = self._loose_version >= Version('0.24')
 
     cdef inline _check_import(self, bint raise_=True):
         if self._tried_importing_pandas:

python/pyarrow/tests/parquet/test_dataset.py

@@ -17,7 +17,6 @@
 
 import datetime
 import os
-from distutils.version import LooseVersion
 
 import numpy as np
 import pytest
@@ -30,6 +29,7 @@
     parametrize_legacy_dataset, parametrize_legacy_dataset_fixed,
     parametrize_legacy_dataset_not_supported)
 from pyarrow.util import guid
+from pyarrow.vendored.version import Version
 
 try:
     import pyarrow.parquet as pq
@@ -633,7 +633,7 @@ def test_read_partitioned_directory_s3fs_wrapper(
 
     from pyarrow.filesystem import S3FSWrapper
 
-    if s3fs.__version__ >= LooseVersion("0.5"):
+    if Version(s3fs.__version__) >= Version("0.5"):
         pytest.skip("S3FSWrapper no longer working for s3fs 0.5+")
 
     fs, path = s3_example_s3fs

python/pyarrow/tests/parquet/test_pandas.py

@@ -17,7 +17,6 @@
 
 import io
 import json
-from distutils.version import LooseVersion
 
 import numpy as np
 import pytest
@@ -26,6 +25,7 @@
 from pyarrow.tests.parquet.common import (
     parametrize_legacy_dataset, parametrize_legacy_dataset_not_supported)
 from pyarrow.util import guid
+from pyarrow.vendored.version import Version
 
 try:
     import pyarrow.parquet as pq
@@ -559,7 +559,7 @@ def test_write_to_dataset_pandas_preserve_extensiondtypes(
     tempdir, use_legacy_dataset
 ):
     # ARROW-8251 - preserve pandas extension dtypes in roundtrip
-    if LooseVersion(pd.__version__) < "1.0.0":
+    if Version(pd.__version__) < Version("1.0.0"):
         pytest.skip("__arrow_array__ added to pandas in 1.0.0")
 
     df = pd.DataFrame({'part': 'a', "col": [1, 2, 3]})

python/pyarrow/tests/test_fs.py

@@ -16,7 +16,6 @@
 # under the License.
 
 from datetime import datetime, timezone, timedelta
-from distutils.version import LooseVersion
 import gzip
 import os
 import pathlib
@@ -28,6 +27,8 @@
 
 import pyarrow as pa
 from pyarrow.tests.test_io import assert_file_not_found
+from pyarrow.vendored.version import Version
+
 from pyarrow.fs import (FileType, FileInfo, FileSelector, FileSystem,
                         LocalFileSystem, SubTreeFileSystem, _MockFileSystem,
                         FileSystemHandler, PyFileSystem, FSSpecHandler)
@@ -355,7 +356,8 @@ def py_fsspec_memoryfs(request, tempdir):
 @pytest.fixture
 def py_fsspec_s3fs(request, s3_connection, s3_server):
     s3fs = pytest.importorskip("s3fs")
-    if sys.version_info < (3, 7) and s3fs.__version__ >= LooseVersion("0.5"):
+    if (sys.version_info < (3, 7) and
+            Version(s3fs.__version__) >= Version("0.5")):
         pytest.skip("s3fs>=0.5 version is async and requires Python >= 3.7")
 
     host, port, access_key, secret_key = s3_connection

python/pyarrow/tests/test_pandas.py

@@ -23,7 +23,6 @@
 
 from collections import OrderedDict
 from datetime import date, datetime, time, timedelta, timezone
-from distutils.version import LooseVersion
 
 import hypothesis as h
 import hypothesis.extra.pytz as tzst
@@ -36,6 +35,7 @@
 from pyarrow.pandas_compat import get_logical_type, _pandas_api
 from pyarrow.tests.util import invoke_script, random_ascii, rands
 import pyarrow.tests.strategies as past
+from pyarrow.vendored.version import Version
 
 import pyarrow as pa
 try:
@@ -1042,7 +1042,7 @@ def test_python_datetime_with_pytz_timezone(self, tz):
     def test_python_datetime_with_timezone_tzinfo(self):
         from datetime import timezone
 
-        if LooseVersion(pd.__version__) > "0.25.0":
+        if Version(pd.__version__) > Version("0.25.0"):
             # older pandas versions fail on datetime.timezone.utc (as in input)
             # vs pytz.UTC (as in result)
             values = [datetime(2018, 1, 1, 12, 23, 45, tzinfo=timezone.utc)]
@@ -1467,8 +1467,9 @@ def test_array_from_pandas_date_with_mask(self):
         expected = pd.Series([None, date(1991, 1, 1), None])
         assert pa.Array.from_pandas(expected).equals(result)
 
-    @pytest.mark.skipif('1.16.0' <= LooseVersion(np.__version__) < '1.16.1',
-                        reason='Until numpy/numpy#12745 is resolved')
+    @pytest.mark.skipif(
+        Version('1.16.0') <= Version(np.__version__) < Version('1.16.1'),
+        reason='Until numpy/numpy#12745 is resolved')
     def test_fixed_offset_timezone(self):
         df = pd.DataFrame({
             'a': [
@@ -2827,7 +2828,7 @@ def _fully_loaded_dataframe_example():
         9: pd.period_range('2013', periods=10, freq='M')
     }
 
-    if LooseVersion(pd.__version__) >= '0.21':
+    if Version(pd.__version__) >= Version('0.21'):
         # There is an issue with pickling IntervalIndex in pandas 0.20.x
         data[10] = pd.interval_range(start=1, freq=1, periods=10)
 
@@ -2859,8 +2860,9 @@ def _check_serialize_components_roundtrip(pd_obj):
         tm.assert_series_equal(pd_obj, deserialized)
 
 
-@pytest.mark.skipif('1.16.0' <= LooseVersion(np.__version__) < '1.16.1',
-                    reason='Until numpy/numpy#12745 is resolved')
+@pytest.mark.skipif(
+    Version('1.16.0') <= Version(np.__version__) < Version('1.16.1'),
+    reason='Until numpy/numpy#12745 is resolved')
 def test_serialize_deserialize_pandas():
     # ARROW-1784, serialize and deserialize DataFrame by decomposing
     # BlockManager
@@ -2908,7 +2910,7 @@ class A:
         pa.Table.from_pandas(df)
 
     # period unsupported for pandas <= 0.25
-    if LooseVersion(pd.__version__) <= '0.25':
+    if Version(pd.__version__) <= Version('0.25'):
         df = pd.DataFrame({
             'a': pd.period_range('2000-01-01', periods=20),
         })
@@ -3817,12 +3819,12 @@ def test_dictionary_from_pandas_specified_type():
 
 
 def test_array_protocol():
-    if LooseVersion(pd.__version__) < '0.24.0':
+    if Version(pd.__version__) < Version('0.24.0'):
         pytest.skip('IntegerArray only introduced in 0.24')
 
     df = pd.DataFrame({'a': pd.Series([1, 2, None], dtype='Int64')})
 
-    if LooseVersion(pd.__version__) < '0.26.0.dev':
+    if Version(pd.__version__) < Version('0.26.0.dev'):
         # with pandas<=0.25, trying to convert nullable integer errors
         with pytest.raises(TypeError):
             pa.table(df)
@@ -3872,7 +3874,7 @@ def PandasArray__arrow_array__(self, type=None):
 def test_array_protocol_pandas_extension_types(monkeypatch):
     # ARROW-7022 - ensure protocol works for Period / Interval extension dtypes
 
-    if LooseVersion(pd.__version__) < '0.24.0':
+    if Version(pd.__version__) < Version('0.24.0'):
         pytest.skip('Period/IntervalArray only introduced in 0.24')
 
     storage = pa.array([1, 2, 3], type=pa.int64())
@@ -3921,7 +3923,7 @@ def _Int64Dtype__from_arrow__(self, array):
 
 
 def test_convert_to_extension_array(monkeypatch):
-    if LooseVersion(pd.__version__) < "0.26.0.dev":
+    if Version(pd.__version__) < Version("0.26.0.dev"):
         pytest.skip("Conversion from IntegerArray to arrow not yet supported")
 
     import pandas.core.internals as _int
@@ -3949,7 +3951,7 @@ def test_convert_to_extension_array(monkeypatch):
     tm.assert_frame_equal(result, df2)
 
     # monkeypatch pandas Int64Dtype to *not* have the protocol method
-    if LooseVersion(pd.__version__) < "1.3.0.dev":
+    if Version(pd.__version__) < Version("1.3.0.dev"):
         monkeypatch.delattr(
             pd.core.arrays.integer._IntegerDtype, "__from_arrow__")
     else:
@@ -3977,14 +3979,14 @@ def test_conversion_extensiontype_to_extensionarray(monkeypatch):
     # converting extension type to linked pandas ExtensionDtype/Array
     import pandas.core.internals as _int
 
-    if LooseVersion(pd.__version__) < "0.24.0":
+    if Version(pd.__version__) < Version("0.24.0"):
         pytest.skip("ExtensionDtype introduced in pandas 0.24")
 
     storage = pa.array([1, 2, 3, 4], pa.int64())
     arr = pa.ExtensionArray.from_storage(MyCustomIntegerType(), storage)
     table = pa.table({'a': arr})
 
-    if LooseVersion(pd.__version__) < "0.26.0.dev":
+    if Version(pd.__version__) < Version("0.26.0.dev"):
         # ensure pandas Int64Dtype has the protocol method (for older pandas)
         monkeypatch.setattr(
             pd.Int64Dtype, '__from_arrow__', _Int64Dtype__from_arrow__,
@@ -4004,9 +4006,9 @@ def test_conversion_extensiontype_to_extensionarray(monkeypatch):
 
     # monkeypatch pandas Int64Dtype to *not* have the protocol method
     # (remove the version added above and the actual version for recent pandas)
-    if LooseVersion(pd.__version__) < "0.26.0.dev":
+    if Version(pd.__version__) < Version("0.26.0.dev"):
         monkeypatch.delattr(pd.Int64Dtype, "__from_arrow__")
-    elif LooseVersion(pd.__version__) < "1.3.0.dev":
+    elif Version(pd.__version__) < Version("1.3.0.dev"):
         monkeypatch.delattr(
             pd.core.arrays.integer._IntegerDtype, "__from_arrow__")
     else:
@@ -4023,7 +4025,7 @@ def test_conversion_extensiontype_to_extensionarray(monkeypatch):
 
 
 def test_to_pandas_extension_dtypes_mapping():
-    if LooseVersion(pd.__version__) < "0.26.0.dev":
+    if Version(pd.__version__) < Version("0.26.0.dev"):
         pytest.skip("Conversion to pandas IntegerArray not yet supported")
 
     table = pa.table({'a': pa.array([1, 2, 3], pa.int64())})
@@ -4051,7 +4053,7 @@ def test_to_pandas_extension_dtypes_mapping():
 
 
 def test_array_to_pandas():
-    if LooseVersion(pd.__version__) < "1.1":
+    if Version(pd.__version__) < Version("1.1"):
         pytest.skip("ExtensionDtype to_pandas method missing")
 
     for arr in [pd.period_range("2012-01-01", periods=3, freq="D").array,

python/pyarrow/tests/test_schema.py

@@ -20,13 +20,12 @@
 import sys
 import weakref
 
-from distutils.version import LooseVersion
-
 import pytest
 import numpy as np
 import pyarrow as pa
 
 import pyarrow.tests.util as test_util
+from pyarrow.vendored.version import Version
 
 
 def test_schema_constructor_errors():
@@ -656,7 +655,7 @@ def test_schema_from_pandas():
             '2010-08-13T05:46:57.437699912'
         ], dtype='datetime64[ns]'),
     ]
-    if LooseVersion(pd.__version__) >= '1.0.0':
+    if Version(pd.__version__) >= Version('1.0.0'):
         inputs.append(pd.array([1, 2, None], dtype=pd.Int32Dtype()))
     for data in inputs:
         df = pd.DataFrame({'a': data})

python/pyarrow/vendored/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.