Merge remote-tracking branch 'pandas-dev/master' into bigquery-udf-resources

Author: necnec

.travis.yml

@@ -66,19 +66,6 @@ matrix:
         apt:
           packages:
           - python-gtk2
-    - python: 3.4
-      env:
-        - PYTHON_VERSION=3.4
-        - JOB_NAME: "34_nslow"
-        - NOSE_ARGS="not slow and not disabled"
-        - FULL_DEPS=true
-        - CLIPBOARD=xsel
-        - CACHE_NAME="34_nslow"
-        - USE_CACHE=true
-      addons:
-        apt:
-          packages:
-          - xsel
     - python: 3.5
       env:
         - PYTHON_VERSION=3.5
@@ -93,6 +80,32 @@ matrix:
         apt:
           packages:
           - xsel
+    - python: 3.6
+      env:
+        - PYTHON_VERSION=3.6
+        - JOB_NAME: "36"
+        - NOSE_ARGS="not slow and not network and not disabled"
+        - PANDAS_TESTING_MODE="deprecate"
+      addons:
+        apt:
+          packages:
+          - libatlas-base-dev
+          - gfortran
+#    In allow_failures
+    - python: 2.7
+      env:
+        - PYTHON_VERSION=2.7
+        - JOB_NAME: "27_nslow_nnet_COMPAT"
+        - NOSE_ARGS="not slow and not network and not disabled"
+        - LOCALE_OVERRIDE="it_IT.UTF-8"
+        - INSTALL_TEST=true
+        - JOB_TAG=_COMPAT
+        - CACHE_NAME="27_nslow_nnet_COMPAT"
+        - USE_CACHE=true
+      addons:
+        apt:
+          packages:
+          - language-pack-it
 #    In allow_failures
     - python: 2.7
       env:
@@ -103,45 +116,46 @@ matrix:
         - FULL_DEPS=true
         - CACHE_NAME="27_slow"
         - USE_CACHE=true
+#    In allow_failures
+    - python: 2.7
+      env:
+        - PYTHON_VERSION=2.7
+        - JOB_NAME: "27_build_test_conda"
+        - JOB_TAG=_BUILD_TEST
+        - NOSE_ARGS="not slow and not disabled"
+        - FULL_DEPS=true
+        - BUILD_TEST=true
+        - CACHE_NAME="27_build_test_conda"
+        - USE_CACHE=true
 #    In allow_failures
     - python: 3.4
       env:
         - PYTHON_VERSION=3.4
-        - JOB_NAME: "34_slow"
-        - JOB_TAG=_SLOW
-        - NOSE_ARGS="slow and not network and not disabled"
+        - JOB_NAME: "34_nslow"
+        - NOSE_ARGS="not slow and not disabled"
         - FULL_DEPS=true
         - CLIPBOARD=xsel
-        - CACHE_NAME="34_slow"
+        - CACHE_NAME="34_nslow"
         - USE_CACHE=true
       addons:
         apt:
           packages:
           - xsel
 #    In allow_failures
-    - python: 2.7
+    - python: 3.4
       env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_build_test_conda"
-        - JOB_TAG=_BUILD_TEST
-        - NOSE_ARGS="not slow and not disabled"
+        - PYTHON_VERSION=3.4
+        - JOB_NAME: "34_slow"
+        - JOB_TAG=_SLOW
+        - NOSE_ARGS="slow and not network and not disabled"
         - FULL_DEPS=true
-        - BUILD_TEST=true
-        - CACHE_NAME="27_build_test_conda"
+        - CLIPBOARD=xsel
+        - CACHE_NAME="34_slow"
         - USE_CACHE=true
-#    In allow_failures
-    - python: 3.6-dev
-      env:
-        - PYTHON_VERSION=3.6
-        - JOB_NAME: "36_dev"
-        - JOB_TAG=_DEV
-        - NOSE_ARGS="not slow and not network and not disabled"
-        - PANDAS_TESTING_MODE="deprecate"
       addons:
         apt:
           packages:
-          - libatlas-base-dev
-          - gfortran
+          - xsel
 #    In allow_failures
     - python: 3.5
       env:
@@ -157,21 +171,6 @@ matrix:
           packages:
           - libatlas-base-dev
           - gfortran
-#    In allow_failures
-    - python: 2.7
-      env:
-        - PYTHON_VERSION=2.7
-        - JOB_NAME: "27_nslow_nnet_COMPAT"
-        - NOSE_ARGS="not slow and not network and not disabled"
-        - LOCALE_OVERRIDE="it_IT.UTF-8"
-        - INSTALL_TEST=true
-        - JOB_TAG=_COMPAT
-        - CACHE_NAME="27_nslow_nnet_COMPAT"
-        - USE_CACHE=true
-      addons:
-        apt:
-          packages:
-          - language-pack-it
 #    In allow_failures
     - python: 3.5
       env:
@@ -226,18 +225,19 @@ matrix:
         - BUILD_TEST=true
         - CACHE_NAME="27_build_test_conda"
         - USE_CACHE=true
-      - python: 3.6-dev
+      - python: 3.4
         env:
-        - PYTHON_VERSION=3.6
-        - JOB_NAME: "36_dev"
-        - JOB_TAG=_DEV
-        - NOSE_ARGS="not slow and not network and not disabled"
-        - PANDAS_TESTING_MODE="deprecate"
+          - PYTHON_VERSION=3.4
+          - JOB_NAME: "34_nslow"
+          - NOSE_ARGS="not slow and not disabled"
+          - FULL_DEPS=true
+          - CLIPBOARD=xsel
+          - CACHE_NAME="34_nslow"
+          - USE_CACHE=true
         addons:
           apt:
             packages:
-            - libatlas-base-dev
-            - gfortran
+            - xsel
       - python: 3.5
         env:
           - PYTHON_VERSION=3.5

appveyor.yml

@@ -80,6 +80,7 @@ install:
   - cmd: conda config --set ssl_verify false
 
   # add the pandas channel *before* defaults to have defaults take priority
+  - cmd: conda config --add channels conda-forge
   - cmd: conda config --add channels pandas
   - cmd: conda config --remove channels defaults
   - cmd: conda config --add channels defaults

asv_bench/benchmarks/algorithms.py

@@ -8,6 +8,7 @@ class Algorithms(object):
 
     def setup(self):
         N = 100000
+        np.random.seed(1234)
 
         self.int_unique = pd.Int64Index(np.arange(N * 5))
         # cache is_unique
@@ -17,17 +18,24 @@ def setup(self):
         self.float = pd.Float64Index(np.random.randn(N).repeat(5))
 
         # Convenience naming.
-        self.checked_add = pd.core.nanops._checked_add_with_arr
+        self.checked_add = pd.core.algorithms.checked_add_with_arr
 
         self.arr = np.arange(1000000)
         self.arrpos = np.arange(1000000)
         self.arrneg = np.arange(-1000000, 0)
         self.arrmixed = np.array([1, -1]).repeat(500000)
+        self.strings = tm.makeStringIndex(100000)
+
+        self.arr_nan = np.random.choice([True, False], size=1000000)
+        self.arrmixed_nan = np.random.choice([True, False], size=1000000)
 
         # match
         self.uniques = tm.makeStringIndex(1000).values
         self.all = self.uniques.repeat(10)
 
+    def time_factorize_string(self):
+        self.strings.factorize()
+
     def time_factorize_int(self):
         self.int.factorize()
 
@@ -64,6 +72,16 @@ def time_add_overflow_neg_arr(self):
     def time_add_overflow_mixed_arr(self):
         self.checked_add(self.arr, self.arrmixed)
 
+    def time_add_overflow_first_arg_nan(self):
+        self.checked_add(self.arr, self.arrmixed, arr_mask=self.arr_nan)
+
+    def time_add_overflow_second_arg_nan(self):
+        self.checked_add(self.arr, self.arrmixed, b_mask=self.arrmixed_nan)
+
+    def time_add_overflow_both_arg_nan(self):
+        self.checked_add(self.arr, self.arrmixed, arr_mask=self.arr_nan,
+                         b_mask=self.arrmixed_nan)
+
 
 class Hashing(object):
     goal_time = 0.2

asv_bench/benchmarks/frame_methods.py

@@ -68,6 +68,8 @@ class Iteration(object):
     def setup(self):
         self.df = DataFrame(randn(10000, 1000))
         self.df2 = DataFrame(np.random.randn(50000, 10))
+        self.df3 = pd.DataFrame(np.random.randn(1000,5000),
+                                columns=['C'+str(c) for c in range(5000)])
 
     def f(self):
         if hasattr(self.df, '_item_cache'):
@@ -85,6 +87,11 @@ def time_iteritems(self):
     def time_iteritems_cached(self):
         self.g()
 
+    def time_iteritems_indexing(self):
+        df = self.df3
+        for col in df:
+            df[col]
+
     def time_itertuples(self):
         for row in self.df2.itertuples():
             pass

asv_bench/benchmarks/gil.py

@@ -379,3 +379,38 @@ def pg_read_csv_datetime(self):
 
     def time_read_csv_datetime(self):
         self.pg_read_csv_datetime()
+
+
+class nogil_factorize(object):
+    number = 1
+    repeat = 5
+
+    def setup(self):
+        if (not have_real_test_parallel):
+            raise NotImplementedError
+
+        np.random.seed(1234)
+        self.strings = tm.makeStringIndex(100000)
+
+    def factorize_strings(self):
+        pd.factorize(self.strings)
+
+    @test_parallel(num_threads=4)
+    def _pg_factorize_strings_4(self):
+        self.factorize_strings()
+
+    def time_factorize_strings_4(self):
+        for i in range(2):
+            self._pg_factorize_strings_4()
+
+    @test_parallel(num_threads=2)
+    def _pg_factorize_strings_2(self):
+        self.factorize_strings()
+
+    def time_factorize_strings_2(self):
+        for i in range(4):
+            self._pg_factorize_strings_2()
+
+    def time_factorize_strings(self):
+        for i in range(8):
+            self.factorize_strings()

asv_bench/benchmarks/io_bench.py

@@ -153,7 +153,7 @@ def setup(self, compression, engine):
             # The Python 2 C parser can't read bz2 from open files.
             raise NotImplementedError
         try:
-            import boto
+            import s3fs
         except ImportError:
             # Skip these benchmarks if `boto` is not installed.
             raise NotImplementedError

asv_bench/benchmarks/join_merge.py

@@ -302,12 +302,19 @@ def setup(self):
         self.df1 = self.df1.sort_values('time')
         self.df2 = self.df2.sort_values('time')
 
+        self.df1['time32'] = np.int32(self.df1.time)
+        self.df2['time32'] = np.int32(self.df2.time)
+
         self.df1a = self.df1[['time', 'value1']]
         self.df2a = self.df2[['time', 'value2']]
         self.df1b = self.df1[['time', 'key', 'value1']]
         self.df2b = self.df2[['time', 'key', 'value2']]
         self.df1c = self.df1[['time', 'key2', 'value1']]
         self.df2c = self.df2[['time', 'key2', 'value2']]
+        self.df1d = self.df1[['time32', 'value1']]
+        self.df2d = self.df2[['time32', 'value2']]
+        self.df1e = self.df1[['time', 'key', 'key2', 'value1']]
+        self.df2e = self.df2[['time', 'key', 'key2', 'value2']]
 
     def time_noby(self):
         merge_asof(self.df1a, self.df2a, on='time')
@@ -318,6 +325,12 @@ def time_by_object(self):
     def time_by_int(self):
         merge_asof(self.df1c, self.df2c, on='time', by='key2')
 
+    def time_on_int32(self):
+        merge_asof(self.df1d, self.df2d, on='time32')
+
+    def time_multiby(self):
+        merge_asof(self.df1e, self.df2e, on='time', by=['key', 'key2'])
+
 
 #----------------------------------------------------------------------
 # data alignment

asv_bench/benchmarks/period.py

@@ -49,3 +49,28 @@ def time_value_counts_pindex(self):
         self.i.value_counts()
 
 
+class period_standard_indexing(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.index = PeriodIndex(start='1985', periods=1000, freq='D')
+        self.series = Series(range(1000), index=self.index)
+        self.period = self.index[500]
+
+    def time_get_loc(self):
+        self.index.get_loc(self.period)
+
+    def time_shape(self):
+        self.index.shape
+
+    def time_shallow_copy(self):
+        self.index._shallow_copy()
+
+    def time_series_loc(self):
+        self.series.loc[self.period]
+
+    def time_align(self):
+        pd.DataFrame({'a': self.series, 'b': self.series[:500]})
+
+    def time_intersection(self):
+        self.index[:750].intersection(self.index[250:])

asv_bench/benchmarks/reshape.py

@@ -1,5 +1,5 @@
 from .pandas_vb_common import *
-from pandas.core.reshape import melt
+from pandas.core.reshape import melt, wide_to_long
 
 
 class melt_dataframe(object):
@@ -74,3 +74,25 @@ def setup(self):
 
     def time_unstack_sparse_keyspace(self):
         self.idf.unstack()
+
+
+class wide_to_long_big(object):
+    goal_time = 0.2
+
+    def setup(self):
+        vars = 'ABCD'
+        nyrs = 20
+        nidvars = 20
+        N = 5000
+        yrvars = []
+        for var in vars:
+            for yr in range(1, nyrs + 1):
+                yrvars.append(var + str(yr))
+
+        self.df = pd.DataFrame(np.random.randn(N, nidvars + len(yrvars)),
+                               columns=list(range(nidvars)) + yrvars)
+        self.vars = vars
+
+    def time_wide_to_long_big(self):
+        self.df['id'] = self.df.index
+        wide_to_long(self.df, list(self.vars), i='id', j='year')