Merge tag 'v0.19.1' into debian

Version 0.19.1

* tag 'v0.19.1': (43 commits)
  RLS: v0.19.1
  DOC: update whatsnew/release notes for 0.19.1 (pandas-dev#14573)
  [Backport pandas-dev#14545] BUG/API: Index.append with mixed object/Categorical indices (pandas-dev#14545)
  DOC: rst fixes
  [Backport pandas-dev#14567] DEPR: add deprecation warning for com.array_equivalent (pandas-dev#14567)
  [Backport pandas-dev#14551] PERF: casting loc to labels dtype before searchsorted (pandas-dev#14551)
  [Backport pandas-dev#14536] BUG: DataFrame.quantile with NaNs (GH14357) (pandas-dev#14536)
  [Backport pandas-dev#14520] BUG: don't close user-provided file handles in C parser (GH14418) (pandas-dev#14520)
  [Backport pandas-dev#14392] BUG: Dataframe constructor when given dict with None value (pandas-dev#14392)
  [Backport pandas-dev#14514] BUG: Don't parse inline quotes in skipped lines (pandas-dev#14514)
  [Bacport pandas-dev#14543] BUG: tseries ceil doc fix (pandas-dev#14543)
  [Backport pandas-dev#14541] DOC: Simplify the gbq integration testing procedure for contributors (pandas-dev#14541)
  [Backport pandas-dev#14527] BUG/ERR: raise correct error when sql driver is not installed (pandas-dev#14527)
  [Backport pandas-dev#14501] BUG: fix DatetimeIndex._maybe_cast_slice_bound for empty index (GH14354) (pandas-dev#14501)
  [Backport pandas-dev#14442] DOC: Expand on reference docs for read_json() (pandas-dev#14442)
  BLD: fix 3.4 build for cython to 0.24.1
  [Backport pandas-dev#14492] BUG: Accept unicode quotechars again in pd.read_csv
  [Backport pandas-dev#14496] BLD: Support Cython 0.25
  [Backport pandas-dev#14498] COMPAT/TST: fix test for range testing of negative integers to neg powers
  [Backport pandas-dev#14476] PERF: performance regression in Series.asof (pandas-dev#14476)
  ...

Author: yarikoptic

asv_bench/benchmarks/algorithms.py

@@ -15,6 +15,14 @@ def setup(self):
         self.int = pd.Int64Index(np.arange(N).repeat(5))
         self.float = pd.Float64Index(np.random.randn(N).repeat(5))
 
+        # Convenience naming.
+        self.checked_add = pd.core.nanops._checked_add_with_arr
+
+        self.arr = np.arange(1000000)
+        self.arrpos = np.arange(1000000)
+        self.arrneg = np.arange(-1000000, 0)
+        self.arrmixed = np.array([1, -1]).repeat(500000)
+
     def time_int_factorize(self):
         self.int.factorize()
 
@@ -29,3 +37,21 @@ def time_int_duplicated(self):
 
     def time_float_duplicated(self):
         self.float.duplicated()
+
+    def time_add_overflow_pos_scalar(self):
+        self.checked_add(self.arr, 1)
+
+    def time_add_overflow_neg_scalar(self):
+        self.checked_add(self.arr, -1)
+
+    def time_add_overflow_zero_scalar(self):
+        self.checked_add(self.arr, 0)
+
+    def time_add_overflow_pos_arr(self):
+        self.checked_add(self.arr, self.arrpos)
+
+    def time_add_overflow_neg_arr(self):
+        self.checked_add(self.arr, self.arrneg)
+
+    def time_add_overflow_mixed_arr(self):
+        self.checked_add(self.arr, self.arrmixed)

asv_bench/benchmarks/attrs_caching.py

@@ -20,4 +20,4 @@ def setup(self):
         self.cur_index = self.df.index
 
     def time_setattr_dataframe_index(self):
-        self.df.index = self.cur_index
+        self.df.index = self.cur_index

asv_bench/benchmarks/ctors.py

@@ -49,4 +49,4 @@ def setup(self):
         self.s = Series(([Timestamp('20110101'), Timestamp('20120101'), Timestamp('20130101')] * 1000))
 
     def time_index_from_series_ctor(self):
-        Index(self.s)
+        Index(self.s)

asv_bench/benchmarks/frame_ctor.py

@@ -1703,4 +1703,4 @@ def setup(self):
         self.dict_list = [dict(zip(self.columns, row)) for row in self.frame.values]
 
     def time_series_ctor_from_dict(self):
-        Series(self.some_dict)
+        Series(self.some_dict)

asv_bench/benchmarks/groupby.py

@@ -548,6 +548,32 @@ def time_groupby_sum(self):
         self.df.groupby(['a'])['b'].sum()
 
 
+class groupby_period(object):
+    # GH 14338
+    goal_time = 0.2
+
+    def make_grouper(self, N):
+        return pd.period_range('1900-01-01', freq='D', periods=N)
+
+    def setup(self):
+        N = 10000
+        self.grouper = self.make_grouper(N)
+        self.df = pd.DataFrame(np.random.randn(N, 2))
+
+    def time_groupby_sum(self):
+        self.df.groupby(self.grouper).sum()
+
+
+class groupby_datetime(groupby_period):
+    def make_grouper(self, N):
+        return pd.date_range('1900-01-01', freq='D', periods=N)
+
+
+class groupby_datetimetz(groupby_period):
+    def make_grouper(self, N):
+        return pd.date_range('1900-01-01', freq='D', periods=N,
+                             tz='US/Central')
+
 #----------------------------------------------------------------------
 # Series.value_counts
 

asv_bench/benchmarks/hdfstore_bench.py

@@ -348,4 +348,4 @@ def remove(self, f):
         try:
             os.remove(self.f)
         except:
-            pass
+            pass

asv_bench/benchmarks/index_object.py

@@ -344,4 +344,4 @@ def setup(self):
         self.mi = MultiIndex.from_product([self.level1, self.level2])
 
     def time_multiindex_with_datetime_level_sliced(self):
-        self.mi[:10].values
+        self.mi[:10].values

asv_bench/benchmarks/io_sql.py

@@ -212,4 +212,4 @@ def setup(self):
         self.df = DataFrame({'float1': randn(10000), 'float2': randn(10000), 'string1': (['foo'] * 10000), 'bool1': ([True] * 10000), 'int1': np.random.randint(0, 100000, size=10000), }, index=self.index)
 
     def time_sql_write_sqlalchemy(self):
-        self.df.to_sql('test1', self.engine, if_exists='replace')
+        self.df.to_sql('test1', self.engine, if_exists='replace')

asv_bench/benchmarks/packers.py

@@ -547,6 +547,31 @@ def remove(self, f):
             pass
 
 
+class packers_write_json_lines(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.f = '__test__.msg'
+        self.N = 100000
+        self.C = 5
+        self.index = date_range('20000101', periods=self.N, freq='H')
+        self.df = DataFrame(dict([('float{0}'.format(i), randn(self.N)) for i in range(self.C)]), index=self.index)
+        self.remove(self.f)
+        self.df.index = np.arange(self.N)
+
+    def time_packers_write_json_lines(self):
+        self.df.to_json(self.f, orient="records", lines=True)
+
+    def teardown(self):
+        self.remove(self.f)
+
+    def remove(self, f):
+        try:
+            os.remove(self.f)
+        except:
+            pass
+
+
 class packers_write_json_T(object):
     goal_time = 0.2
 

asv_bench/benchmarks/panel_ctor.py

@@ -61,4 +61,4 @@ def setup(self):
             self.data_frames[x] = self.df
 
     def time_panel_from_dict_two_different_indexes(self):
-        Panel.from_dict(self.data_frames)
+        Panel.from_dict(self.data_frames)

asv_bench/benchmarks/panel_methods.py

@@ -53,4 +53,4 @@ def setup(self):
         self.panel = Panel(np.random.randn(100, len(self.index), 1000))
 
     def time_panel_shift_minor(self):
-        self.panel.shift(1, axis='minor')
+        self.panel.shift(1, axis='minor')

asv_bench/benchmarks/replace.py

@@ -45,4 +45,4 @@ def setup(self):
         self.ts = Series(np.random.randn(self.N), index=self.rng)
 
     def time_replace_replacena(self):
-        self.ts.replace(np.nan, 0.0, inplace=True)
+        self.ts.replace(np.nan, 0.0, inplace=True)

asv_bench/benchmarks/reshape.py

@@ -73,4 +73,4 @@ def setup(self):
                 break
 
     def time_unstack_sparse_keyspace(self):
-        self.idf.unstack()
+        self.idf.unstack()

asv_bench/benchmarks/stat_ops.py

@@ -258,4 +258,4 @@ def time_rolling_skew(self):
         rolling_skew(self.arr, self.win)
 
     def time_rolling_kurt(self):
-        rolling_kurt(self.arr, self.win)
+        rolling_kurt(self.arr, self.win)

asv_bench/benchmarks/strings.py

@@ -390,4 +390,4 @@ def time_strings_upper(self):
         self.many.str.upper()
 
     def make_series(self, letters, strlen, size):
-        return Series([str(x) for x in np.fromiter(IT.cycle(letters), count=(size * strlen), dtype='|S1').view('|S{}'.format(strlen))])
+        return Series([str(x) for x in np.fromiter(IT.cycle(letters), count=(size * strlen), dtype='|S1').view('|S{}'.format(strlen))])

asv_bench/benchmarks/timedelta.py

@@ -1,5 +1,5 @@
 from .pandas_vb_common import *
-from pandas import to_timedelta
+from pandas import to_timedelta, Timestamp
 
 
 class timedelta_convert_int(object):
@@ -47,3 +47,14 @@ def time_timedelta_convert_coerce(self):
 
     def time_timedelta_convert_ignore(self):
         to_timedelta(self.arr, errors='ignore')
+
+
+class timedelta_add_overflow(object):
+    goal_time = 0.2
+
+    def setup(self):
+        self.td = to_timedelta(np.arange(1000000))
+        self.ts = Timestamp('2000')
+
+    def test_add_td_ts(self):
+        self.td + self.ts

asv_bench/benchmarks/timeseries.py

@@ -284,56 +284,77 @@ class timeseries_asof(object):
     goal_time = 0.2
 
     def setup(self):
-        self.N = 100000
-        self.rng = date_range(start='1/1/2000', periods=self.N, freq='T')
-        if hasattr(Series, 'convert'):
-            Series.resample = Series.convert
-        self.ts = Series(np.random.randn(self.N), index=self.rng)
         self.N = 10000
         self.rng = date_range(start='1/1/1990', periods=self.N, freq='53s')
-        self.ts = Series(np.random.randn(self.N), index=self.rng)
         self.dates = date_range(start='1/1/1990', periods=(self.N * 10), freq='5s')
+        self.ts = Series(np.random.randn(self.N), index=self.rng)
+        self.ts2 = self.ts.copy()
+        self.ts2[250:5000] = np.nan
+        self.ts3 = self.ts.copy()
+        self.ts3[-5000:] = np.nan
 
-    def time_timeseries_asof(self):
+    # test speed of pre-computing NAs.
+    def time_asof_list(self):
         self.ts.asof(self.dates)
 
+    # should be roughly the same as above.
+    def time_asof_nan_list(self):
+        self.ts2.asof(self.dates)
 
-class timeseries_asof_nan(object):
-    goal_time = 0.2
+    # test speed of the code path for a scalar index
+    # without *while* loop
+    def time_asof_single(self):
+        self.ts.asof(self.dates[0])
 
-    def setup(self):
-        self.N = 100000
-        self.rng = date_range(start='1/1/2000', periods=self.N, freq='T')
-        if hasattr(Series, 'convert'):
-            Series.resample = Series.convert
-        self.ts = Series(np.random.randn(self.N), index=self.rng)
-        self.N = 10000
-        self.rng = date_range(start='1/1/1990', periods=self.N, freq='53s')
-        self.ts = Series(np.random.randn(self.N), index=self.rng)
-        self.dates = date_range(start='1/1/1990', periods=(self.N * 10), freq='5s')
-        self.ts[250:5000] = np.nan
+    # test speed of the code path for a scalar index
+    # before the start. should be the same as above.
+    def time_asof_single_early(self):
+        self.ts.asof(self.dates[0] - dt.timedelta(10))
 
-    def time_timeseries_asof_nan(self):
-        self.ts.asof(self.dates)
+    # test the speed of the code path for a scalar index
+    # with a long *while* loop. should still be much
+    # faster than pre-computing all the NAs.
+    def time_asof_nan_single(self):
+        self.ts3.asof(self.dates[-1])
 
 
-class timeseries_asof_single(object):
+class timeseries_dataframe_asof(object):
     goal_time = 0.2
 
     def setup(self):
-        self.N = 100000
-        self.rng = date_range(start='1/1/2000', periods=self.N, freq='T')
-        if hasattr(Series, 'convert'):
-            Series.resample = Series.convert
-        self.ts = Series(np.random.randn(self.N), index=self.rng)
         self.N = 10000
+        self.M = 100
         self.rng = date_range(start='1/1/1990', periods=self.N, freq='53s')
-        self.ts = Series(np.random.randn(self.N), index=self.rng)
         self.dates = date_range(start='1/1/1990', periods=(self.N * 10), freq='5s')
+        self.ts = DataFrame(np.random.randn(self.N, self.M), index=self.rng)
+        self.ts2 = self.ts.copy()
+        self.ts2.iloc[250:5000] = np.nan
+        self.ts3 = self.ts.copy()
+        self.ts3.iloc[-5000:] = np.nan
+
+    # test speed of pre-computing NAs.
+    def time_asof_list(self):
+        self.ts.asof(self.dates)
 
-    def time_timeseries_asof_single(self):
+    # should be roughly the same as above.
+    def time_asof_nan_list(self):
+        self.ts2.asof(self.dates)
+
+    # test speed of the code path for a scalar index
+    # with pre-computing all NAs.
+    def time_asof_single(self):
         self.ts.asof(self.dates[0])
 
+    # should be roughly the same as above.
+    def time_asof_nan_single(self):
+        self.ts3.asof(self.dates[-1])
+
+    # test speed of the code path for a scalar index
+    # before the start. should be without the cost of
+    # pre-computing all the NAs.
+    def time_asof_single_early(self):
+        self.ts.asof(self.dates[0] - dt.timedelta(10))
+
 
 class timeseries_custom_bday_apply(object):
     goal_time = 0.2

ci/requirements-3.4.build

@@ -1,3 +1,3 @@
 numpy=1.8.1
-cython
+cython=0.24.1
 libgfortran=1.0

ci/travis_encrypt_gbq.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
 
 GBQ_JSON_FILE=$1
-GBQ_PROJECT_ID=$2
 
-if [[ $# -ne 2 ]]; then
+if [[ $# -ne 1 ]]; then
     echo -e "Too few arguments.\nUsage: ./travis_encrypt_gbq.sh "\
-    "<gbq-json-credentials-file> <gbq-project-id>"
+    "<gbq-json-credentials-file>"
     exit 1
 fi
 
@@ -23,9 +22,9 @@ echo "Encrypting $GBQ_JSON_FILE..."
 read -d "\n" TRAVIS_KEY TRAVIS_IV <<<$(travis encrypt-file $GBQ_JSON_FILE \
 travis_gbq.json.enc -f | grep -o "\w*_iv\|\w*_key");
 
-echo "Adding your secure key and project id to travis_gbq_config.txt ..."
-echo -e "TRAVIS_IV_ENV=$TRAVIS_IV\nTRAVIS_KEY_ENV=$TRAVIS_KEY\n"\
-"GBQ_PROJECT_ID='$GBQ_PROJECT_ID'" > travis_gbq_config.txt
+echo "Adding your secure key to travis_gbq_config.txt ..."
+echo -e "TRAVIS_IV_ENV=$TRAVIS_IV\nTRAVIS_KEY_ENV=$TRAVIS_KEY"\
+> travis_gbq_config.txt
 
 echo "Done. Removing file $GBQ_JSON_FILE"
 rm $GBQ_JSON_FILE

ci/travis_gbq_config.txt

@@ -1,3 +1,2 @@
 TRAVIS_IV_ENV=encrypted_1d9d7b1f171b_iv
 TRAVIS_KEY_ENV=encrypted_1d9d7b1f171b_key
-GBQ_PROJECT_ID='pandas-travis'

ci/travis_process_gbq_encryption.sh

@@ -2,10 +2,12 @@
 
 source ci/travis_gbq_config.txt
 
-if [[ -n ${!TRAVIS_IV_ENV} ]]; then
+if [[ -n ${SERVICE_ACCOUNT_KEY} ]]; then
+    echo "${SERVICE_ACCOUNT_KEY}" > ci/travis_gbq.json;
+elif [[ -n ${!TRAVIS_IV_ENV} ]]; then
     openssl aes-256-cbc -K ${!TRAVIS_KEY_ENV} -iv ${!TRAVIS_IV_ENV} \
     -in ci/travis_gbq.json.enc -out ci/travis_gbq.json -d;
-    export GBQ_PROJECT_ID=$GBQ_PROJECT_ID;
+    export GBQ_PROJECT_ID='pandas-travis';
     echo 'Successfully decrypted gbq credentials'
 fi
 

doc/README.rst

@@ -155,9 +155,9 @@ Where to start?
 ---------------
 
 There are a number of issues listed under `Docs
-<https://github.com/pydata/pandas/issues?labels=Docs&sort=updated&state=open>`_
+<https://github.com/pandas-dev/pandas/issues?labels=Docs&sort=updated&state=open>`_
 and `Good as first PR
-<https://github.com/pydata/pandas/issues?labels=Good+as+first+PR&sort=updated&state=open>`_
+<https://github.com/pandas-dev/pandas/issues?labels=Good+as+first+PR&sort=updated&state=open>`_
 where you could start out.
 
 Or maybe you have an idea of your own, by using pandas, looking for something

doc/_templates/autosummary/accessor_attribute.rst

@@ -3,4 +3,4 @@
 
 .. currentmodule:: {{ module.split('.')[0] }}
 
-.. autoaccessorattribute:: {{ [module.split('.')[1], objname]|join('.') }}
+.. autoaccessorattribute:: {{ [module.split('.')[1], objname]|join('.') }}

doc/_templates/autosummary/accessor_method.rst

@@ -3,4 +3,4 @@
 
 .. currentmodule:: {{ module.split('.')[0] }}
 
-.. autoaccessormethod:: {{ [module.split('.')[1], objname]|join('.') }}
+.. autoaccessormethod:: {{ [module.split('.')[1], objname]|join('.') }}