BUG: updated the copy method to work with supplied options

Author: jreback

RELEASE.rst

@@ -50,7 +50,7 @@ pandas 0.10.1
       to do multiple-table append/selection
     - added support for datetime64 in columns
     - added method ``unique`` to select the unique values in an indexable or data column
-    - added method ``copy_to`` to copy an existing store (and possibly upgrade)
+    - added method ``copy`` to copy an existing store (and possibly upgrade)
   - Add ``logx`` option to DataFrame/Series.plot (GH2327_, #2565)
   - Support reading gzipped data from file-like object
   - ``pivot_table`` aggfunc can be anything used in GroupBy.aggregate (GH2643_)

doc/source/io.rst

@@ -1343,7 +1343,7 @@ Or on-the-fly compression (this only applies to tables). You can turn off file c
 
    - ``ptrepack --chunkshape=auto --propindexes --complevel=9 --complib=blosc in.h5 out.h5``
 
-Furthermore ``ptrepack in.h5 out.h5`` will *repack* the file to allow you to reuse previously deleted space (alternatively, one can simply remove the file and write again).
+Furthermore ``ptrepack in.h5 out.h5`` will *repack* the file to allow you to reuse previously deleted space. Aalternatively, one can simply remove the file and write again, or use the ``copy`` method.
 
 Notes & Caveats
 ~~~~~~~~~~~~~~~
@@ -1367,9 +1367,7 @@ Notes & Caveats
 Compatibility
 ~~~~~~~~~~~~~
 
-0.10.1 of ``HDFStore`` is backwards compatible for reading tables created in a prior version of pandas
-however, query terms using the prior (undocumented) methodology are unsupported. ``HDFStore`` will issue a warning if you try to use a prior-version format file. You must read in the entire
-file and write it out using the new format to take advantage of the updates. The group attribute ``pandas_version`` contains the version information.
+0.10.1 of ``HDFStore`` is backwards compatible for reading tables created in a prior version of pandas however, query terms using the prior (undocumented) methodology are unsupported. ``HDFStore`` will issue a warning if you try to use a prior-version format file. You must read in the entire file and write it out using the new format, using the method ``copy`` to take advantage of the updates. The group attribute ``pandas_version`` contains the version information. ``copy`` takes a number of options, please see the docstring.
 
 
      .. ipython:: python
@@ -1380,7 +1378,7 @@ file and write it out using the new format to take advantage of the updates. The
         legacy_store
 
         # copy (and return the new handle)
-	new_store = legacy_store.copy_to('store_new.h5')
+	new_store = legacy_store.copy('store_new.h5')
 	new_store
         new_store.close()
 

pandas/io/pytables.py

@@ -630,15 +630,45 @@ def get_table(self, key):
         t.infer_axes()
         return t
 
-    def copy_to(self, file):
-        """ copy the existing store to a new file, upgrading in place """
-        new_store = HDFStore(file, mode = 'w')
-        for k, g in self.iteritems():
-            data = self.select(k)
-            if _is_table_type(g):
-                new_store.append(k,data)
-            else:
-                new_store.put(k,data)
+    def is_table(self, key):
+        """ return a boolean if I am a table """
+        group = self.get_node(key)
+        if group is None:
+            raise KeyError('No object named %s in the file' % key)
+        return _is_table_type(group)
+
+    def copy(self, file, mode = 'w', propindexes = True, keys = None, complib = None, complevel = None, fletcher32 = False):
+        """ copy the existing store to a new file, upgrading in place
+
+            Parameters
+            ----------
+            propindexes: restore indexes in copied file (defaults to True)
+            keys       : list of keys to include in the copy (defaults to all)
+            mode, complib, complevel, fletcher32 same as in HDFStore.__init__
+
+            Returns
+            -------
+            open file handle of the new store
+
+        """
+        new_store = HDFStore(file, mode = mode, complib = complib, complevel = complevel, fletcher32 = fletcher32)
+        if keys is None:
+            keys = self.keys()
+        if not isinstance(keys, (tuple,list)):
+            keys = [ keys ]
+        for k in keys:
+            n    = self.get_node(k)
+            if n is not None:
+                data = self.select(k)
+                if _is_table_type(n):
+
+                    t = self.get_table(k)
+                    index = False
+                    if propindexes:
+                        index = [ a.name for a in t.axes if a.is_indexed ]
+                    new_store.append(k,data, index=index, data_columns=getattr(t,'data_columns',None))
+                else:
+                    new_store.put(k,data)
         return new_store
 
     ###### private methods ######
@@ -1131,6 +1161,14 @@ def __eq__(self, other):
     def __ne__(self, other):
         return not self.__eq__(other)
 
+    @property
+    def is_indexed(self):
+        """ return whether I am an indexed column """
+        try:
+            return getattr(self.table.cols,self.cname).is_indexed
+        except:
+            False
+
     def copy(self):
         new_self = copy.copy(self)
         return new_self
@@ -1543,6 +1581,13 @@ def __repr__(self):
 
     __str__ = __repr__
 
+    def __getitem__(self, c):
+        """ return the axis for c """
+        for a in self.axes:
+            if c == a.name:
+                return a
+        return None
+
     def copy(self):
         new_self = copy.copy(self)
         return new_self

pandas/io/tests/test_pytables.py

@@ -1735,25 +1735,59 @@ def test_legacy_0_10_read(self):
             store.select(k)
         store.close()
 
-    def test_legacy_copy_to(self):
+    def test_copy(self):
         pth = curpath()
+        def do_copy(f = None, keys = None, propindexes = True, **kwargs):
+            try:
+                import os
+
+                if f is None:
+                    f = os.path.join(pth, 'legacy_0.10.h5')
+
+                store = HDFStore(f, 'r')
+                import tempfile
+                tmp = tempfile.mkstemp()[1]
+                tstore = store.copy(tmp, keys = keys, propindexes = propindexes, **kwargs)
+                
+                # check keys
+                if keys is None:
+                    keys = store.keys()
+                self.assert_(set(keys) == set(tstore.keys()))
+
+                # check indicies & nrows
+                for k in tstore.keys():
+                    if tstore.is_table(k):
+                        new_t = tstore.get_table(k)
+                        orig_t = store.get_table(k)
+
+                        self.assert_(orig_t.nrows == new_t.nrows)
+                        for a in orig_t.axes:
+                            if a.is_indexed:
+                                self.assert_(new_t[a.name].is_indexed == True)
+
+            except:
+                pass
+            finally:
+                store.close()
+                tstore.close()
+                import os
+                os.remove(tmp)
+
+        do_copy()
+        do_copy(keys = ['df'])
+        do_copy(propindexes = False)
+
+        # new table
+        df = tm.makeDataFrame()
         try:
-            import os
-            store = HDFStore(os.path.join(pth, 'legacy_0.10.h5'), 'r')
-            import tempfile
-            tmp = tempfile.mkstemp()[1]
-            tstore = store.copy_to(tmp)
-            
-            # the tmp store
-            for k in tstore.keys():
-                self.assert_(k in store)
-        except:
-            pass
+            st = HDFStore(self.scratchpath)
+            st.append('df', df, data_columns = ['A'])
+            st.close()
+            do_copy(f = self.scratchpath)
+            do_copy(f = self.scratchpath, propindexes = False)
         finally:
-            store.close()
-            tstore.close()
             import os
-            os.remove(tmp)
+            os.remove(self.scratchpath)
 
     def test_legacy_table_write(self):
         raise nose.SkipTest