ENH: can ungzip data from file handle in parsers. close #2593

Author: wesm

pandas/io/parsers.py

@@ -1085,6 +1085,16 @@ def TextParser(*args, **kwds):
 def count_empty_vals(vals):
     return sum([1 for v in vals if v == '' or v is None])
 
+def _wrap_compressed(f, compression):
+    compression = compression.lower()
+    if compression == 'gzip':
+        import gzip
+        return gzip.GzipFile(fileobj=f)
+    elif compression == 'bz2':
+        raise ValueError('Python cannot read bz2 data from file handle')
+    else:
+        raise ValueError('do not recognize compression method %s'
+                         % compression)
 
 class PythonParser(ParserBase):
 
@@ -1130,6 +1140,8 @@ def __init__(self, f, **kwds):
         if isinstance(f, basestring):
             f = com._get_handle(f, 'r', encoding=self.encoding,
                                 compression=self.compression)
+        elif self.compression:
+            f = _wrap_compressed(f, self.compression)
 
         if hasattr(f, 'readline'):
             self._make_reader(f)

pandas/io/tests/test_parsers.py

@@ -1861,6 +1861,9 @@ def test_decompression(self):
 
             result = self.read_csv('__tmp__', compression='gzip')
             tm.assert_frame_equal(result, expected)
+
+            result = self.read_csv(open('__tmp__', 'rb'), compression='gzip')
+            tm.assert_frame_equal(result, expected)
         finally:
             # try:
             #     os.remove('__tmp__')
@@ -1876,6 +1879,9 @@ def test_decompression(self):
             result = self.read_csv('__tmp__', compression='bz2')
             tm.assert_frame_equal(result, expected)
 
+            # result = self.read_csv(open('__tmp__', 'rb'), compression='bz2')
+            # tm.assert_frame_equal(result, expected)
+
             self.assertRaises(ValueError, self.read_csv,
                               '__tmp__', compression='bz3')
         finally:

pandas/src/parser.pyx

@@ -469,13 +469,20 @@ cdef class TextReader:
         self.parser.cb_io = NULL
         self.parser.cb_cleanup = NULL
 
-        if isinstance(source, basestring) and self.compression:
+        if self.compression:
             if self.compression == 'gzip':
                 import gzip
-                source = gzip.GzipFile(source, 'rb')
+                if isinstance(source, basestring):
+                    source = gzip.GzipFile(source, 'rb')
+                else:
+                    source = gzip.GzipFile(fileobj=source)
             elif self.compression == 'bz2':
                 import bz2
-                source = bz2.BZ2File(source, 'rb')
+                if isinstance(source, basestring):
+                    source = bz2.BZ2File(source, 'rb')
+                else:
+                    raise ValueError('Python cannot read bz2 from open file '
+                                     'handle')
             else:
                 raise ValueError('Unrecognized compression type: %s' %
                                  self.compression)
@@ -1789,4 +1796,3 @@ def _maybe_encode(values):
     if values is None:
         return []
     return [x.encode('utf-8') if isinstance(x, unicode) else x for x in values]
-