API: to_json now writes to a file by default (if None is provided it will return a StringIO object)

     read_json will read from a string-like or filebuf or url (consistent with other parsers)

Author: jreback

doc/source/io.rst

@@ -939,7 +939,6 @@ The Series object also has a ``to_string`` method, but with only the ``buf``,
 which, if set to ``True``, will additionally output the length of the Series.
 
 
-
 JSON
 ----
 
@@ -953,6 +952,8 @@ Writing JSON
 A ``Series`` or ``DataFrame`` can be converted to a valid JSON string. Use ``to_json``
 with optional parameters:
 
+- path_or_buf : the pathname or buffer to write the output
+  This can be ``None`` in which case a ``StringIO`` converted string is returned
 - orient : The format of the JSON string, default is ``index`` for ``Series``, ``columns`` for ``DataFrame``
 
   * split   : dict like {index -> [index], columns -> [columns], data -> [values]}
@@ -969,8 +970,8 @@ Note NaN's and None will be converted to null and datetime objects will be conve
 .. ipython:: python
 
    df = DataFrame(randn(10, 2), columns=list('AB'))
-   s = df.to_json()
-   s
+   json = df.to_json(None)
+   json.getvalue()
 
 Reading JSON
 ~~~~~~~~~~~~
@@ -979,7 +980,11 @@ Reading a JSON string to pandas object can take a number of parameters.
 The parser will try to parse a ``DataFrame`` if ``typ`` is not supplied or
 is ``None``. To explicity force ``Series`` parsing, pass ``typ=series``
 
-- json   : The JSON string to parse.
+- filepath_or_buffer : a **VALID** JSON string or file handle / StringIO. The string could be
+  a URL. Valid URL schemes include http, ftp, s3, and file. For file URLs, a host
+  is expected. For instance, a local file could be
+  file ://localhost/path/to/table.json
+- json : a VALID JSON string, optional, used if filepath_or_buffer is not provided
 - typ    : type of object to recover (series or frame), default 'frame'
 - orient : The format of the JSON string, one of the following
 
@@ -993,9 +998,17 @@ is ``None``. To explicity force ``Series`` parsing, pass ``typ=series``
 The parser will raise one of ``ValueError/TypeError/AssertionError`` if the JSON is
 not parsable.
 
+Reading from a JSON string
+
+.. ipython:: python
+
+   pd.read_json(json='{"0":{"0":1,"1":3},"1":{"0":2,"1":4}}')
+
+Reading from a StringIO
+
 .. ipython:: python
 
-   pd.read_json(s)
+   pd.read_json(json)
 
 HTML
 ----

pandas/core/generic.py

@@ -495,7 +495,7 @@ def to_clipboard(self):
         from pandas.io import clipboard
         clipboard.to_clipboard(self)
 
-    def to_json(self, orient=None, double_precision=10,
+    def to_json(self, path_or_buf, orient=None, double_precision=10,
                 force_ascii=True):
         """
         Convert the object to a JSON string.
@@ -505,6 +505,8 @@ def to_json(self, orient=None, double_precision=10,
 
         Parameters
         ----------
+        path_or_buf : the path or buffer to write the result string
+            if this is None, return a StringIO of the converted string
         orient : {'split', 'records', 'index', 'columns', 'values'},
             default is 'index' for Series, 'columns' for DataFrame
 
@@ -521,10 +523,13 @@ def to_json(self, orient=None, double_precision=10,
 
         Returns
         -------
-        result : JSON compatible string
+        result : a JSON compatible string written to the path_or_buf;
+                 if the path_or_buf is none, return a StringIO of the result
+
         """
+
         from pandas.io import json
-        return json.to_json(self, orient=orient, double_precision=double_precision,
+        return json.to_json(path_or_buf, self, orient=orient, double_precision=double_precision,
                             force_ascii=force_ascii)
 
 # install the indexerse

pandas/io/common.py

@@ -2,6 +2,7 @@
 
 import urlparse
 from pandas.util import py3compat
+from StringIO import StringIO
 
 _VALID_URLS = set(urlparse.uses_relative + urlparse.uses_netloc +
                   urlparse.uses_params)

pandas/io/excel.py

@@ -11,7 +11,7 @@
 
 from pandas.io.parsers import TextParser
 from pandas.tseries.period import Period
-import json
+from pandas import json
 
 def read_excel(path_or_buf, sheetname, kind=None, **kwds):
     """Read an Excel table into a pandas DataFrame

pandas/io/json.py

@@ -1,23 +1,27 @@
 
 # pylint: disable-msg=E1101,W0613,W0603
 from pandas import Series, DataFrame
+from pandas.io.common import get_filepath_or_buffer
+from StringIO import StringIO
 
 import pandas.json as _json
 loads = _json.loads
 dumps = _json.dumps
 
 ### interface to/from ###
 
-def to_json(obj, orient=None, double_precision=10,
+def to_json(path_or_buf, obj, orient=None, double_precision=10,
             force_ascii=True):
         """
-        Convert the object to a JSON string.
+        Convert the object to a JSON string
 
         Note NaN's and None will be converted to null and datetime objects
         will be converted to UNIX timestamps.
 
         Parameters
         ----------
+        path_or_buf : the pathname or buffer to write the output
+            if this is None, return a StringIO of the converted string
         orient : {'split', 'records', 'index', 'columns', 'values'},
             default is 'index' for Series, 'columns' for DataFrame
 
@@ -34,7 +38,9 @@ def to_json(obj, orient=None, double_precision=10,
 
         Returns
         -------
-        result : JSON compatible string
+        result : a JSON compatible string written to the path_or_buf;
+                 if the path_or_buf is none, return a StringIO of the result
+
         """
 
         if orient is None:
@@ -43,16 +49,27 @@ def to_json(obj, orient=None, double_precision=10,
             elif isinstance(obj, DataFrame):
                 orient = 'columns'
 
-        return dumps(obj, orient=orient, double_precision=double_precision,
-                     ensure_ascii=force_ascii)
+        s = dumps(obj, orient=orient, double_precision=double_precision,
+                          ensure_ascii=force_ascii)
+        if isinstance(path_or_buf, basestring):
+            with open(path_or_buf,'w') as fh:
+                fh.write(s)
+        elif path_or_buf is None:
+            return StringIO(s)
+        else:
+            path_or_buf.write(s)
 
-def read_json(json, typ='frame', orient=None, dtype=None, numpy=True):
+def read_json(filepath_or_buffer=None, json=None, typ='frame', orient=None, dtype=None, numpy=True):
     """
     Convert JSON string to pandas object
 
     Parameters
     ----------
-    json : The JSON string to parse.
+    filepath_or_buffer : a VALID JSON StringIO or file handle / StringIO. The string could be
+        a URL. Valid URL schemes include http, ftp, s3, and file. For file URLs, a host
+        is expected. For instance, a local file could be
+        file ://localhost/path/to/table.json
+    json : a VALID JSON string, optional, used if filepath_or_buffer is not provided
     typ : type of object to recover (series or frame), default 'frame'
     orient : {'split', 'records', 'index'}, default 'index'
         The format of the JSON string
@@ -69,6 +86,16 @@ def read_json(json, typ='frame', orient=None, dtype=None, numpy=True):
     result : Series or DataFrame
     """
 
+    if json is None:
+        filepath_or_buffer,_ = get_filepath_or_buffer(filepath_or_buffer)
+        if isinstance(filepath_or_buffer, basestring):
+                with open(filepath_or_buffer,'r') as fh:
+                        json = fh.read()
+        elif hasattr(filepath_or_buffer, 'read'):
+                json = filepath_or_buffer.read()
+        else:
+                json = filepath_or_buffer
+
     obj = None
     if typ == 'frame':
         if orient is None:

pandas/io/parsers.py

@@ -23,7 +23,6 @@
 import pandas.tslib as tslib
 import pandas.parser as _parser
 from pandas.tseries.period import Period
-import json
 
 
 class DateConversionError(Exception):

pandas/io/tests/test_json/test_pandas.py

@@ -15,8 +15,9 @@
 read_json = pd.read_json
 
 from pandas.util.testing import (assert_almost_equal, assert_frame_equal,
-                                 assert_series_equal)
+                                 assert_series_equal, network)
 import pandas.util.testing as tm
+from numpy.testing.decorators import slow
 
 _seriesd = tm.getSeriesData()
 _tsd = tm.getTimeSeriesData()
@@ -56,7 +57,7 @@ def test_frame_from_json_to_json(self):
 
         def _check_orient(df, orient, dtype=None, numpy=True):
             df = df.sort()
-            dfjson = df.to_json(orient=orient)
+            dfjson = df.to_json(None, orient=orient)
             unser = read_json(dfjson, orient=orient, dtype=dtype,
                               numpy=numpy)
             unser = unser.sort()
@@ -93,8 +94,8 @@ def _check_all_orients(df, dtype=None):
 
         # basic
         _check_all_orients(self.frame)
-        self.assertEqual(self.frame.to_json(),
-                         self.frame.to_json(orient="columns"))
+        self.assertEqual(self.frame.to_json(None).read(),
+                         self.frame.to_json(None,orient="columns").read())
 
         _check_all_orients(self.intframe, dtype=self.intframe.values.dtype)
 
@@ -138,61 +139,61 @@ def _check_all_orients(df, dtype=None):
         _check_orient(df.transpose().transpose(), "index")
 
     def test_frame_from_json_bad_data(self):
-        self.assertRaises(ValueError, read_json, '{"key":b:a:d}')
+        self.assertRaises(ValueError, read_json, StringIO('{"key":b:a:d}'))
 
         # too few indices
-        json = ('{"columns":["A","B"],'
-                '"index":["2","3"],'
-                '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}"')
+        json = StringIO('{"columns":["A","B"],'
+                        '"index":["2","3"],'
+                        '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}"')
         self.assertRaises(ValueError, read_json, json,
                           orient="split")
 
         # too many columns
-        json = ('{"columns":["A","B","C"],'
-                '"index":["1","2","3"],'
-                '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}"')
+        json = StringIO('{"columns":["A","B","C"],'
+                        '"index":["1","2","3"],'
+                        '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}"')
         self.assertRaises(AssertionError, read_json, json,
                           orient="split")
 
         # bad key
-        json = ('{"badkey":["A","B"],'
-                '"index":["2","3"],'
-                '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}"')
+        json = StringIO('{"badkey":["A","B"],'
+                        '"index":["2","3"],'
+                        '"data":[[1.0,"1"],[2.0,"2"],[null,"3"]]}"')
         self.assertRaises(TypeError, read_json, json,
                           orient="split")
 
     def test_frame_from_json_nones(self):
         df = DataFrame([[1, 2], [4, 5, 6]])
-        unser = read_json(df.to_json())
+        unser = read_json(df.to_json(None))
         self.assert_(np.isnan(unser['2'][0]))
 
         df = DataFrame([['1', '2'], ['4', '5', '6']])
-        unser = read_json(df.to_json())
+        unser = read_json(df.to_json(None))
         self.assert_(unser['2'][0] is None)
 
-        unser = read_json(df.to_json(), numpy=False)
+        unser = read_json(df.to_json(None), numpy=False)
         self.assert_(unser['2'][0] is None)
 
         # infinities get mapped to nulls which get mapped to NaNs during
         # deserialisation
         df = DataFrame([[1, 2], [4, 5, 6]])
         df[2][0] = np.inf
-        unser = read_json(df.to_json())
+        unser = read_json(df.to_json(None))
         self.assert_(np.isnan(unser['2'][0]))
 
         df[2][0] = np.NINF
-        unser = read_json(df.to_json())
+        unser = read_json(df.to_json(None))
         self.assert_(np.isnan(unser['2'][0]))
 
     def test_frame_to_json_except(self):
         df = DataFrame([1, 2, 3])
-        self.assertRaises(ValueError, df.to_json, orient="garbage")
+        self.assertRaises(ValueError, df.to_json, None, orient="garbage")
 
     def test_series_from_json_to_json(self):
 
         def _check_orient(series, orient, dtype=None, numpy=True):
             series = series.sort_index()
-            unser = read_json(series.to_json(orient=orient), typ='series',
+            unser = read_json(series.to_json(None,orient=orient), typ='series',
                               orient=orient, numpy=numpy, dtype=dtype)
             unser = unser.sort_index()
             if series.index.dtype.type == np.datetime64:
@@ -222,8 +223,8 @@ def _check_all_orients(series, dtype=None):
 
         # basic
         _check_all_orients(self.series)
-        self.assertEqual(self.series.to_json(),
-                         self.series.to_json(orient="index"))
+        self.assertEqual(self.series.to_json(None).read(),
+                         self.series.to_json(None,orient="index").read())
 
         objSeries = Series([str(d) for d in self.objSeries],
                            index=self.objSeries.index,
@@ -239,18 +240,35 @@ def _check_all_orients(series, dtype=None):
 
     def test_series_to_json_except(self):
         s = Series([1, 2, 3])
-        self.assertRaises(ValueError, s.to_json, orient="garbage")
+        self.assertRaises(ValueError, s.to_json, None, orient="garbage")
 
     def test_typ(self):
 
         s = Series(range(6), index=['a','b','c','d','e','f'])
-        result = read_json(s.to_json(),typ=None)
+        result = read_json(s.to_json(None),typ=None)
         assert_series_equal(result,s)
 
     def test_reconstruction_index(self):
 
         df = DataFrame([[1, 2, 3], [4, 5, 6]])
-        result = read_json(df.to_json())
+        result = read_json(df.to_json(None))
 
         # the index is serialized as strings....correct?
         #assert_frame_equal(result,df)
+
+    @network
+    @slow
+    def test_url(self):
+        import urllib2
+        try:
+            # HTTP(S)
+            url = 'https://api.github.com/repos/pydata/pandas/issues?per_page=5'
+            result = read_json(url)
+            #print result
+            
+            url = 'http://search.twitter.com/search.json?q=pandas%20python'
+            result = read_json(url)
+            #print result
+            
+        except urllib2.URLError:
+            raise nose.SkipTest