Merge pull request #4976 from Komnomnomnom/json-locale-fix

FIX: JSON support for non C locales

Author: jreback

doc/source/release.rst

@@ -358,6 +358,8 @@ Bug Fixes
     dtypes, surfaced in (:issue:`4377`)
   - Fixed bug with duplicate columns and type conversion in ``read_json`` when
     ``orient='split'`` (:issue:`4377`)
+  - Fixed JSON bug where locales with decimal separators other than '.' threw
+    exceptions when encoding / decoding certain values. (:issue:`4918`)
   - Fix ``.iat`` indexing with a ``PeriodIndex`` (:issue:`4390`)
   - Fixed an issue where ``PeriodIndex`` joining with self was returning a new
     instance rather than the same instance (:issue:`4379`); also adds a test

pandas/io/tests/test_json/test_ujson.py

@@ -83,6 +83,19 @@ def test_doubleLongDecimalIssue(self):
         decoded = ujson.decode(encoded)
         self.assertEqual(sut, decoded)
 
+    def test_encodeNonCLocale(self):
+        import locale
+        savedlocale = locale.getlocale(locale.LC_NUMERIC)
+        try:
+            locale.setlocale(locale.LC_NUMERIC, 'it_IT.UTF-8')
+        except:
+            try:
+                locale.setlocale(locale.LC_NUMERIC, 'Italian_Italy')
+            except:
+                raise nose.SkipTest('Could not set locale for testing')
+        self.assertEqual(ujson.loads(ujson.dumps(4.78e60)), 4.78e60)
+        self.assertEqual(ujson.loads('4.78', precise_float=True), 4.78)
+        locale.setlocale(locale.LC_NUMERIC, savedlocale)
 
     def test_encodeDecodeLongDecimal(self):
         sut = {u('a'): -528656961.4399388}

pandas/src/ujson/lib/ultrajsondec.c

@@ -43,6 +43,7 @@ Numeric decoder derived from from TCL library
 #include <wchar.h>
 #include <stdlib.h>
 #include <errno.h>
+#include <locale.h>
 
 #ifndef TRUE
 #define TRUE 1
@@ -824,7 +825,7 @@ FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_object( struct DecoderState *ds)
 
       default:
         ds->dec->releaseObject(ds->prv, newObj, ds->dec);
-        return SetError(ds, -1, "Unexpected character in found when decoding object value");
+        return SetError(ds, -1, "Unexpected character found when decoding object value");
     }
   }
 }
@@ -874,6 +875,7 @@ JSOBJ JSON_DecodeObject(JSONObjectDecoder *dec, const char *buffer, size_t cbBuf
 {
   /*
   FIXME: Base the size of escBuffer of that of cbBuffer so that the unicode escaping doesn't run into the wall each time */
+  char *locale;
   struct DecoderState ds;
   wchar_t escBuffer[(JSON_MAX_STACK_BUFFER_SIZE / sizeof(wchar_t))];
   JSOBJ ret;
@@ -892,7 +894,15 @@ JSOBJ JSON_DecodeObject(JSONObjectDecoder *dec, const char *buffer, size_t cbBuf
 
   ds.dec = dec;
 
+  locale = strdup(setlocale(LC_NUMERIC, NULL));
+  if (!locale)
+  {
+    return SetError(&ds, -1, "Could not reserve memory block");
+  }
+  setlocale(LC_NUMERIC, "C");
   ret = decode_any (&ds);
+  setlocale(LC_NUMERIC, locale);
+  free(locale);
 
   if (ds.escHeap)
   {

pandas/src/ujson/lib/ultrajsonenc.c

@@ -41,6 +41,7 @@ Numeric decoder derived from from TCL library
 #include <string.h>
 #include <stdlib.h>
 #include <math.h>
+#include <locale.h>
 
 #include <float.h>
 
@@ -877,6 +878,7 @@ void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t cbName)
 
 char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *_buffer, size_t _cbBuffer)
 {
+  char *locale;
   enc->malloc = enc->malloc ? enc->malloc : malloc;
   enc->free =  enc->free ? enc->free : free;
   enc->realloc = enc->realloc ? enc->realloc : realloc;
@@ -915,7 +917,16 @@ char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *_buffer, size_t
   enc->end = enc->start + _cbBuffer;
   enc->offset = enc->start;
 
+  locale = strdup(setlocale(LC_NUMERIC, NULL));
+  if (!locale)
+  {
+    SetError(NULL, enc, "Could not reserve memory block");
+    return NULL;
+  }
+  setlocale(LC_NUMERIC, "C");
   encode (obj, enc, NULL, 0);
+  setlocale(LC_NUMERIC, locale);
+  free(locale);
 
   Buffer_Reserve(enc, 1);
   if (enc->errorMsg)