From df22a1e43da2f0bd221dab126b500fd621e32bf0 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Mon, 10 Dec 2012 16:43:16 -0500 Subject: [PATCH 1/6] by default, only sqlite tests are run --- pandas/io/sql.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 0caf83838fb2d..6552280788bf0 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -305,3 +305,83 @@ def sequence2dict(seq): for k,v in zip(range(1, 1 + len(seq)), seq): d[str(k)] = v return d +<<<<<<< HEAD +======= + +def test_sqlite(name, testdf): + print '\nsqlite, using detect_types=sqlite3.PARSE_DECLTYPES for datetimes' + import sqlite3 + with sqlite3.connect('test.db', detect_types=sqlite3.PARSE_DECLTYPES) as conn: + #conn.row_factory = sqlite3.Row + write_frame(testdf, name, con=conn, flavor='sqlite', if_exists='replace') + df_sqlite = read_db('select * from '+name, con=conn) + print 'loaded dataframe from sqlite', len(df_sqlite) + print 'done with sqlite' + + +def test_oracle(name, testdf): + print '\nOracle' + import cx_Oracle + with cx_Oracle.connect('YOURCONNECTION') as ora_conn: + testdf['d64'] = np.datetime64( testdf['hire_date'] ) + write_frame(testdf, name, con=ora_conn, flavor='oracle', if_exists='replace') + df_ora2 = read_db('select * from '+name, con=ora_conn) + + print 'done with oracle' + return df_ora2 + + +def test_postgresql(name, testdf): + #from pg8000 import DBAPI as pg + import psycopg2 as pg + print '\nPostgresQL, Greenplum' + pgcn = pg.connect(YOURCONNECTION) + print 'df frame_query' + try: + write_frame(testdf, name, con=pgcn, flavor='postgresql', if_exists='replace') + print 'pg copy_from' + postgresql_copy_from(testdf, name, con=pgcn) + df_gp = read_db('select * from '+name, con=pgcn) + print 'loaded dataframe from greenplum', len(df_gp) + finally: + pgcn.commit() + pgcn.close() + print 'done with greenplum' + + +def test_mysql(name, testdf): + import MySQLdb + print '\nmysql' + cn= MySQLdb.connect(YOURCONNECTION) + try: + write_frame(testdf, name='test_df', con=cn, flavor='mysql', if_exists='replace') + df_mysql = read_db('select * from '+name, con=cn) + print 'loaded dataframe from mysql', len(df_mysql) + finally: + cn.close() + print 'mysql done' + + +############################################################################## + +if __name__=='__main__': + + print """sqlite should work out of the box. For the other test, you must + install the driver and provide a connection string.""" + + test_data = { + "name": [ 'Joe', 'Bob', 'Jim', 'Suzy', 'Cathy', 'Sarah' ], + "hire_date": [ datetime(2012,1,1), datetime(2012,2,1), datetime(2012,3,1), datetime(2012,4,1), datetime(2012,5,1), datetime(2012,6,1) ], + "erank": [ 1, 2, 3, 4, 5, 6 ], + "score": [ 1.1, 2.2, 3.1, 2.5, 3.6, 1.8] + } + df = DataFrame(test_data) + + name='test_df' + test_sqlite(name, df) + #test_oracle(name, df) + #test_postgresql(name, df) + #test_mysql(name, df) + + print 'done' +>>>>>>> by default, only sqlite tests are run From 81ae141c008f022386efd81a66eb577d3eb7b6de Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Mon, 10 Dec 2012 17:02:29 -0500 Subject: [PATCH 2/6] remove print statement used for debugging --- pandas/io/sql.py | 80 ------------------------------------------------ 1 file changed, 80 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 6552280788bf0..0caf83838fb2d 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -305,83 +305,3 @@ def sequence2dict(seq): for k,v in zip(range(1, 1 + len(seq)), seq): d[str(k)] = v return d -<<<<<<< HEAD -======= - -def test_sqlite(name, testdf): - print '\nsqlite, using detect_types=sqlite3.PARSE_DECLTYPES for datetimes' - import sqlite3 - with sqlite3.connect('test.db', detect_types=sqlite3.PARSE_DECLTYPES) as conn: - #conn.row_factory = sqlite3.Row - write_frame(testdf, name, con=conn, flavor='sqlite', if_exists='replace') - df_sqlite = read_db('select * from '+name, con=conn) - print 'loaded dataframe from sqlite', len(df_sqlite) - print 'done with sqlite' - - -def test_oracle(name, testdf): - print '\nOracle' - import cx_Oracle - with cx_Oracle.connect('YOURCONNECTION') as ora_conn: - testdf['d64'] = np.datetime64( testdf['hire_date'] ) - write_frame(testdf, name, con=ora_conn, flavor='oracle', if_exists='replace') - df_ora2 = read_db('select * from '+name, con=ora_conn) - - print 'done with oracle' - return df_ora2 - - -def test_postgresql(name, testdf): - #from pg8000 import DBAPI as pg - import psycopg2 as pg - print '\nPostgresQL, Greenplum' - pgcn = pg.connect(YOURCONNECTION) - print 'df frame_query' - try: - write_frame(testdf, name, con=pgcn, flavor='postgresql', if_exists='replace') - print 'pg copy_from' - postgresql_copy_from(testdf, name, con=pgcn) - df_gp = read_db('select * from '+name, con=pgcn) - print 'loaded dataframe from greenplum', len(df_gp) - finally: - pgcn.commit() - pgcn.close() - print 'done with greenplum' - - -def test_mysql(name, testdf): - import MySQLdb - print '\nmysql' - cn= MySQLdb.connect(YOURCONNECTION) - try: - write_frame(testdf, name='test_df', con=cn, flavor='mysql', if_exists='replace') - df_mysql = read_db('select * from '+name, con=cn) - print 'loaded dataframe from mysql', len(df_mysql) - finally: - cn.close() - print 'mysql done' - - -############################################################################## - -if __name__=='__main__': - - print """sqlite should work out of the box. For the other test, you must - install the driver and provide a connection string.""" - - test_data = { - "name": [ 'Joe', 'Bob', 'Jim', 'Suzy', 'Cathy', 'Sarah' ], - "hire_date": [ datetime(2012,1,1), datetime(2012,2,1), datetime(2012,3,1), datetime(2012,4,1), datetime(2012,5,1), datetime(2012,6,1) ], - "erank": [ 1, 2, 3, 4, 5, 6 ], - "score": [ 1.1, 2.2, 3.1, 2.5, 3.6, 1.8] - } - df = DataFrame(test_data) - - name='test_df' - test_sqlite(name, df) - #test_oracle(name, df) - #test_postgresql(name, df) - #test_mysql(name, df) - - print 'done' ->>>>>>> by default, only sqlite tests are run From cb0d8c8d31a4e601815f4f098e49b1fd28dcf16d Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Tue, 18 Dec 2012 17:28:40 -0500 Subject: [PATCH 3/6] added mysql nosetests and made them optional --- pandas/io/sql.py | 7 +++-- pandas/io/tests/test_sql.py | 57 +++++++++++++++++++++++++++---------- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 0caf83838fb2d..ef9609d4a5c29 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -241,9 +241,10 @@ def table_exists(name, con, flavor): def get_sqltype(pytype, flavor): sqltype = {'mysql': 'VARCHAR (63)', - 'sqlite': 'TEXT'} - - if issubclass(pytype, np.floating): + 'oracle': 'VARCHAR2', + 'sqlite': 'TEXT', + 'postgresql': 'TEXT'} + if issubclass(pytype, np.number): sqltype['mysql'] = 'FLOAT' sqltype['sqlite'] = 'REAL' diff --git a/pandas/io/tests/test_sql.py b/pandas/io/tests/test_sql.py index 3216a7dbbaa71..94ee06e5c4e77 100644 --- a/pandas/io/tests/test_sql.py +++ b/pandas/io/tests/test_sql.py @@ -1,6 +1,7 @@ from pandas.util.py3compat import StringIO import unittest import sqlite3 +import MySQLdb import sys import nose @@ -27,6 +28,40 @@ bool: lambda x: "'%s'" % x, } +def format_query(sql, *args): + """ + + """ + processed_args = [] + for arg in args: + if isinstance(arg, float) and isnull(arg): + arg = None + + formatter = _formatters[type(arg)] + processed_args.append(formatter(arg)) + + return sql % tuple(processed_args) + +def _skip_if_no_MySQLdb(): + try: + import MySQLdb + except ImportError: + raise nose.SkipTest('MySQLdb not installed, skipping') + +from datetime import datetime + +_formatters = { + datetime: lambda dt: "'%s'" % date_format(dt), + str: lambda x: "'%s'" % x, + np.str_: lambda x: "'%s'" % x, + unicode: lambda x: "'%s'" % x, + float: lambda x: "%.8f" % x, + int: lambda x: "%s" % x, + type(None): lambda x: "NULL", + np.float64: lambda x: "%.10f" % x, + bool: lambda x: "'%s'" % x, +} + def format_query(sql, *args): """ @@ -218,19 +253,15 @@ def test_keyword_as_column_names(self): df = DataFrame({'From':np.ones(5)}) sql.write_frame(df, con = self.db, name = 'testkeywords') - class TestMySQL(unittest.TestCase): - + _multiprocess_can_split_ = True def setUp(self): - try: - import MySQLdb - except ImportError: - raise nose.SkipTest + _skip_if_no_MySQLdb() try: self.db = MySQLdb.connect(read_default_group='pandas') except MySQLdb.Error, e: raise nose.SkipTest( - "Cannot connect to database. " + "Cannot connect to database. " "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " "typically located at ~/.my.cnf or /etc/.my.cnf. ") @@ -239,6 +270,7 @@ def setUp(self): "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " "typically located at ~/.my.cnf or /etc/.my.cnf. ") + def test_basic(self): _skip_if_no_MySQLdb() @@ -261,6 +293,7 @@ def test_write_row_by_row(self): self.db.commit() +>>>>>>> added mysql nosetests and made them optional result = sql.read_frame("select * from test", con=self.db) result.index = frame.index tm.assert_frame_equal(result, frame) @@ -386,10 +419,7 @@ def _check_roundtrip(self, frame): tm.assert_frame_equal(expected, result) def test_tquery(self): - try: - import MySQLdb - except ImportError: - raise nose.SkipTest + _skip_if_no_MySQLdb() frame = tm.makeTimeDataFrame() drop_sql = "DROP TABLE IF EXISTS test_table" cur = self.db.cursor() @@ -411,10 +441,7 @@ def test_tquery(self): sys.stdout = sys.__stdout__ def test_uquery(self): - try: - import MySQLdb - except ImportError: - raise nose.SkipTest + _skip_if_no_MySQLdb() frame = tm.makeTimeDataFrame() drop_sql = "DROP TABLE IF EXISTS test_table" cur = self.db.cursor() From 3bc40ec85ba373d6cee251ce6fec40fff0bceb78 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Tue, 18 Dec 2012 17:38:59 -0500 Subject: [PATCH 4/6] added test keyword_as_column_names --- pandas/io/sql.py | 3 +++ pandas/io/tests/test_sql.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index ef9609d4a5c29..73fa76f662437 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -277,7 +277,10 @@ def get_schema(frame, name, flavor, keys=None): columns = ',\n '.join('[%s] %s' % x for x in column_types) else: columns = ',\n '.join('`%s` %s' % x for x in column_types) +<<<<<<< HEAD +======= +>>>>>>> added test keyword_as_column_names keystr = '' if keys is not None: if isinstance(keys, basestring): diff --git a/pandas/io/tests/test_sql.py b/pandas/io/tests/test_sql.py index 94ee06e5c4e77..7aabe0c0403c9 100644 --- a/pandas/io/tests/test_sql.py +++ b/pandas/io/tests/test_sql.py @@ -467,7 +467,7 @@ def test_keyword_as_column_names(self): ''' _skip_if_no_MySQLdb() df = DataFrame({'From':np.ones(5)}) - sql.write_frame(df, con = self.db, name = 'testkeywords', + sql.write_frame(df, con = self.db, name = 'testkeywords', if_exists='replace', flavor='mysql') From c52ef4393476c889d9bd4a9f13ce2c4e443113f9 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Tue, 18 Dec 2012 17:43:50 -0500 Subject: [PATCH 5/6] imported MySQL inside TestMySQLdb only, not globally --- pandas/io/tests/test_sql.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pandas/io/tests/test_sql.py b/pandas/io/tests/test_sql.py index 7aabe0c0403c9..ecaeb4948a607 100644 --- a/pandas/io/tests/test_sql.py +++ b/pandas/io/tests/test_sql.py @@ -1,7 +1,6 @@ from pandas.util.py3compat import StringIO import unittest import sqlite3 -import MySQLdb import sys import nose @@ -256,7 +255,10 @@ def test_keyword_as_column_names(self): class TestMySQL(unittest.TestCase): _multiprocess_can_split_ = True def setUp(self): - _skip_if_no_MySQLdb() + try: + import MySQLdb + except ImportError: + raise nose.SkipTest try: self.db = MySQLdb.connect(read_default_group='pandas') except MySQLdb.Error, e: @@ -270,7 +272,6 @@ def setUp(self): "Create a group of connection parameters under the heading " "[pandas] in your system's mysql default file, " "typically located at ~/.my.cnf or /etc/.my.cnf. ") - def test_basic(self): _skip_if_no_MySQLdb() @@ -419,7 +420,10 @@ def _check_roundtrip(self, frame): tm.assert_frame_equal(expected, result) def test_tquery(self): - _skip_if_no_MySQLdb() + try: + import MySQLdb + except ImportError: + raise nose.SkipTest frame = tm.makeTimeDataFrame() drop_sql = "DROP TABLE IF EXISTS test_table" cur = self.db.cursor() @@ -441,7 +445,10 @@ def test_tquery(self): sys.stdout = sys.__stdout__ def test_uquery(self): - _skip_if_no_MySQLdb() + try: + import MySQLdb + except ImportError: + raise nose.SkipTest frame = tm.makeTimeDataFrame() drop_sql = "DROP TABLE IF EXISTS test_table" cur = self.db.cursor() From 87f78cf743d1d83b29955a160e8027df3a1139e8 Mon Sep 17 00:00:00 2001 From: Dan Allan Date: Wed, 19 Dec 2012 07:05:29 -0500 Subject: [PATCH 6/6] Restored append keyword with a FutureWarning --- pandas/io/sql.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 73fa76f662437..0c1e5ee1f7ca3 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -173,10 +173,10 @@ def write_frame(frame, name, con, flavor='sqlite', if_exists='fail', **kwargs): replace: If table exists, drop it, recreate it, and insert data. append: If table exists, insert data. Create if does not exist. """ - + if 'append' in kwargs: import warnings - warnings.warn("append is deprecated, use if_exists instead", + warnings.warn("append is deprecated, use if_exists instead", FutureWarning) if kwargs['append']: if_exists='append' @@ -195,7 +195,12 @@ def write_frame(frame, name, con, flavor='sqlite', if_exists='fail', **kwargs): if create is not None: cur = con.cursor() +<<<<<<< HEAD cur.execute(create) +======= + create_table = get_schema(frame, name, flavor) + cur.execute(create_table) +>>>>>>> Restored append keyword with a FutureWarning cur.close() cur = con.cursor() @@ -247,12 +252,22 @@ def get_sqltype(pytype, flavor): if issubclass(pytype, np.number): sqltype['mysql'] = 'FLOAT' sqltype['sqlite'] = 'REAL' +<<<<<<< HEAD if issubclass(pytype, np.integer): #TODO: Refine integer size. sqltype['mysql'] = 'BIGINT' sqltype['sqlite'] = 'INTEGER' +======= + sqltype['postgresql'] = 'NUMBER' + if issubclass(pytype, np.integer): + #TODO: Refine integer size. + sqltype['mysql'] = 'BIGINT' + sqltype['oracle'] = 'PLS_INTEGER' + sqltype['sqlite'] = 'INTEGER' + sqltype['postgresql'] = 'INTEGER' +>>>>>>> Restored append keyword with a FutureWarning if issubclass(pytype, np.datetime64) or pytype is datetime: # Caution: np.datetime64 is also a subclass of np.number. sqltype['mysql'] = 'DATETIME'