Modified tests for sustainability

- S3DataGrabber uses anon connection to grab from openfmri bucket
- S3DataSink uses fakes3 server to test functionality
- Added 'gem install fakes3' to circle.yml

Author: shariqiqbal2810

circle.yml

@@ -9,7 +9,8 @@ dependencies:
   override:
     - pip install --upgrade pip
     - pip install -e .
-    - pip install matplotlib sphinx ipython boto3
+    - pip install matplotlib sphinx ipython boto
+    - gem install fakes3
     - if [[ ! -d ~/examples/data ]]; then wget "http://tcpdiag.dl.sourceforge.net/project/nipy/nipype/nipype-0.2/nipype-tutorial.tar.bz2"; tar jxvf nipype-tutorial.tar.bz2; mkdir ~/examples; mv nipype-tutorial/* ~/examples/; fi
     # we download this manually because CircleCI does not cache apt
     - if [[ ! -d ~/examples/feeds ]]; then wget "http://fsl.fmrib.ox.ac.uk/fsldownloads/fsl-5.0.8-feeds.tar.gz"; tar zxvf fsl-5.0.8-feeds.tar.gz; mv feeds ~/examples/; fi

nipype/interfaces/io.py

@@ -46,7 +46,8 @@
     pass
 
 try:
-    import boto3
+    import boto
+    from boto.s3.connection import S3Connection, OrdinaryCallingFormat
 except:
     pass
 
@@ -377,13 +378,16 @@ def _list_outputs(self):
 
 
 class S3DataSinkInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
+    testing = traits.Bool(False, usedefault=True,
+                                   desc='Flag for using local fakes3 server.'
+                                        ' (for testing purposes only)')
+    anon = traits.Bool(False, usedefault=True,
+                                   desc='Use anonymous connection to s3')
     bucket = traits.Str(mandatory=True,
                         desc='Amazon S3 bucket where your data is stored')
     bucket_path = traits.Str('', usedefault=True,
                              desc='Location within your bucket to store '
                                    'data.')
-    region = traits.Str('us-east-1', usedefault=True,
-                             desc='Region of s3 bucket')
     base_directory = Directory(
         desc='Path to the base directory for storing data.')
     container = traits.Str(
@@ -435,8 +439,14 @@ def _list_outputs(self):
         return outputs
 
     def localtos3(self, paths):
-        client = boto3.client('s3', self.inputs.region)
-        transfer = boto3.s3.transfer.S3Transfer(client)
+        if self.inputs.testing:
+            conn = S3Connection(anon=True, is_secure=False, port=4567,
+                                host='localhost',
+                                calling_format=OrdinaryCallingFormat())
+
+        else:
+            conn = S3Connection(anon=self.inputs.anon)
+        bkt = conn.get_bucket(self.inputs.bucket)
         s3paths = []
 
         for path in paths:
@@ -446,17 +456,23 @@ def localtos3(self, paths):
                 s3path = path[bd_index+len(self.inputs.base_directory):]  # cut out base directory
                 if s3path[0] == os.path.sep:
                     s3path = s3path[1:]
-            else: # base_directory isn't in path, simply place all files in bucket_path folder
-                s3path = os.path.split(path)[1] # take filename from path
-            s3path = os.path.join(self.inputs.bucket_path, path)
+            else:  # base_directory isn't in path, simply place all files in bucket_path folder
+                s3path = os.path.split(path)[1]  # take filename from path
+            s3path = os.path.join(self.inputs.bucket_path, s3path)
+            if s3path[-1] == os.path.sep:
+                s3path = s3path[:-1]
             s3paths.append(s3path)
 
-            transfer.upload_file(path, self.inputs.bucket, s3path)
+            k = boto.s3.key.Key(bkt)
+            k.key = s3path
+            k.set_contents_from_filename(path)
 
         return s3paths
 
 
 class S3DataGrabberInputSpec(DynamicTraitedSpec, BaseInterfaceInputSpec):
+    anon = traits.Bool(False, usedefault=True,
+                                   desc='Use anonymous connection to s3')
     region = traits.Str('us-east-1', usedefault=True,
                              desc='Region of s3 bucket')
     bucket = traits.Str(mandatory=True,
@@ -557,9 +573,9 @@ def _list_outputs(self):
 
         outputs = {}
         # get list of all files in s3 bucket
-        s3 = boto3.resource('s3')
-        bkt = s3.Bucket(self.inputs.bucket)
-        bkt_files = list(k.key for k in bkt.objects.all())
+        conn = boto.connect_s3(anon=self.inputs.anon)
+        bkt = conn.get_bucket(self.inputs.bucket)
+        bkt_files = list(k.key for k in bkt.list())
 
         # keys are outfields, args are template args for the outfield
         for key, args in self.inputs.template_args.items():
@@ -642,15 +658,15 @@ def _list_outputs(self):
                 paths = outputs[key]
                 for i in range(len(paths)):
                     path = paths[i]
-                    outputs[key][i] = self.s3tolocal(path)
+                    outputs[key][i] = self.s3tolocal(path, bkt)
             elif type(outputs[key]) == str:
-                outputs[key] = self.s3tolocal(outputs[key])
+                outputs[key] = self.s3tolocal(outputs[key], bkt)
 
         return outputs
 
     # Takes an s3 address and downloads the file to a local
     # directory, returning the local path.
-    def s3tolocal(self, s3path):
+    def s3tolocal(self, s3path, bkt):
         # path formatting
         if not os.path.split(self.inputs.local_directory)[1] == '':
             self.inputs.local_directory += '/'
@@ -663,9 +679,9 @@ def s3tolocal(self, s3path):
         localdir = os.path.split(localpath)[0]
         if not os.path.exists(localdir):
             os.makedirs(localdir)
-        client = boto3.client('s3', self.inputs.region)
-        transfer = boto3.s3.transfer.S3Transfer(client)
-        transfer.download_file(self.inputs.bucket, s3path, localpath)
+        k = boto.s3.key.Key(bkt)
+        k.key = s3path
+        k.get_contents_to_filename(localpath)
         return localpath
 
 

nipype/interfaces/tests/test_io.py

@@ -5,6 +5,7 @@
 import shutil
 import os.path as op
 from tempfile import mkstemp, mkdtemp
+from subprocess import Popen
 
 from nose.tools import assert_raises
 import nipype
@@ -14,7 +15,8 @@
 
 noboto = False
 try:
-    import boto3
+    import boto
+    from boto.s3.connection import S3Connection, OrdinaryCallingFormat
 except:
     noboto = True
 
@@ -88,6 +90,7 @@ def test_selectfiles_valueerror():
 @skipif(noboto)
 def test_s3datagrabber_communication():
     dg = nio.S3DataGrabber(infields=['subj_id', 'run_num'], outfields=['func', 'struct'])
+    dg.inputs.anon = True
     dg.inputs.bucket = 'openfmri'
     dg.inputs.bucket_path = 'ds001/'
     tempdir = mkdtemp()
@@ -202,8 +205,19 @@ def test_s3datasink_substitutions():
         f = os.path.join(indir, n)
         files.append(f)
         open(f, 'w')
+
+    # run fakes3 server and set up bucket
+    fakes3dir = op.expanduser('~/fakes3')
+    proc = Popen(['fakes3', '-r', fakes3dir, '-p', '4567'], stdout=open(os.devnull, 'wb'))
+    conn = S3Connection(anon=True, is_secure=False, port=4567,
+                          host='localhost',
+                          calling_format=OrdinaryCallingFormat())
+    conn.create_bucket('test')
+
     ds = nio.S3DataSink(
-        bucket='nipype-test',
+        testing=True,
+        anon=True,
+        bucket='test',
         bucket_path='output/',
         parametrization=False,
         base_directory=outdir,
@@ -222,27 +236,33 @@ def test_s3datasink_substitutions():
                   x in glob.glob(os.path.join(outdir, '*'))]), \
           ['!-yz-b.n', 'ABABAB.n']  # so we got re used 2nd and both patterns
 
-    s3 = boto3.resource('s3')
-    bkt = s3.Bucket(ds.inputs.bucket)
-    bkt_files = list(k.key for k in bkt.objects.all())
+    bkt = conn.get_bucket(ds.inputs.bucket)
+    bkt_files = list(k for k in bkt.list())
 
     found = [False, False]
-    del_dict = {'Objects':[],'Quiet': True}
-    for key in bkt_files:
-        if '!-yz-b.n' in key:
+    failed_deletes = 0
+    for k in bkt_files:
+        if '!-yz-b.n' in k.key:
             found[0] = True
-            del_dict["Objects"].append({'Key': key})
-        if 'ABABAB.n' in key:
+            try:
+                bkt.delete_key(k)
+            except:
+                failed_deletes += 1
+        elif 'ABABAB.n' in k.key:
             found[1] = True
-            del_dict["Objects"].append({'Key': key})
+            try:
+                bkt.delete_key(k)
+            except:
+                failed_deletes += 1
+
+    # ensure delete requests were successful
+    yield assert_equal, failed_deletes, 0
 
     # ensure both keys are found in bucket
     yield assert_equal, found.count(True), 2
 
-    resp = bkt.delete_objects(Delete=del_dict, RequestPayer='requester')
-    # ensure delete request was successful
-    yield assert_equal, resp["ResponseMetadata"]["HTTPStatusCode"], 200
-
+    proc.kill()
+    shutil.rmtree(fakes3dir)
     shutil.rmtree(indir)
     shutil.rmtree(outdir)