From d0f29dcf0d0849abd0857612bbac97da034fe8ac Mon Sep 17 00:00:00 2001 From: anibalsolon Date: Tue, 19 Jun 2018 12:03:46 -0400 Subject: [PATCH 1/5] avoid much string manip --- nipype/interfaces/io.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index c9487ca5cd..9afd56f66c 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -592,10 +592,8 @@ def _upload_to_s3(self, bucket, src, dst): s3_prefix = s3_str + bucket.name # Explicitly lower-case the "s3" - if dst.lower().startswith(s3_str): - dst_sp = dst.split('/') - dst_sp[0] = dst_sp[0].lower() - dst = '/'.join(dst_sp) + if dst[:len(s3_str)].lower().startswith(s3_str): + dst = s3_str + dst[len(s3_str):] # If src is a directory, collect files (this assumes dst is a dir too) if os.path.isdir(src): From ad93735918def26195bd0490b35738f156ed82b7 Mon Sep 17 00:00:00 2001 From: anibalsolon Date: Tue, 19 Jun 2018 15:05:19 -0400 Subject: [PATCH 2/5] Try to connect to S3 using IAM Roles --- nipype/interfaces/io.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 9afd56f66c..0044482aa5 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -536,19 +536,37 @@ def _fetch_bucket(self, bucket_name): session = boto3.session.Session( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) - s3_resource = session.resource('s3', use_ssl=True) - # Otherwise, connect anonymously else: - iflogger.info('Connecting to AWS: %s anonymously...', bucket_name) + iflogger.info('Connecting to S3 bucket: %s with IAM role...', + bucket_name) + + # Lean on AWS environment / IAM role authentication and authorization session = boto3.session.Session() - s3_resource = session.resource('s3', use_ssl=True) + + s3_resource = session.resource('s3', use_ssl=True) + + # And try fetch the bucket with the name argument + try: + self._get_head_bucket(s3_resource, bucket_name) + except Exception as exc: + + # Try to connect anonymously s3_resource.meta.client.meta.events.register( 'choose-signer.s3.*', botocore.handlers.disable_signing) + iflogger.info('Connecting to AWS: %s anonymously...', bucket_name) + self._get_head_bucket(s3_resource, bucket_name) + # Explicitly declare a secure SSL connection for bucket object bucket = s3_resource.Bucket(bucket_name) + # Return the bucket + return bucket + + + def _get_head_bucket(self, s3_resource, bucket_name): + # And try fetch the bucket with the name argument try: s3_resource.meta.client.head_bucket(Bucket=bucket_name) @@ -570,8 +588,6 @@ def _fetch_bucket(self, bucket_name): % (bucket_name, exc) raise Exception(err_msg) - # Return the bucket - return bucket # Send up to S3 method def _upload_to_s3(self, bucket, src, dst): From aab85cb36373d7171e7b56fb2fc5b57cb17fe559 Mon Sep 17 00:00:00 2001 From: anibalsolon Date: Tue, 19 Jun 2018 16:36:44 -0400 Subject: [PATCH 3/5] import botocore --- nipype/interfaces/io.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 0044482aa5..d9116997d9 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -543,7 +543,7 @@ def _fetch_bucket(self, bucket_name): # Lean on AWS environment / IAM role authentication and authorization session = boto3.session.Session() - + s3_resource = session.resource('s3', use_ssl=True) # And try fetch the bucket with the name argument @@ -567,6 +567,8 @@ def _fetch_bucket(self, bucket_name): def _get_head_bucket(self, s3_resource, bucket_name): + import botocore + # And try fetch the bucket with the name argument try: s3_resource.meta.client.head_bucket(Bucket=bucket_name) From 502b4980bd622c61372b38d26e23491f56fe2844 Mon Sep 17 00:00:00 2001 From: anibalsolon Date: Mon, 23 Jul 2018 11:06:29 -0400 Subject: [PATCH 4/5] fix string condition, move bounded-wannabe method as function --- nipype/interfaces/io.py | 61 +++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index d9116997d9..9b6ff40642 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -123,6 +123,35 @@ def add_traits(base, names, trait_type=None): return base +def get_head_bucket(s3_resource, bucket_name): + """ Try to get the header info of a bucket, in order to + check if it exists and its permissions + """ + + import botocore + + # Try fetch the bucket with the name argument + try: + s3_resource.meta.client.head_bucket(Bucket=bucket_name) + except botocore.exceptions.ClientError as exc: + error_code = int(exc.response['Error']['Code']) + if error_code == 403: + err_msg = 'Access to bucket: %s is denied; check credentials'\ + % bucket_name + raise Exception(err_msg) + elif error_code == 404: + err_msg = 'Bucket: %s does not exist; check spelling and try '\ + 'again' % bucket_name + raise Exception(err_msg) + else: + err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\ + % (bucket_name, exc) + except Exception as exc: + err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\ + % (bucket_name, exc) + raise Exception(err_msg) + + class IOBase(BaseInterface): def _run_interface(self, runtime): return runtime @@ -548,7 +577,7 @@ def _fetch_bucket(self, bucket_name): # And try fetch the bucket with the name argument try: - self._get_head_bucket(s3_resource, bucket_name) + get_head_bucket(s3_resource, bucket_name) except Exception as exc: # Try to connect anonymously @@ -556,7 +585,7 @@ def _fetch_bucket(self, bucket_name): 'choose-signer.s3.*', botocore.handlers.disable_signing) iflogger.info('Connecting to AWS: %s anonymously...', bucket_name) - self._get_head_bucket(s3_resource, bucket_name) + get_head_bucket(s3_resource, bucket_name) # Explicitly declare a secure SSL connection for bucket object bucket = s3_resource.Bucket(bucket_name) @@ -565,32 +594,6 @@ def _fetch_bucket(self, bucket_name): return bucket - def _get_head_bucket(self, s3_resource, bucket_name): - - import botocore - - # And try fetch the bucket with the name argument - try: - s3_resource.meta.client.head_bucket(Bucket=bucket_name) - except botocore.exceptions.ClientError as exc: - error_code = int(exc.response['Error']['Code']) - if error_code == 403: - err_msg = 'Access to bucket: %s is denied; check credentials'\ - % bucket_name - raise Exception(err_msg) - elif error_code == 404: - err_msg = 'Bucket: %s does not exist; check spelling and try '\ - 'again' % bucket_name - raise Exception(err_msg) - else: - err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\ - % (bucket_name, exc) - except Exception as exc: - err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\ - % (bucket_name, exc) - raise Exception(err_msg) - - # Send up to S3 method def _upload_to_s3(self, bucket, src, dst): ''' @@ -610,7 +613,7 @@ def _upload_to_s3(self, bucket, src, dst): s3_prefix = s3_str + bucket.name # Explicitly lower-case the "s3" - if dst[:len(s3_str)].lower().startswith(s3_str): + if dst[:len(s3_str)].lower() == s3_str: dst = s3_str + dst[len(s3_str):] # If src is a directory, collect files (this assumes dst is a dir too) From 5abe1c9a0a6cb489a038c199d770b7799d2253fe Mon Sep 17 00:00:00 2001 From: anibalsolon Date: Mon, 23 Jul 2018 11:12:43 -0400 Subject: [PATCH 5/5] privatize function --- nipype/interfaces/io.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nipype/interfaces/io.py b/nipype/interfaces/io.py index 9b6ff40642..5d50b339d3 100644 --- a/nipype/interfaces/io.py +++ b/nipype/interfaces/io.py @@ -123,7 +123,7 @@ def add_traits(base, names, trait_type=None): return base -def get_head_bucket(s3_resource, bucket_name): +def _get_head_bucket(s3_resource, bucket_name): """ Try to get the header info of a bucket, in order to check if it exists and its permissions """ @@ -577,7 +577,7 @@ def _fetch_bucket(self, bucket_name): # And try fetch the bucket with the name argument try: - get_head_bucket(s3_resource, bucket_name) + _get_head_bucket(s3_resource, bucket_name) except Exception as exc: # Try to connect anonymously @@ -585,7 +585,7 @@ def _fetch_bucket(self, bucket_name): 'choose-signer.s3.*', botocore.handlers.disable_signing) iflogger.info('Connecting to AWS: %s anonymously...', bucket_name) - get_head_bucket(s3_resource, bucket_name) + _get_head_bucket(s3_resource, bucket_name) # Explicitly declare a secure SSL connection for bucket object bucket = s3_resource.Bucket(bucket_name)