remove 3.5, better support 3.8, Pipfile (#769)

Author: thehesiod

.flake8

@@ -1,3 +1,3 @@
 [flake8]
-# Bump by 6 to account for "await " as compared to botocore
-max-line-length = 86
+# to match longest line in botocore: https://github.com/boto/botocore/blob/develop/botocore/client.py#L730
+max-line-length = 88

.pyup.yml

@@ -2,3 +2,7 @@
 label_prs: deps-update
 
 schedule: every week
+
+requirements:
+  - Pipfile
+  - docs/requirements.txt

.travis.yml

@@ -7,7 +7,7 @@ python:
     - 3.6.10
     - 3.7.6
     - 3.8.1
-# moto is incompatible with 3.5 because it relies on the key order of its regex rules
+# moto is incompatible with 3.5 because it relies on the key order of its url path regex rules
 #    - 3.5.9
 
 matrix:
@@ -18,16 +18,26 @@ matrix:
     env: EXTRA="boto3"
 
 install:
-    - pip install -U setuptools
-    - if [[ -v EXTRA ]]; then pip install pipdeptree; fi
-    - if ! [[ -v EXTRA ]]; then pip install -r requirements-dev.txt; fi
-    - pip install codecov
-    - pip freeze
+    - pip install -U setuptools pip
+    - if ! [[ -v EXTRA ]]; then
+        pip install -U pipenv &&
+        pipenv lock &&
+        pipenv sync --dev &&
+        pipenv check &&
+        pipenv graph;
+      else
+        pip install codecov &&
+        pip check &&
+        pip freeze;
+      fi
 
 script:
-    # requests from idna and docker packages conflicts with rest of the system...
-    - if [[ -v EXTRA ]]; then pip install -U .[$EXTRA] && pipdeptree; fi
-    - if ! [[ -v EXTRA ]]; then make flake mototest; fi
+    - if [[ -v EXTRA ]]; then
+        pip install -U pipdeptree .[$EXTRA] &&
+        pipdeptree;
+      else
+        make flake mototest;
+      fi
 after_success:
     codecov
 deploy:

CHANGES.rst

@@ -3,8 +3,11 @@ Changes
 
 0.12.0 (TBD)
 ^^^^^^^^^^^^
-
 * Bump botocore and extras
+* Drop support for 3.5 given we are unable to test it with moto
+  and it will soon be unsupported
+* Remove loop parameters for Python 3.8 compliance
+* Remove deprecated AioPageIterator.next_page
 
 0.11.1 (2020-01-03)
 ^^^^^^^^^^^^^^^^^^^

CONTRIBUTING.rst

@@ -22,8 +22,7 @@ For example using *virtualenvwrapper* commands could look like::
 
 After that please install libraries required for development::
 
-    $ pip install -r requirements-dev.txt
-    $ pip install -e .
+    $ pipenv sync --dev
 
 Congratulations, you are ready to run the test suite::
 
@@ -78,7 +77,7 @@ versions that are compatible with the above changes.
 See next section describing types of changes we must validate and support.
 
 Hashes of Botocore Code (important)
------------------------
+-----------------------------------
 Because of the way aiobotocore is implemented (see Background section), it is very tightly coupled with botocore.  The validity of these couplings are enforced in test_patches.py.  We also depend on some private properties in aiohttp, and because of this have entries in test_patches.py for this too.
 
 These patches are important to catch cases where botocore functionality was added/removed and needs to be reflected in our overridden methods.  Changes include:

Makefile

@@ -3,24 +3,24 @@
 FLAGS=
 
 flake: checkrst
-	python3 -m flake8 --format=abspath
+	pipenv run python3 -m flake8 --format=abspath
 
 test: flake
-	python3 -m pytest -s $(FLAGS) ./tests/
+	pipenv run python3 -Wd -m pytest -s -vv $(FLAGS) ./tests/
 
 vtest:
-	python3 -m pytest -s -v $(FLAGS) ./tests/
+	pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -s -vv $(FLAGS) ./tests/
 
 checkrst:
-	python3 setup.py check -rms
+	pipenv run python3 setup.py check -rms
 
 cov cover coverage: flake
-	python3 -m pytest -s -v --cov-report term --cov-report html --cov aiobotocore ./tests
+	pipenv run python3 -Wd -m pytest -s -vv --cov-report term --cov-report html --cov aiobotocore ./tests
 	@echo "open file://`pwd`/htmlcov/index.html"
 
 # BOTO_CONFIG solves https://github.com/travis-ci/travis-ci/issues/7940
 mototest:
-	BOTO_CONFIG=/dev/null python3 -m pytest -v -m moto -n auto --cov-report term --cov-report html --cov aiobotocore tests
+	BOTO_CONFIG=/dev/null pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov aiobotocore tests
 	@echo "open file://`pwd`/htmlcov/index.html"
 
 

Pipfile

@@ -0,0 +1,23 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+codecov = "*"
+coverage = "==5.0.3"
+flake8 = "==3.7.9"
+flake8-formatter-abspath = "==1.0.1"
+moto = {extras = ["server"],version = "==1.3.14"}
+pytest = "==5.3.5"
+pytest-cov = "==2.8.1"
+pytest-asyncio = "==0.10.0"
+pytest-xdist = "==1.31.0"
+dill = "==0.3.1.1"
+
+# for some reason this is needed when running setup.py check -rms on travis on 3.6.10
+Pygments = "*"
+
+[packages]
+aiohttp = "==3.3.2"
+aiobotocore = {editable = true,extras = ["awscli", "boto3"],path = "."}

README.rst

@@ -42,13 +42,13 @@ Basic Example
     AWS_SECRET_ACCESS_KEY = "xxx"
 
 
-    async def go(loop):
+    async def go():
         bucket = 'dataintake'
         filename = 'dummy.bin'
         folder = 'aiobotocore'
         key = '{}/{}'.format(folder, filename)
 
-        session = aiobotocore.get_session(loop=loop)
+        session = aiobotocore.get_session()
         async with session.create_client('s3', region_name='us-west-2',
                                        aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
                                        aws_access_key_id=AWS_ACCESS_KEY_ID) as client:
@@ -80,7 +80,7 @@ Basic Example
             print(resp)
 
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(go(loop))
+    loop.run_until_complete(go())
 
 
 Supported AWS Services
@@ -121,7 +121,7 @@ secret accessible via environment variables:
     $ cd aiobotocore
     $ export AWS_ACCESS_KEY_ID=xxx
     $ export AWS_SECRET_ACCESS_KEY=xxx
-    $ pip install -Ur requirements-dev.txt
+    $ pipenv sync --dev
 
 Execute tests suite:
 

aiobotocore/args.py

@@ -1,6 +1,6 @@
-import asyncio
 import copy
-import botocore.args
+
+from botocore.args import ClientArgsCreator
 import botocore.serialize
 import botocore.parsers
 from botocore.signers import RequestSigner
@@ -9,11 +9,7 @@
 from .endpoint import AioEndpointCreator
 
 
-class AioClientArgsCreator(botocore.args.ClientArgsCreator):
-    def __init__(self, *args, loop=None, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._loop = loop or asyncio.get_event_loop()
-
+class AioClientArgsCreator(ClientArgsCreator):
     # NOTE: we override this so we can pull out the custom AioConfig params and
     #       use an AioEndpointCreator
     def get_client_args(self, service_model, region_name, is_secure,
@@ -45,13 +41,14 @@ def get_client_args(self, service_model, region_name, is_secure,
 
         config_kwargs['s3'] = s3_config
 
+        # aiobotocore addition
         if isinstance(client_config, AioConfig):
             connector_args = client_config.connector_args
         else:
             connector_args = None
 
         new_config = AioConfig(connector_args, **config_kwargs)
-        endpoint_creator = AioEndpointCreator(event_emitter, loop=self._loop)
+        endpoint_creator = AioEndpointCreator(event_emitter)
 
         endpoint = endpoint_creator.create_endpoint(
             service_model, region_name=endpoint_region_name,
@@ -77,6 +74,5 @@ def get_client_args(self, service_model, region_name, is_secure,
             'loader': self._loader,
             'client_config': new_config,
             'partition': partition,
-            'exceptions_factory': self._exceptions_factory,
-            'loop': self._loop
+            'exceptions_factory': self._exceptions_factory
         }

aiobotocore/client.py

@@ -1,5 +1,3 @@
-import asyncio
-
 from botocore.client import logger, PaginatorDocstring, ClientCreator, BaseClient
 from botocore.exceptions import OperationNotPageableError
 from botocore.history import get_global_history_recorder
@@ -15,11 +13,6 @@
 
 
 class AioClientCreator(ClientCreator):
-
-    def __init__(self, *args, loop=None, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._loop = loop or asyncio.get_event_loop()
-
     def _create_client_class(self, service_name, service_model):
         class_attributes = self._create_methods(service_model)
         py_name_to_operation_name = self._create_name_mapping(service_model)
@@ -42,18 +35,13 @@ def _get_client_args(self, service_model, region_name, is_secure,
         args_creator = AioClientArgsCreator(
             self._event_emitter, self._user_agent,
             self._response_parser_factory, self._loader,
-            self._exceptions_factory, config_store=self._config_store,
-            loop=self._loop)
+            self._exceptions_factory, config_store=self._config_store)
         return args_creator.get_client_args(
             service_model, region_name, is_secure, endpoint_url,
             verify, credentials, scoped_config, client_config, endpoint_bridge)
 
 
 class AioBaseClient(BaseClient):
-    def __init__(self, *args, **kwargs):
-        self._loop = kwargs.pop('loop', None) or asyncio.get_event_loop()
-        super().__init__(*args, **kwargs)
-
     async def _make_api_call(self, operation_name, api_params):
         operation_model = self._service_model.operation_model(operation_name)
         service_name = self._service_model.service_name
@@ -103,11 +91,9 @@ async def _make_api_call(self, operation_name, api_params):
         else:
             return parsed_response
 
-    async def _make_request(self, operation_model, request_dict,
-                            request_context):
+    async def _make_request(self, operation_model, request_dict, request_context):
         try:
-            return await self._endpoint.make_request(operation_model,
-                                                     request_dict)
+            return await self._endpoint.make_request(operation_model, request_dict)
         except Exception as e:
             self.meta.events.emit(
                 'after-call-error.{service_id}.{operation_name}'.format(
@@ -167,14 +153,15 @@ def paginate(self, **kwargs):
             documented_paginator_cls = type(
                 paginator_class_name, (AioPaginator,), {'paginate': paginate})
 
-            operation_model = self._service_model.operation_model(
-                actual_operation_name)
+            operation_model = self._service_model.operation_model(actual_operation_name)
             paginator = documented_paginator_cls(
                 getattr(self, operation_name),
                 paginator_config,
                 operation_model)
             return paginator
 
+    # NOTE: this method does not differ from botocore, however it's important to keep
+    #   as the "waiter" value points to our own asyncio waiter module
     def get_waiter(self, waiter_name):
         """Returns an object that can wait for some condition.
 
@@ -196,7 +183,7 @@ def get_waiter(self, waiter_name):
             raise ValueError("Waiter does not exist: %s" % waiter_name)
 
         return waiter.create_waiter_with_client(
-            mapping[waiter_name], model, self, loop=self._loop)
+            mapping[waiter_name], model, self)
 
     async def __aenter__(self):
         await self._endpoint.http_session.__aenter__()

aiobotocore/endpoint.py

@@ -59,10 +59,8 @@ async def convert_to_response_dict(http_response, operation_model):
 
 
 class AioEndpoint(Endpoint):
-    def __init__(self, *args, loop=None, proxies=None, **kwargs):
+    def __init__(self, *args, proxies=None, **kwargs):
         super().__init__(*args, **kwargs)
-
-        self._loop = loop or asyncio.get_event_loop()
         self.proxies = proxies or {}
 
     async def _send_request(self, request_dict, operation_model):
@@ -184,7 +182,7 @@ async def _needs_retry(self, attempts, operation_model, request_dict,
             # for the specified number of times.
             logger.debug("Response received to retry, sleeping for "
                          "%s seconds", handler_response)
-            await asyncio.sleep(handler_response, loop=self._loop)
+            await asyncio.sleep(handler_response)
             return True
 
     async def _send(self, request):
@@ -229,10 +227,6 @@ async def _send(self, request):
 
 
 class AioEndpointCreator(EndpointCreator):
-    def __init__(self, *args, loop=None, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._loop = loop
-
     # TODO: handle socket_options
     def create_endpoint(self, service_model, region_name, endpoint_url,
                         verify=None, response_parser_factory=None,
@@ -282,7 +276,6 @@ def create_endpoint(self, service_model, region_name, endpoint_url,
             ssl_context.load_cert_chain(cert_file, key_file)
 
         connector = aiohttp.TCPConnector(
-            loop=self._loop,
             limit=max_pool_connections,
             verify_ssl=self._get_verify_value(verify),
             ssl_context=ssl_context,
@@ -293,7 +286,6 @@ def create_endpoint(self, service_model, region_name, endpoint_url,
             timeout=timeout,
             skip_auto_headers={'CONTENT-TYPE'},
             response_class=ClientResponseProxy,
-            loop=self._loop,
             auto_decompress=False)
 
         return AioEndpoint(
@@ -302,4 +294,4 @@ def create_endpoint(self, service_model, region_name, endpoint_url,
             event_emitter=self._event_emitter,
             response_parser_factory=response_parser_factory,
             http_session=aio_session,
-            loop=self._loop, proxies=proxies)
+            proxies=proxies)

aiobotocore/eventstream.py

@@ -1,25 +1,22 @@
 from botocore.eventstream import EventStream, EventStreamBuffer
-from async_generator import async_generator, yield_
 
 
 class AioEventStream(EventStream):
-    @async_generator
     async def _create_raw_event_generator(self):
         event_stream_buffer = EventStreamBuffer()
         async for chunk, _ in self._raw_stream.iter_chunks():
             event_stream_buffer.add_data(chunk)
             for event in event_stream_buffer:
-                await yield_(event)
+                yield event
 
     def __iter__(self):
         raise NotImplementedError('Use async-for instead')
 
     def __aiter__(self):
         return self.__anext__()
 
-    @async_generator
     async def __anext__(self):
         async for event in self._event_generator:
             parsed_event = self._parse_event(event)
             if parsed_event:
-                await yield_(parsed_event)
+                yield parsed_event