From a1bf8b7d9ead518b5d6fda0b7bd124e9108434b8 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Mon, 5 Apr 2021 13:34:29 -0700 Subject: [PATCH 01/11] readme updates --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 95cf508..88838f4 100644 --- a/README.rst +++ b/README.rst @@ -100,9 +100,9 @@ Retrieve a list of `Task` objects, with filters for: ``project_name``, ``batch_n ``get_tasks()`` is a **generator** method and yields ``Task`` objects. -`A generator is another type of function, returns an iterable that you can loop over like a list. +*A generator is another type of function, returns an iterable that you can loop over like a list. However, unlike lists, generators do not store the content in the memory. -That helps you to process a large number of objects without increasing memory usage.` +That helps you to process a large number of objects without increasing memory usage.* If you will iterate through the tasks and process them once, using a generator is the most efficient method. However, if you need to process the list of tasks multiple times, you can wrap the generator in a ``list(...)`` @@ -207,9 +207,9 @@ Retrieve a list of Batches. Optional parameters are ``project_name``, ``batch_st ``get_batches()`` is a **generator** method and yields ``Batch`` objects. -`A generator is another type of function, returns an iterable that you can loop over like a list. +*A generator is another type of function, returns an iterable that you can loop over like a list. However, unlike lists, generators do not store the content in the memory. -That helps you to process a large number of objects without increasing memory usage.` +That helps you to process a large number of objects without increasing memory usage.* When wrapped in a ``list(...)`` statement, it returns a list of Batches by loading them into the memory. From 4ab3c1f3828c12f56ee8ab2f7d82804865d3f1f7 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Tue, 6 Apr 2021 10:57:52 -0700 Subject: [PATCH 02/11] readme updates --- README.rst | 67 +++++++++++++++++++++++++---------------- docs/developer_guide.md | 2 +- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/README.rst b/README.rst index 88838f4..28f886f 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,8 @@ _____ .. code-block:: python import scaleapi - client = scaleapi.ScaleClient('YOUR_API_KEY_HERE') + + client = scaleapi.ScaleClient("YOUR_API_KEY_HERE") Tasks _____ @@ -62,22 +63,28 @@ __ https://docs.scale.com/reference from scaleapi.tasks import TaskType - client.create_task( - TaskType.ImageAnnotation, - project = 'test_project', + payload = dict( + project = "test_project", callback_url = "http://www.example.com/callback", - instruction= "Draw a box around each baby cow and big cow.", + instruction = "Draw a box around each baby cow and big cow.", attachment_type = "image", attachment = "http://i.imgur.com/v4cBreD.jpg", + unique_id = "c235d023af73", geometries = { "box": { - "objects_to_annotate": ["Baby Cow", "Big Cow"], - "min_height": 10, - "min_width": 10 + "objects_to_annotate": ["Baby Cow", "Big Cow"], + "min_height": 10, + "min_width": 10, } - } + }, ) + try: + client.create_task(TaskType.ImageAnnotation, **payload) + except ScaleDuplicateTask as err: + print(err.message) # If unique_id is already used for a different task + + Retrieve a task ^^^^^^^^^^^^^^^ @@ -87,8 +94,8 @@ __ https://docs.scale.com/reference#retrieve-tasks .. code-block :: python - task = client.get_task('30553edd0b6a93f8f05f0fee') - print(task.status) # Task status ('pending', 'completed', 'error', 'canceled') + task = client.get_task("30553edd0b6a93f8f05f0fee") + print(task.status) # Task status ("pending", "completed", "error", "canceled") print(task.response) # If task is complete List Tasks @@ -157,9 +164,9 @@ __ https://docs.scale.com/reference#batch-creation .. code-block:: python client.create_batch( - project = 'test_project', + project = "test_project", callback = "http://www.example.com/callback", - name = 'batch_name_01_07_2021' + name = "batch_name_01_07_2021" ) Finalize Batch @@ -171,7 +178,11 @@ __ https://docs.scale.com/reference#batch-finalization .. code-block:: python - client.finalize_batch(batch_name = 'batch_name_01_07_2021') + client.finalize_batch(batch_name="batch_name_01_07_2021") + + # Alternative method + batch = client.get_batch(batch_name="batch_name_01_07_2021") + batch.finalize() Check Batch Status ^^^^^^^^^^^^^^^^^^ @@ -182,10 +193,10 @@ __ https://docs.scale.com/reference#batch-status .. code-block:: python - client.batch_status(batch_name = 'batch_name_01_07_2021') + client.batch_status(batch_name = "batch_name_01_07_2021") # Alternative via Batch.get_status() - batch = client.get_batch('batch_name_01_07_2021') + batch = client.get_batch("batch_name_01_07_2021") batch.get_status() # Refreshes tasks_{status} attributes of Batch print(batch.tasks_pending, batch.tasks_completed) @@ -198,7 +209,7 @@ __ https://docs.scale.com/reference#batch-retrieval .. code-block:: python - client.get_batch(batch_name = 'batch_name_01_07_2021') + client.get_batch(batch_name = "batch_name_01_07_2021") List Batches ^^^^^^^^^^^^ @@ -229,7 +240,7 @@ __ https://docs.scale.com/reference#batch-list counter = 0 for batch in batches: counter += 1 - print(f'Downloading batch {counter} | {batch.name} | {batch.project}') + print(f"Downloading batch {counter} | {batch.name} | {batch.project}") # Alternative for accessing as a Batch list batch_list = list(batches) @@ -247,12 +258,16 @@ __ https://docs.scale.com/reference#project-creation .. code-block:: python - client.create_project( - project_name = 'test_project', - type = 'imageannotation, - params = {'instruction':'Please label the kittens'} + from scaleapi.tasks import TaskType + + project = client.create_project( + project_name = "Test_Project", + task_type = TaskType.ImageAnnotation, + params = {"instruction": "Please label the kittens"}, ) + print(project.name) # Test_Project + Retrieve Project ^^^^^^^^^^^^^^^^ @@ -262,7 +277,7 @@ __ https://docs.scale.com/reference#project-retrieval .. code-block:: python - client.get_project(project_name = 'test_project') + client.get_project(project_name = "test_project") List Projects ^^^^^^^^^^^^^ @@ -290,9 +305,9 @@ __ https://docs.scale.com/reference#project-update-parameters .. code-block :: python data = client.update_project( - project_name='test_project', + project_name="test_project", patch = false, - instruction='update: Please label all the stuff', + instruction="update: Please label all the stuff", ) Error handling @@ -319,7 +334,7 @@ For example: from scaleapi.exceptions import ScaleException try: - client.create_task(TaskType.TextCollection, attachment='Some parameters are missing.') + client.create_task(TaskType.TextCollection, attachment="Some parameters are missing.") except ScaleException as err: print(err.code) # 400 print(err.message) # Parameter is invalid, reason: "attachments" is required diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 50f44c6..727a12b 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -19,7 +19,7 @@ $ pip install -r docs/dev_requirements.txt ``` ### 3. Setup pre-commit -Assure pre-commit1 is installed: +Assure pre-commit[1] is installed: ```bash $ pre-commit --version # pre-commit 2.11.1 From b9cd4396122d0951d2f8189b6a4c822e55db9eb7 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Tue, 6 Apr 2021 10:58:17 -0700 Subject: [PATCH 03/11] version bumped --- scaleapi/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaleapi/_version.py b/scaleapi/_version.py index 9c9b48f..1b8e2a8 100644 --- a/scaleapi/_version.py +++ b/scaleapi/_version.py @@ -1,2 +1,2 @@ -__version__ = "2.0.0" +__version__ = "2.0.1" __package_name__ = "scaleapi" From 38b275da8a67ce192c97df226a71ebb5798d2c97 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Tue, 6 Apr 2021 11:48:37 -0700 Subject: [PATCH 04/11] docstring tools for developer guide --- docs/developer_guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 727a12b..4589a2b 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -52,6 +52,8 @@ Append following lines to the json file: }, ``` +In Python SDK we follow [Google's Python Docstring Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) for comments and docstring of modules, functions and classes. [Python Docstring Generator](https://marketplace.visualstudio.com/items?itemName=njpwerner.autodocstring) is a useful VS Code extension that helps to generate docstrings. + ### 5. Running pre-commit Tests Manually You can run following command to run pre-commit linter for all files, without a commit. It provides a report for issues as well as fixes formatting. From fff92f5aea2b674d1fe9c3d6e8bf2999eaa71871 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Tue, 6 Apr 2021 16:31:45 -0700 Subject: [PATCH 05/11] circleci configurations --- .circleci/config.yml | 80 +++++++++++++++++++++++++++++++++++++++ docs/dev_requirements.txt | 1 + scaleapi/_version.py | 2 +- tests/test_client.py | 1 + 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..a788bf8 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,80 @@ +version: 2.1 +jobs: + build_test: + docker: + - image: cimg/python:3.6 + steps: + - checkout # checkout source code to working directory + - run: + name: Initialize environment + command: | # install env dependencies + sudo pip install -r docs/dev_requirements.txt + - run: + name: Black Formatting Check # Only validation, without re-formatting + command: | + black --check -t py36 . + - run: + name: isort Import Ordering Check # Only validation, without re-formatting + command: | + isort --check-only --profile black . + - run: + name: Flake8 Lint Check # Uses setup.cfg for configuration + command: | + flake8 . --count --statistics + - run: + name: Pylint Lint Check + command: | + pylint scaleapi + - run: + name: Build Package # create whl and install package + command: | + python setup.py sdist bdist_wheel + sudo pip install --no-cache-dir dist/*.whl + - run: + name: Pytest Test Cases + command: | # Run test suite, uses SCALE_TEST_API_KEY env variable + pytest + - run: + name: Twine PyPI Check + command: | # Validate distribution and setup.py configuration + twine check --strict dist/* + pypi_publish: + docker: + - image: cimg/python:3.6 + steps: + - checkout # checkout source code to working directory + - run: + name: Validate SDK Version + command: | + PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//') + + if pip install "scaleapi>=${PKG_VERSION}" > /dev/null 2>&1; + then + echo "ERROR: You need to increment to a new version before publishing!" + echo "Version found in _version.py file: ${PKG_VERSION}" + exit 1; + fi + - run: + name: Initialize Environment + command: | # install env dependencies + sudo pip install twine + - run: + name: Build and Validate + command: | # create whl, validate with twine + python setup.py sdist bdist_wheel + twine check --strict dist/* + - run: + name: Publish to PyPI + command: | + twine upload dist/* +workflows: + build_test_publish: + jobs: + - build_test + - pypi_publish: + requires: + - build_test + filters: + branches: + only: + - master diff --git a/docs/dev_requirements.txt b/docs/dev_requirements.txt index 8a5d25e..f0c4120 100644 --- a/docs/dev_requirements.txt +++ b/docs/dev_requirements.txt @@ -4,3 +4,4 @@ pre-commit==2.11.1 isort>=5.7.0 pytest>=6.2.2 pylint>=2.7.2 +twine>=3.4.1 diff --git a/scaleapi/_version.py b/scaleapi/_version.py index 1b8e2a8..9c9b48f 100644 --- a/scaleapi/_version.py +++ b/scaleapi/_version.py @@ -1,2 +1,2 @@ -__version__ = "2.0.1" +__version__ = "2.0.0" __package_name__ = "scaleapi" diff --git a/tests/test_client.py b/tests/test_client.py index c2af8f7..be43498 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -19,6 +19,7 @@ TEST_PROJECT_NAME = "scaleapi-python-sdk" try: + print(f"SDK Version: {scaleapi.__version__}") test_api_key = os.environ["SCALE_TEST_API_KEY"] client = scaleapi.ScaleClient(test_api_key, "pytest") except KeyError as err: From bc385adf3c855e6bd4038299ef7d4b7b790bccc2 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Tue, 6 Apr 2021 16:40:34 -0700 Subject: [PATCH 06/11] pytest test key validation --- tests/test_client.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index be43498..d956587 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -21,7 +21,11 @@ try: print(f"SDK Version: {scaleapi.__version__}") test_api_key = os.environ["SCALE_TEST_API_KEY"] - client = scaleapi.ScaleClient(test_api_key, "pytest") + + if test_api_key.startswith("test_") or test_api_key.endswith("|test"): + client = scaleapi.ScaleClient(test_api_key, "pytest") + else: + raise Exception("Please provide a valid TEST environment key.") except KeyError as err: raise Exception( "Please set the environment variable SCALE_TEST_API_KEY to run tests." From 590ebdace4080c15be3ef2965c87acf7a78b3103 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Wed, 7 Apr 2021 10:40:29 -0700 Subject: [PATCH 07/11] added 500 error code to retry --- scaleapi/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaleapi/api.py b/scaleapi/api.py index cdbe693..7ec0ee5 100644 --- a/scaleapi/api.py +++ b/scaleapi/api.py @@ -12,7 +12,7 @@ # Parameters for HTTP retry HTTP_TOTAL_RETRIES = 3 # Number of total retries HTTP_RETRY_BACKOFF_FACTOR = 2 # Wait 1, 2, 4 seconds between retries -HTTP_STATUS_FORCE_LIST = [429, 504] # Status codes to force retry +HTTP_STATUS_FORCE_LIST = [429, 500, 504] # Status codes to force retry HTTP_RETRY_ALLOWED_METHODS = frozenset({"GET", "POST"}) From 6e5d52c77c543e5d900ed5865c1f9ddb562d9740 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Wed, 7 Apr 2021 10:41:13 -0700 Subject: [PATCH 08/11] updated publish script --- publish.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/publish.sh b/publish.sh index 4fbceca..6256ebd 100755 --- a/publish.sh +++ b/publish.sh @@ -23,15 +23,7 @@ echo "Active Git Branch: ${BRANCH_NAME}" # release-1.0.5 # BRANCH_PREFIX="${strarr[0]}" # release # BRANCH_VERSION="${strarr[1]}" # 1.0.5 -while IFS= read -r line; do - if [[ $line == __version__* ]]; - then - IFS=' = ' read -ra strarr <<< "$line" - PKG_VERSION=$( sed -e 's/^"//' -e 's/"$//' <<< "${strarr[1]}" ) - echo "SDK Package Version: ${PKG_VERSION}" - break - fi -done < "${DIR}/${VERSION_FILE}" +PKG_VERSION=$(sed -n 's/^__version__ = //p' "${DIR}/${VERSION_FILE}" | sed -e 's/^"//' -e 's/"$//') if [ "$BRANCH_NAME" != "master" ]; then From 9bfe8e01bc538e763d905622d3cd396cf4ea2d1c Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Wed, 7 Apr 2021 10:41:27 -0700 Subject: [PATCH 09/11] version bumped --- scaleapi/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaleapi/_version.py b/scaleapi/_version.py index 9c9b48f..1b8e2a8 100644 --- a/scaleapi/_version.py +++ b/scaleapi/_version.py @@ -1,2 +1,2 @@ -__version__ = "2.0.0" +__version__ = "2.0.1" __package_name__ = "scaleapi" From 2ed2d25aabb913a978f0a3f06e999bdd7b9c9d78 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Wed, 7 Apr 2021 10:41:57 -0700 Subject: [PATCH 10/11] circleci config updates --- .circleci/config.yml | 45 +++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a788bf8..5ad03da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,14 +1,19 @@ +# CircleCI jobs are only enabled to on Pull Requests and commits to master branch. +# "Only build pull requests" enabled in Project's Advanced Settings. version: 2.1 jobs: build_test: docker: - image: cimg/python:3.6 + resource_class: small steps: - checkout # checkout source code to working directory - run: - name: Initialize environment + name: Install Environment Dependencies command: | # install env dependencies - sudo pip install -r docs/dev_requirements.txt + set -e + pip install --upgrade pip + pip install -r docs/dev_requirements.txt - run: name: Black Formatting Check # Only validation, without re-formatting command: | @@ -22,18 +27,19 @@ jobs: command: | flake8 . --count --statistics - run: - name: Pylint Lint Check + name: Pylint Lint Check # Uses .pylintrc for configuration command: | pylint scaleapi - run: name: Build Package # create whl and install package command: | + set -e python setup.py sdist bdist_wheel - sudo pip install --no-cache-dir dist/*.whl + pip install --no-cache-dir dist/*.whl - run: name: Pytest Test Cases command: | # Run test suite, uses SCALE_TEST_API_KEY env variable - pytest + pytest -v - run: name: Twine PyPI Check command: | # Validate distribution and setup.py configuration @@ -44,28 +50,44 @@ jobs: steps: - checkout # checkout source code to working directory - run: - name: Validate SDK Version + name: Validate Tag Version # Check if the tag name matches the package version + command: | + PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//') + + if [[ "$CIRCLE_TAG" != "v${PKG_VERSION}" ]]; then + echo "ERROR: Tag name ($CIRCLE_TAG) must match package version (v${PKG_VERSION})." + exit 1; + fi + - run: + name: Validate SDK Version Increment # Check if the version is already on PyPI command: | PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//') if pip install "scaleapi>=${PKG_VERSION}" > /dev/null 2>&1; then echo "ERROR: You need to increment to a new version before publishing!" - echo "Version found in _version.py file: ${PKG_VERSION}" + echo "Version (${PKG_VERSION}) already exists on PyPI." exit 1; fi - run: - name: Initialize Environment + name: Install Environment Dependencies command: | # install env dependencies - sudo pip install twine + set -e + pip install --upgrade pip + pip install twine - run: name: Build and Validate command: | # create whl, validate with twine + set -e python setup.py sdist bdist_wheel twine check --strict dist/* - run: name: Publish to PyPI command: | + if test -z "${TWINE_USERNAME}" || test -z "${TWINE_PASSWORD}" ; then + echo "ERROR: Please assign TWINE_USERNAME and TWINE_PASSWORD as environment variables" + exit 1 + fi twine upload dist/* workflows: build_test_publish: @@ -75,6 +97,7 @@ workflows: requires: - build_test filters: + tags: + only: /^v\d+\.\d+\.\d+$/ # Runs only for tags with the format [v1.2.3] branches: - only: - - master + ignore: /.*/ # Runs for none of the branches From b52f636ad91ff3fbf3f81f48bcfa0d865485ef8a Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Wed, 7 Apr 2021 16:33:40 -0700 Subject: [PATCH 11/11] circleci tag filter for dependant job --- .circleci/config.yml | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5ad03da..cbecb14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,23 +7,23 @@ jobs: - image: cimg/python:3.6 resource_class: small steps: - - checkout # checkout source code to working directory + - checkout # checkout source code to working directory - run: name: Install Environment Dependencies - command: | # install env dependencies + command: | # install env dependencies set -e pip install --upgrade pip pip install -r docs/dev_requirements.txt - run: - name: Black Formatting Check # Only validation, without re-formatting + name: Black Formatting Check # Only validation, without re-formatting command: | black --check -t py36 . - run: - name: isort Import Ordering Check # Only validation, without re-formatting + name: isort Import Ordering Check # Only validation, without re-formatting command: | isort --check-only --profile black . - run: - name: Flake8 Lint Check # Uses setup.cfg for configuration + name: Flake8 Lint Check # Uses setup.cfg for configuration command: | flake8 . --count --statistics - run: @@ -38,19 +38,19 @@ jobs: pip install --no-cache-dir dist/*.whl - run: name: Pytest Test Cases - command: | # Run test suite, uses SCALE_TEST_API_KEY env variable + command: | # Run test suite, uses SCALE_TEST_API_KEY env variable pytest -v - run: name: Twine PyPI Check - command: | # Validate distribution and setup.py configuration + command: | # Validate distribution and setup.py configuration twine check --strict dist/* pypi_publish: docker: - image: cimg/python:3.6 steps: - - checkout # checkout source code to working directory + - checkout # checkout source code to working directory - run: - name: Validate Tag Version # Check if the tag name matches the package version + name: Validate Tag Version # Check if the tag name matches the package version command: | PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//') @@ -59,7 +59,7 @@ jobs: exit 1; fi - run: - name: Validate SDK Version Increment # Check if the version is already on PyPI + name: Validate SDK Version Increment # Check if the version is already on PyPI command: | PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//') @@ -71,13 +71,13 @@ jobs: fi - run: name: Install Environment Dependencies - command: | # install env dependencies + command: | # install env dependencies set -e pip install --upgrade pip pip install twine - run: name: Build and Validate - command: | # create whl, validate with twine + command: | # create whl, validate with twine set -e python setup.py sdist bdist_wheel twine check --strict dist/* @@ -92,12 +92,15 @@ jobs: workflows: build_test_publish: jobs: - - build_test + - build_test: + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ # Runs only for tags with the format [v1.2.3] - pypi_publish: requires: - build_test filters: - tags: - only: /^v\d+\.\d+\.\d+$/ # Runs only for tags with the format [v1.2.3] branches: - ignore: /.*/ # Runs for none of the branches + ignore: /.*/ # Runs for none of the branches + tags: + only: /^v\d+\.\d+\.\d+$/ # Runs only for tags with the format [v1.2.3]