diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..5ad03da --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,103 @@ +# CircleCI jobs are only enabled to on Pull Requests and commits to master branch. +# "Only build pull requests" enabled in Project's Advanced Settings. +version: 2.1 +jobs: + build_test: + docker: + - image: cimg/python:3.6 + resource_class: small + steps: + - checkout # checkout source code to working directory + - run: + name: Install Environment Dependencies + command: | # install env dependencies + set -e + pip install --upgrade pip + pip install -r docs/dev_requirements.txt + - run: + name: Black Formatting Check # Only validation, without re-formatting + command: | + black --check -t py36 . + - run: + name: isort Import Ordering Check # Only validation, without re-formatting + command: | + isort --check-only --profile black . + - run: + name: Flake8 Lint Check # Uses setup.cfg for configuration + command: | + flake8 . --count --statistics + - run: + name: Pylint Lint Check # Uses .pylintrc for configuration + command: | + pylint scaleapi + - run: + name: Build Package # create whl and install package + command: | + set -e + python setup.py sdist bdist_wheel + pip install --no-cache-dir dist/*.whl + - run: + name: Pytest Test Cases + command: | # Run test suite, uses SCALE_TEST_API_KEY env variable + pytest -v + - run: + name: Twine PyPI Check + command: | # Validate distribution and setup.py configuration + twine check --strict dist/* + pypi_publish: + docker: + - image: cimg/python:3.6 + steps: + - checkout # checkout source code to working directory + - run: + name: Validate Tag Version # Check if the tag name matches the package version + command: | + PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//') + + if [[ "$CIRCLE_TAG" != "v${PKG_VERSION}" ]]; then + echo "ERROR: Tag name ($CIRCLE_TAG) must match package version (v${PKG_VERSION})." + exit 1; + fi + - run: + name: Validate SDK Version Increment # Check if the version is already on PyPI + command: | + PKG_VERSION=$(sed -n 's/^__version__ = //p' scaleapi/_version.py | sed -e 's/^"//' -e 's/"$//') + + if pip install "scaleapi>=${PKG_VERSION}" > /dev/null 2>&1; + then + echo "ERROR: You need to increment to a new version before publishing!" + echo "Version (${PKG_VERSION}) already exists on PyPI." + exit 1; + fi + - run: + name: Install Environment Dependencies + command: | # install env dependencies + set -e + pip install --upgrade pip + pip install twine + - run: + name: Build and Validate + command: | # create whl, validate with twine + set -e + python setup.py sdist bdist_wheel + twine check --strict dist/* + - run: + name: Publish to PyPI + command: | + if test -z "${TWINE_USERNAME}" || test -z "${TWINE_PASSWORD}" ; then + echo "ERROR: Please assign TWINE_USERNAME and TWINE_PASSWORD as environment variables" + exit 1 + fi + twine upload dist/* +workflows: + build_test_publish: + jobs: + - build_test + - pypi_publish: + requires: + - build_test + filters: + tags: + only: /^v\d+\.\d+\.\d+$/ # Runs only for tags with the format [v1.2.3] + branches: + ignore: /.*/ # Runs for none of the branches diff --git a/README.rst b/README.rst index 95cf508..28f886f 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,8 @@ _____ .. code-block:: python import scaleapi - client = scaleapi.ScaleClient('YOUR_API_KEY_HERE') + + client = scaleapi.ScaleClient("YOUR_API_KEY_HERE") Tasks _____ @@ -62,22 +63,28 @@ __ https://docs.scale.com/reference from scaleapi.tasks import TaskType - client.create_task( - TaskType.ImageAnnotation, - project = 'test_project', + payload = dict( + project = "test_project", callback_url = "http://www.example.com/callback", - instruction= "Draw a box around each baby cow and big cow.", + instruction = "Draw a box around each baby cow and big cow.", attachment_type = "image", attachment = "http://i.imgur.com/v4cBreD.jpg", + unique_id = "c235d023af73", geometries = { "box": { - "objects_to_annotate": ["Baby Cow", "Big Cow"], - "min_height": 10, - "min_width": 10 + "objects_to_annotate": ["Baby Cow", "Big Cow"], + "min_height": 10, + "min_width": 10, } - } + }, ) + try: + client.create_task(TaskType.ImageAnnotation, **payload) + except ScaleDuplicateTask as err: + print(err.message) # If unique_id is already used for a different task + + Retrieve a task ^^^^^^^^^^^^^^^ @@ -87,8 +94,8 @@ __ https://docs.scale.com/reference#retrieve-tasks .. code-block :: python - task = client.get_task('30553edd0b6a93f8f05f0fee') - print(task.status) # Task status ('pending', 'completed', 'error', 'canceled') + task = client.get_task("30553edd0b6a93f8f05f0fee") + print(task.status) # Task status ("pending", "completed", "error", "canceled") print(task.response) # If task is complete List Tasks @@ -100,9 +107,9 @@ Retrieve a list of `Task` objects, with filters for: ``project_name``, ``batch_n ``get_tasks()`` is a **generator** method and yields ``Task`` objects. -`A generator is another type of function, returns an iterable that you can loop over like a list. +*A generator is another type of function, returns an iterable that you can loop over like a list. However, unlike lists, generators do not store the content in the memory. -That helps you to process a large number of objects without increasing memory usage.` +That helps you to process a large number of objects without increasing memory usage.* If you will iterate through the tasks and process them once, using a generator is the most efficient method. However, if you need to process the list of tasks multiple times, you can wrap the generator in a ``list(...)`` @@ -157,9 +164,9 @@ __ https://docs.scale.com/reference#batch-creation .. code-block:: python client.create_batch( - project = 'test_project', + project = "test_project", callback = "http://www.example.com/callback", - name = 'batch_name_01_07_2021' + name = "batch_name_01_07_2021" ) Finalize Batch @@ -171,7 +178,11 @@ __ https://docs.scale.com/reference#batch-finalization .. code-block:: python - client.finalize_batch(batch_name = 'batch_name_01_07_2021') + client.finalize_batch(batch_name="batch_name_01_07_2021") + + # Alternative method + batch = client.get_batch(batch_name="batch_name_01_07_2021") + batch.finalize() Check Batch Status ^^^^^^^^^^^^^^^^^^ @@ -182,10 +193,10 @@ __ https://docs.scale.com/reference#batch-status .. code-block:: python - client.batch_status(batch_name = 'batch_name_01_07_2021') + client.batch_status(batch_name = "batch_name_01_07_2021") # Alternative via Batch.get_status() - batch = client.get_batch('batch_name_01_07_2021') + batch = client.get_batch("batch_name_01_07_2021") batch.get_status() # Refreshes tasks_{status} attributes of Batch print(batch.tasks_pending, batch.tasks_completed) @@ -198,7 +209,7 @@ __ https://docs.scale.com/reference#batch-retrieval .. code-block:: python - client.get_batch(batch_name = 'batch_name_01_07_2021') + client.get_batch(batch_name = "batch_name_01_07_2021") List Batches ^^^^^^^^^^^^ @@ -207,9 +218,9 @@ Retrieve a list of Batches. Optional parameters are ``project_name``, ``batch_st ``get_batches()`` is a **generator** method and yields ``Batch`` objects. -`A generator is another type of function, returns an iterable that you can loop over like a list. +*A generator is another type of function, returns an iterable that you can loop over like a list. However, unlike lists, generators do not store the content in the memory. -That helps you to process a large number of objects without increasing memory usage.` +That helps you to process a large number of objects without increasing memory usage.* When wrapped in a ``list(...)`` statement, it returns a list of Batches by loading them into the memory. @@ -229,7 +240,7 @@ __ https://docs.scale.com/reference#batch-list counter = 0 for batch in batches: counter += 1 - print(f'Downloading batch {counter} | {batch.name} | {batch.project}') + print(f"Downloading batch {counter} | {batch.name} | {batch.project}") # Alternative for accessing as a Batch list batch_list = list(batches) @@ -247,12 +258,16 @@ __ https://docs.scale.com/reference#project-creation .. code-block:: python - client.create_project( - project_name = 'test_project', - type = 'imageannotation, - params = {'instruction':'Please label the kittens'} + from scaleapi.tasks import TaskType + + project = client.create_project( + project_name = "Test_Project", + task_type = TaskType.ImageAnnotation, + params = {"instruction": "Please label the kittens"}, ) + print(project.name) # Test_Project + Retrieve Project ^^^^^^^^^^^^^^^^ @@ -262,7 +277,7 @@ __ https://docs.scale.com/reference#project-retrieval .. code-block:: python - client.get_project(project_name = 'test_project') + client.get_project(project_name = "test_project") List Projects ^^^^^^^^^^^^^ @@ -290,9 +305,9 @@ __ https://docs.scale.com/reference#project-update-parameters .. code-block :: python data = client.update_project( - project_name='test_project', + project_name="test_project", patch = false, - instruction='update: Please label all the stuff', + instruction="update: Please label all the stuff", ) Error handling @@ -319,7 +334,7 @@ For example: from scaleapi.exceptions import ScaleException try: - client.create_task(TaskType.TextCollection, attachment='Some parameters are missing.') + client.create_task(TaskType.TextCollection, attachment="Some parameters are missing.") except ScaleException as err: print(err.code) # 400 print(err.message) # Parameter is invalid, reason: "attachments" is required diff --git a/docs/dev_requirements.txt b/docs/dev_requirements.txt index 8a5d25e..f0c4120 100644 --- a/docs/dev_requirements.txt +++ b/docs/dev_requirements.txt @@ -4,3 +4,4 @@ pre-commit==2.11.1 isort>=5.7.0 pytest>=6.2.2 pylint>=2.7.2 +twine>=3.4.1 diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 50f44c6..4589a2b 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -19,7 +19,7 @@ $ pip install -r docs/dev_requirements.txt ``` ### 3. Setup pre-commit -Assure pre-commit1 is installed: +Assure pre-commit[1] is installed: ```bash $ pre-commit --version # pre-commit 2.11.1 @@ -52,6 +52,8 @@ Append following lines to the json file: }, ``` +In Python SDK we follow [Google's Python Docstring Guide](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) for comments and docstring of modules, functions and classes. [Python Docstring Generator](https://marketplace.visualstudio.com/items?itemName=njpwerner.autodocstring) is a useful VS Code extension that helps to generate docstrings. + ### 5. Running pre-commit Tests Manually You can run following command to run pre-commit linter for all files, without a commit. It provides a report for issues as well as fixes formatting. diff --git a/publish.sh b/publish.sh index 4fbceca..6256ebd 100755 --- a/publish.sh +++ b/publish.sh @@ -23,15 +23,7 @@ echo "Active Git Branch: ${BRANCH_NAME}" # release-1.0.5 # BRANCH_PREFIX="${strarr[0]}" # release # BRANCH_VERSION="${strarr[1]}" # 1.0.5 -while IFS= read -r line; do - if [[ $line == __version__* ]]; - then - IFS=' = ' read -ra strarr <<< "$line" - PKG_VERSION=$( sed -e 's/^"//' -e 's/"$//' <<< "${strarr[1]}" ) - echo "SDK Package Version: ${PKG_VERSION}" - break - fi -done < "${DIR}/${VERSION_FILE}" +PKG_VERSION=$(sed -n 's/^__version__ = //p' "${DIR}/${VERSION_FILE}" | sed -e 's/^"//' -e 's/"$//') if [ "$BRANCH_NAME" != "master" ]; then diff --git a/scaleapi/_version.py b/scaleapi/_version.py index 9c9b48f..1b8e2a8 100644 --- a/scaleapi/_version.py +++ b/scaleapi/_version.py @@ -1,2 +1,2 @@ -__version__ = "2.0.0" +__version__ = "2.0.1" __package_name__ = "scaleapi" diff --git a/scaleapi/api.py b/scaleapi/api.py index cdbe693..7ec0ee5 100644 --- a/scaleapi/api.py +++ b/scaleapi/api.py @@ -12,7 +12,7 @@ # Parameters for HTTP retry HTTP_TOTAL_RETRIES = 3 # Number of total retries HTTP_RETRY_BACKOFF_FACTOR = 2 # Wait 1, 2, 4 seconds between retries -HTTP_STATUS_FORCE_LIST = [429, 504] # Status codes to force retry +HTTP_STATUS_FORCE_LIST = [429, 500, 504] # Status codes to force retry HTTP_RETRY_ALLOWED_METHODS = frozenset({"GET", "POST"}) diff --git a/tests/test_client.py b/tests/test_client.py index c2af8f7..d956587 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -19,8 +19,13 @@ TEST_PROJECT_NAME = "scaleapi-python-sdk" try: + print(f"SDK Version: {scaleapi.__version__}") test_api_key = os.environ["SCALE_TEST_API_KEY"] - client = scaleapi.ScaleClient(test_api_key, "pytest") + + if test_api_key.startswith("test_") or test_api_key.endswith("|test"): + client = scaleapi.ScaleClient(test_api_key, "pytest") + else: + raise Exception("Please provide a valid TEST environment key.") except KeyError as err: raise Exception( "Please set the environment variable SCALE_TEST_API_KEY to run tests."