From 066cb00f4154d278392b51761eefb6ac256060c3 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Fri, 19 Mar 2021 17:12:55 -0700 Subject: [PATCH 1/6] useragent extension, quoted endpoint, fix batch_status --- scaleapi/__init__.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/scaleapi/__init__.py b/scaleapi/__init__.py index 4048d0b..e4f7ccc 100644 --- a/scaleapi/__init__.py +++ b/scaleapi/__init__.py @@ -1,5 +1,6 @@ import requests import platform +import urllib.parse from .tasks import Task from .batches import Batch @@ -63,11 +64,11 @@ class Batchlist(Paginator): class ScaleClient(object): - def __init__(self, api_key): + def __init__(self, api_key, user_agent_extension=None): self.api_key = api_key self._headers = { "Content-Type": "application/json", - "User-Agent": _generate_useragent() + "User-Agent": _generate_useragent(user_agent_extension) } def _getrequest(self, endpoint, params=None): @@ -171,15 +172,15 @@ def create_batch(self, project, batch_name, callback): return Batch(batchdata, self) def finalize_batch(self, batch_name): - batchdata = self._postrequest('batches/%s/finalize' % batch_name) + batchdata = self._postrequest('batches/%s/finalize' % quote_string(batch_name)) return Batch(batchdata, self) def batch_status(self, batch_name): - batchdata = self._getrequest('batches/%s/status' % batch_name) - return Batch.get_status(self) + status_data = self._getrequest('batches/%s/status' % quote_string(batch_name)) + return status_data def get_batch(self, batch_name): - batchdata = self._getrequest('batches/%s' % batch_name) + batchdata = self._getrequest('batches/%s' % quote_string(batch_name)) return Batch(batchdata, self) def list_batches(self, **kwargs): @@ -202,7 +203,7 @@ def create_project(self, project_name, type, params): return Project(projectdata, self) def get_project(self, project_name): - projectdata = self._getrequest('projects/%s' % project_name) + projectdata = self._getrequest('projects/%s' % quote_string(project_name)) return Project(projectdata, self) def projects(self): @@ -215,19 +216,38 @@ def update_project(self, project_name, **kwargs): if key not in allowed_kwargs: raise ScaleInvalidRequest('Illegal parameter %s for ScaleClient.update_project()' % key, None) - projectdata = self._postrequest('projects/%s/setParams' % project_name, payload=kwargs) + projectdata = self._postrequest('projects/%s/setParams' % quote_string(project_name), payload=kwargs) return projectdata -def _generate_useragent(): +def _generate_useragent(extension=None): try: python_version = platform.python_version() os_platform = platform.platform() - user_agent = '%s/%s Python/%s OS/%s' % (__name__, __version__, python_version, os_platform) + user_agent = " ".join( + filter( + None, + [ + "{}/{}".format(__name__, __version__), + "Python/{}".format(python_version), + "OS/{}".format(os_platform), + extension, + ], + ) + ) return user_agent - except: + + except Exception: return "scaleapi-python-client" +def quote_string(text): + """`quote_string('a bc/def')` -> `a%20bc%2Fdef` + Project and Batch names can be a part of URL, which causes an error + in case of a special character used. Quotation assures + the right object to be retrieved from API. + """ + return urllib.parse.quote(text, safe="") + def _AddTaskTypeCreator(task_type): def create_task_wrapper(self, **kwargs): return self.create_task(task_type, **kwargs) From 5e73c1840ac17259affc870484fb08e71f8c243a Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Fri, 19 Mar 2021 17:13:09 -0700 Subject: [PATCH 2/6] bumped version --- scaleapi/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scaleapi/_version.py b/scaleapi/_version.py index 976498a..92192ee 100644 --- a/scaleapi/_version.py +++ b/scaleapi/_version.py @@ -1 +1 @@ -__version__ = "1.0.3" +__version__ = "1.0.4" From aeffb078ee9ce7c0c0956904004049131dcfa0d9 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Fri, 19 Mar 2021 17:13:32 -0700 Subject: [PATCH 3/6] updates to model objects --- scaleapi/batches.py | 11 +++++++++-- scaleapi/projects.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/scaleapi/batches.py b/scaleapi/batches.py index 55ec6c8..d5ca697 100644 --- a/scaleapi/batches.py +++ b/scaleapi/batches.py @@ -2,11 +2,14 @@ class Batch(object): def __init__(self, param_dict, client): self.param_dict = param_dict self.name = param_dict['name'] + self.status = param_dict["status"] + self.pending = None self.completed = None self.error = None self.canceled = None self.client = client + self.get_status() def __hash__(self): return hash(self.name) @@ -18,9 +21,13 @@ def __repr__(self): return 'Batch(%s)' % self.param_dict def finalize(self): - return self.client._postrequest("batches/%s/finalize" % self.name) + res = self.client.finalize_batch(self.name) + self.status = res.status + return res def get_status(self): - res = self.client._getrequest("batches/%s/status" % self.name) + res = self.client.batch_status(self.name) + self.status = res["status"] for stat in ["pending", "completed", "error", "canceled"]: setattr(self, stat, res.get(stat, 0)) + return res diff --git a/scaleapi/projects.py b/scaleapi/projects.py index e619a07..b9dddd7 100644 --- a/scaleapi/projects.py +++ b/scaleapi/projects.py @@ -2,7 +2,7 @@ class Project(object): def __init__(self, param_dict, client): self.param_dict = param_dict self.name = param_dict['name'] - self.client = client + self.client = client def __hash__(self): return hash(self.name) From 9255307d8d8a8eb292685c6508ac83dac2c5d232 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Fri, 19 Mar 2021 17:13:45 -0700 Subject: [PATCH 4/6] fixed/extended test cases --- tests/test_client.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 7891ec5..e2378f9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,11 +4,12 @@ import scaleapi import time from datetime import datetime +from random import randint import os try: test_api_key = os.environ['SCALE_TEST_API_KEY'] - client = scaleapi.ScaleClient(test_api_key) + client = scaleapi.ScaleClient(test_api_key, 'pytest') except KeyError: raise Exception("Please set the environment variable SCALE_TEST_API_KEY to run tests.") @@ -32,6 +33,7 @@ def test_categorize_ok(): callback_url='http://www.example.com/callback', instruction='Is this company public or private?', attachment_type='website', + force=True, attachment='http://www.google.com/', categories=['public', 'private']) @@ -172,6 +174,7 @@ def test_audiotranscription_ok(): task = client.create_audiotranscription_task( callback_url='http://www.example.com/callback', attachment_type='audio', + instruction='Listen the audio file and transcript.', attachment='https://storage.googleapis.com/deepmind-media/pixie/knowing-what-to-say/second-list/speaker-3.wav', verbatim=False, phrases=['avocado', 'stone'] @@ -202,14 +205,13 @@ def test_cancel(): def test_task_retrieval(): task = make_a_task() task2 = client.fetch_task(task.id) - assert task.status == 'pending' assert task2.status == 'completed' assert task2.id == task.id assert task2.callback_url == task.callback_url assert task2.instruction == task.instruction assert task2.attachment_type == task.attachment_type - assert task2.attachments == task.attachments - assert task2.choices == task.choices + assert task2.attachment == task.attachment + assert task2.geometries == task.geometries assert task2.metadata == task.metadata assert task2.type == task.type assert task2.created_at == task.created_at @@ -242,18 +244,30 @@ def test_tasks_invalid(): def create_a_batch(): return client.create_batch( callback = "http://www.example.com/callback", - batch_name = "kitten_labeling_2020-07", - project = "kitten_labeling" + batch_name = "scaleapi-python-sdk-" + str(randint(0, 99999)), + project = "scaleapi-python-sdk" ) def test_finalize_batch(): batch = create_a_batch() - client.finalize_batch(batch.name) + batch = client.finalize_batch(batch.name) + assert batch.status == 'in_progress' -def get_batch_status(): + batch2 = create_a_batch() + batch2.finalize() + assert batch2.status == 'in_progress' + +def test_get_batch_status(): batch = create_a_batch() client.batch_status(batch.name) + assert batch.status == 'staging' + + batch.finalize() + batch.get_status() # Test status update + assert batch.status == 'in_progress' -def get_batch(): +def test_get_batch(): batch = create_a_batch() - client.get_batch(batch.name) + batch2 = client.get_batch(batch.name) + assert batch.name == batch2.name + assert batch2.status == 'staging' From e4edb0505d0456176d530b16a76f313707ee5754 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Fri, 19 Mar 2021 17:19:59 -0700 Subject: [PATCH 5/6] readme updates --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index e2905fe..96bebb6 100644 --- a/README.rst +++ b/README.rst @@ -105,13 +105,13 @@ __ https://docs.scale.com/reference#list-multiple-tasks Cancel Task ^^^^^^^^^^^ -Cancel a task given its id if work has not stared on the task (task status is `queued` in the UI). Check out `Scale's API documentation`__ for more information. +Cancel a task given its id if work has not started on the task (task status is `Queued` in the UI). Check out `Scale's API documentation`__ for more information. __ https://docs.scale.com/reference#cancel-task .. code-block :: python - task = client.cancel_task('asdfasdfasdfasdfasdfasdf') + task = client.cancel_task('30553edd0b6a93f8f05f0fee') Batches _______ @@ -162,7 +162,7 @@ __ https://docs.scale.com/reference#batch-retrieval .. code-block:: python - client.get_batch( batch_name = "batch_name_01_07_2021" ) + client.get_batch(batch_name = 'batch_name_01_07_2021') List Batches ^^^^^^^^^^^^ From c8f7d2e358d19458d69b1cc4ff3af7e89b125d63 Mon Sep 17 00:00:00 2001 From: Fatih Kurtoglu Date: Mon, 22 Mar 2021 09:12:05 -0700 Subject: [PATCH 6/6] fixed a typo --- tests/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_client.py b/tests/test_client.py index e2378f9..05222b1 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -174,7 +174,7 @@ def test_audiotranscription_ok(): task = client.create_audiotranscription_task( callback_url='http://www.example.com/callback', attachment_type='audio', - instruction='Listen the audio file and transcript.', + instruction='Listen to the audio file and transcript.', attachment='https://storage.googleapis.com/deepmind-media/pixie/knowing-what-to-say/second-list/speaker-3.wav', verbatim=False, phrases=['avocado', 'stone']