KUMO-3323: It adds the ability to cancel multiple jobs

WIP: It is missing tests.

Author: rafaelcapucho

.gitignore

@@ -14,3 +14,4 @@ docs/_build
 
 .DS_Store
 pytestdebug.log
+.idea

scrapinghub/client/exceptions.py

@@ -40,6 +40,11 @@ class Unauthorized(ScrapinghubAPIError):
     """Request lacks valid authentication credentials for the target resource."""
 
 
+class Forbidden(ScrapinghubAPIError):
+    """You don't have the permission to access the requested resource.
+    It is either read-protected or not readable by the server."""
+
+
 class NotFound(ScrapinghubAPIError):
     """Entity doesn't exist (e.g. spider or project)."""
 
@@ -68,6 +73,8 @@ def wrapped(*args, **kwargs):
                 raise BadRequest(http_error=exc)
             elif status_code == 401:
                 raise Unauthorized(http_error=exc)
+            elif status_code == 403:
+                raise Forbidden(http_error=exc)
             elif status_code == 404:
                 raise NotFound(http_error=exc)
             elif status_code == 413:

scrapinghub/client/jobs.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+import json
+
 from ..hubstorage.job import JobMeta as _JobMeta
 from ..hubstorage.job import Items as _Items
 from ..hubstorage.job import Logs as _Logs
@@ -77,6 +79,64 @@ def count(self, spider=None, state=None, has_tag=None, lacks_tag=None,
             params['spider'] = self.spider.name
         return next(self._project.jobq.apiget(('count',), params=params))
 
+    def cancel_jobs(self, keys=None, count=None, **params):
+        """Cancel a list of jobs using the keys provided.
+
+        :param keys: (optional) a list of strings containing the job keys in
+            the format: <project>/<spider>/<job_id>.
+        :param count: (optional) it requires admin access. Used for admins
+            to bulk cancel an amount of ``count`` jobs.
+
+        :return: a dict with the amount of jobs cancelled.
+        :rtype: :class:`dict`
+
+        Usage:
+
+        - cancel jobs 123 and 321 from project 111 and spiders 222 and 333::
+
+            >>> project.jobs.cancel_jobs(['111/222/123', '111/333/321'])
+            {'count': 2}
+
+        - cancel 100 jobs asynchronously::
+
+            >>> project.jobs.cancel_jobs(count=100)
+            {'count': 100}
+        """
+        update_kwargs(params, count=count, keys=keys)
+        keys = params.get('keys')
+        count = params.get('count')
+
+        if keys and count:
+            raise ValueError("keys and count can't be defined simultaneously")
+
+        elif not keys and not count:
+            raise ValueError("keys or count should be defined")
+
+        elif keys:
+            if not isinstance(keys, list):
+                raise ValueError("keys should be a list")
+
+            # it raises ValueError if invalid
+            keys = [parse_job_key(k) for k in keys]
+
+            if not all([key.project_id == self.project_id for key in keys]):
+                raise ValueError(
+                    "all keys should belong to project: %s" % self.project_id
+                )
+
+            # change it to the format in which JobQ expects.
+            data = [{"key": str(k)} for k in keys]
+
+            # may raise BadRequest if JobQ doesn't validate
+            return list(self._project.jobq.apipost("cancel",
+                                                   data=json.dumps(data)))[0]
+        elif count:
+            if not isinstance(count, int):
+                raise ValueError("count should be an int")
+
+            # may raise Forbidden
+            return self._project.jobq.apipost("cancel?count=%s" % count)
+
     def iter(self, count=None, start=None, spider=None, state=None,
              has_tag=None, lacks_tag=None, startts=None, endts=None,
              meta=None, **params):

tests/client/cassetes/test_job/test_cancel_jobs.gz

@@ -0,0 +1 @@
+eJy11gd4HMUVAGDJvRcw2LjA+bDFIbTXVKzCYWRZqFkNCbw2mZDV3kiz0t3uvdldFTuXkAZJIIH0npDee+89Ib333nvvnZm3e6crK1tYXyR9+mZn32x587+ZvW1VHmoi6vqamppZym3DMqF2oFbdIo4N06Fc0x3RZ8MqAqsjeVjjhXIKLrUdWJuHdRF1nejJUodZaVivrhEHI8Nj47BBXS2aLjdgo3pItJjj5NpjscbGqPfXnGhvjbfGY1rOiHHXjE7b4t6bcPyElV6AzepR0bTdCSOdcrQpW0nU5bg1TXUnlfR+6uyckaY8xWzFEU+jeId1Wjp9qxggByVgCz4vo5o4Y8PWPGyLqJtEz0025UrnFDUd2E5gh7pH9OUWxDuYiq1z8UzmFHMnYsloIpqAnZq6XZzv1HWac5RuU7fS4jxcROBidas4MXXGyDWE0nQyozkUdmmYES8aLiFwKWaiPlYPuzW8eZdlmhQTC3sIXIZ9M5TmFC1jzFLYq6nbvChHPJ9ygppTDoN9BParq0R/SyMc0HCGChHjCzkKlxO4Qj0oerVcLmPomrx8bF6Zm5tTJi2eVVyeofLJaRpCGj52pyvelxtnMBQOqknRd0yzDT10Orvc31MtEHZddQOisHOCCoUr83DIU2E7muPacDgPdRGcWXl/uGrgPpyVLLVtbYpCBF9reACudpmYoXovNC1zeQ2BBvWAOBq0zIZQMhnqdzOhZDzRFkom2puOtDe1hHoGx0HRcMisxhcgSiCG+fMmoCHUZVkzBoW4lzPdz5kjc5YgkFR3VOQMITZ6M2VTR9G98U0EmtV75RVsPulYM9RU0nR2wppPDdPOxgUtrQ/2jTb3JrPzc6f70z1Do27vMTszcvJUH/SOTA/BKIz2T8x39U5Pjo93Z/TpocSN9sLJnDmjd4TofM4Q6Ut5bxn331I0St+yIzSozUuzqcZEU1NbSzzeERrRHJaKdYTGtCwdMxyaOqHNQ4sHUAgXJQ1HCLRiut1ZQ7e4CW0er0IiMh6vdgIdOA+NrXCtpq6VSclkrDlIEbgOuciqbggNj4z3DQ+NwVHNZZvzcL0/z1wWRKdaJw7Ohr1pD7eHrZlwQ3jaEiUsDvyqjSViTa3hPByzXTcPXREmFpHjEVaycnSzB7Bi3MCqForkA18oktAj6fVGmFgN+thOjYni7me7NCbqd4Dt1pgo1BNsr5dZX9QggSF19P8hAoY1Jup9pFDvoxoT5X0jC2nsgst0zHWZqMzxCBPleFOEiSpkxcq7GSvvZEnlqQROLaPyTpdU3i0EHhRUeSSg8h5M4NbgyntImV6NwESZXj1Qb5oALeidLNU7RYAF6DU8vdNs2WTbBNkZJJtBsllJVq7rPd3jYAqxbYtibcfiYmGLzmajJZtJVLeyMXFlO1ZaCTF5ZwoW2zwkpyBXtT0BAX7u7clGrU7AVuSi31nf75zwe+F+5j0/C+jnTIWfs+jnoZGgjStP4GFqrehvg4dr6k7RUJUxMbuGTpWb/W+O2wg8ArMpCggeqan7fEW2rcircSujdMr5VIa5MWWY8CgCj8Zr1sNjAvbC2wncEazrsVU78OMIPL5yB77TMzjmGbyLwBPwHqa49XxMpLwlGocnevKPyzzfTeCecxbLESyWJ3nqnizUbRTB4Rw15YdEGJ6CsJ6KsJ5WCuvpFwyrzYf1DB/WMyNMSHoW87A8m3k0nuPTeO6KaDzPo/F8pPGCChr3Io0XBtJ4EYEXsyVRvITAS4soXnZ+FC8n8AoWxOGVBF4VzOHVVRxeQ+C1lRxeV8bh9QTeUM3hjSUc3kTgzcvg8BaPw1urOLwNObwdObyjbGt8p/DQvFwPUPSga6ZOM/AusVfuF6NvORueoQvh9lDJApeIi5+wqNV3y+l6D2p5r6/lfb6W9/taPiC1iP3pgzjHyVb40ArwfNjD8xHE89EKPB9DPB8PxHMfgU9gXCIJn1zK0KcIfLpo6DPnN/RZAp8LNPR5Al8INvTFKkNfIvDlSkNfKTP0VQJfqzb09RJD3yDwzWUY+pZn6NvC0BbcyHTLNR0xtfE8fAcZfRcZfa+M0fdXxOgHglFyCUby064hFHSiTeL6oZzOHyGuH/u4fuLj+qmP62c+rp/j3DbH4RcrwPVLD9evENevK3D9BnH9NhDX7wj8voDrD0vh+iOBPxVx/fn8uP5C4K+BuP5G4O/BuP5RheufBP5VievfZbj+Q+C/1bj+t4iL14itk/Ba8f+cxFolMb5KhKEyvlq0qqAl83xNrfz+dvN8rWhIbnwdNor7GF8vR67wG4lvkBfB3YxvxOsD4ZtkH0Lim2UTLfEtsomc+FbZXMHmxreJC6Aivh1vejjPd2Cj1BLfKaPkk10kzwWI4hdjznfJnKOrOL9EJjaYFr8Uo3fLaB8Y34Ph5zHGL8OBe+VbB0jj+/D0fnndIG/8AN6knBy/HAddIQeVw+MhDF+0xw9iaFiGVgrkV2JwEeEhDD28PIR1RYRXeQjlg4QnDdOwGU2HeaRA8OoCwfoKgteshKD/NcUbFgkqBYLRRYKxRYLxRYKJlRJMFgk2Fgg2VRNsLhBsWYrgEcx4awnBtnMQbMfojlKC1y6LYAoHXrcUwaN4+volCXYGETyGg7oCCB6vJNiNoTcEEuwpJ9iLoX3LI9hfJDgQSPCER5C60fsBC/ZY5Q==

tests/client/test_job.py

@@ -8,6 +8,7 @@
 from scrapinghub.client.logs import Logs
 from scrapinghub.client.requests import Requests
 from scrapinghub.client.samples import Samples
+from scrapinghub.client.exceptions import BadRequest
 
 from ..conftest import TEST_PROJECT_ID
 from ..conftest import TEST_SPIDER_NAME
@@ -44,6 +45,49 @@ def test_job_update_tags(spider):
     assert job2.metadata.get('tags') == ['tag2']
 
 
+def test_cancel_jobs(spider):
+    job1 = spider.jobs.run(job_args={'subid': 'tags-1'}, add_tag=['tag1'])
+    job2 = spider.jobs.run(job_args={'subid': 'tags-2'}, add_tag=['tag2'])
+    assert job1.metadata.get('state') == 'pending'
+    assert job2.metadata.get('state') == 'pending'
+
+    with pytest.raises(ValueError) as err:
+        output = spider.jobs.cancel_jobs()
+
+    assert 'keys or count should be defined' in str(err)
+
+    with pytest.raises(ValueError) as err:
+        output = spider.jobs.cancel_jobs(['2222222/1/1'], count=2)
+
+    assert "keys and count can't be defined simultaneously" in str(err)
+
+    with pytest.raises(ValueError) as err:
+        output = spider.jobs.cancel_jobs(keys="testing")
+
+    assert 'keys should be a list' in str(err)
+
+    with pytest.raises(ValueError) as err:
+        output = spider.jobs.cancel_jobs(count=[1,2])
+
+    assert 'count should be an int' in str(err)
+
+    with pytest.raises(ValueError) as err:
+        output = spider.jobs.cancel_jobs(['2222222/1/1', '2222226/1/1'])
+
+    assert 'all keys should belong to project' in str(err)
+
+    # Non-existent job
+    output = spider.jobs.cancel_jobs(['%s/1/10000' % job1.project_id])
+    assert output == {'count': 0}
+
+    # The expected behavior
+    output = spider.jobs.cancel_jobs([job1.key, job2.key])
+
+    assert job1.metadata.get('state') == 'finished'
+    assert job2.metadata.get('state') == 'finished'
+    assert output == {'count': 2}
+
+
 def test_job_start(spider):
     job = spider.jobs.run()
     assert job.metadata.get('state') == 'pending'