Add tests for sh.hs.client/collections

Author: vshlapakov

tests/hubstorage/conftest.py

@@ -11,6 +11,7 @@
 TEST_FRONTIER_NAME = 'test'
 TEST_FRONTIER_SLOT = 'site.com'
 TEST_BOTGROUPS = ['python-hubstorage-test', 'g1']
+TEST_COLLECTION_NAME = "test_collection_123"
 
 VCR_CASSETES_DIR = 'tests/hubstorage/cassetes'
 
@@ -41,27 +42,37 @@ def hsspiderid(hsproject):
     return str(hsproject.ids.spider(TEST_SPIDER_NAME, create=1))
 
 
+@pytest.fixture
+def hscollection(hsproject):
+    return hsproject.collections.new_store(TEST_COLLECTION_NAME)
+
+
 @pytest.fixture(autouse=True, scope='session')
 def setup_session(hsclient, hsproject):
     yield
     hsclient.close()
 
 
 @pytest.fixture(autouse=True)
-def setup_test(hsclient, hsproject, request, vcr_instance):
+def setup_test(hsclient, hsproject, hscollection, request, vcr_instance):
     cassette_name = '{}/{}.yaml'.format(
         request.function.__module__.split('.')[-1],
         request.function.__name__
     )
     with vcr_instance.use_cassette(cassette_name):
         _set_testbotgroup(hsproject)
         _remove_all_jobs(hsproject)
+        _clean_collection(hscollection)
         yield
         _remove_all_jobs(hsproject)
         _unset_testbotgroup(hsproject)
 
 # ----------------------------------------------------------------------------
 
+def _clean_collection(hscollection):
+    for item in hscollection.iter_values():
+        hscollection.delete(item['_key'])
+
 
 def _set_testbotgroup(hsproject):
     hsproject.settings.apipost(jl={'botgroups': [TEST_BOTGROUPS[0]]})
@@ -81,7 +92,6 @@ def _unset_testbotgroup(hsproject):
         requests.delete(url, auth=hsproject.auth)
 
 
-
 def _remove_all_jobs(hsproject):
     for k in list(hsproject.settings.keys()):
         if k != 'botgroups':

tests/hubstorage/test_batchuploader.py

@@ -8,11 +8,12 @@
 
 from scrapinghub.hubstorage import ValueTooLarge
 from .conftest import TEST_SPIDER_NAME, TEST_AUTH
+from .conftest import start_job
 
 
 def _job_and_writer(hsclient, hsproject, **writerargs):
-    job = hsproject.push_job(TEST_SPIDER_NAME)
-    hsproject.jobq.start(job)
+    hsproject.push_job(TEST_SPIDER_NAME)
+    start_job(hsproject)
     batch_uploader = hsclient.batchuploader
     writer = batch_uploader.create_writer(
         job.items.url, auth=TEST_AUTH, **writerargs)

tests/hubstorage/test_client.py

@@ -1,53 +1,59 @@
 """
 Test Client
 """
-from .hstestcase import HSTestCase
 from scrapinghub import HubstorageClient
 from scrapinghub.hubstorage.utils import apipoll
 
-class ClientTest(HSTestCase):
-
-    def test_default_ua(self):
-        self.assertEqual(self.hsclient.user_agent,
-                         HubstorageClient.DEFAULT_USER_AGENT)
-
-    def test_custom_ua(self):
-        client = HubstorageClient(auth=HSTestCase.auth,
-                                  endpoint=HSTestCase.endpoint,
-                                  user_agent='testUA')
-        self.assertEqual(client.user_agent, 'testUA')
-
-    def test_push_job(self):
-        c = self.hsclient
-        c.push_job(self.projectid, self.spidername,
-                   priority=self.project.jobq.PRIO_LOW,
-                   foo='baz')
-        job = self.start_job()
-        m = job.metadata
-        self.assertEqual(m.get('state'), u'running', c.auth)
-        self.assertEqual(m.get('foo'), u'baz')
-        self.project.jobq.finish(job)
-        self.project.jobq.delete(job)
-
-        # job auth token is valid only while job is running
-        m = c.get_job(job.key).metadata
-        self.assertEqual(m.get('state'), u'deleted')
-        self.assertEqual(m.get('foo'), u'baz')
-
-    def test_jobsummaries(self):
-        hsc = self.hsclient
-        # add at least one running or pending job to ensure summary is returned
-        hsc.push_job(self.projectid, self.spidername, state='running')
-
-        def _get_summary():
-            jss = hsc.projects.jobsummaries()
-            mjss = dict((str(js['project']), js) for js in jss)
-            return mjss.get(self.projectid)
-        summary = apipoll(_get_summary)
-        self.assertIsNotNone(summary)
-
-    def test_timestamp(self):
-        ts1 = self.hsclient.server_timestamp()
-        ts2 = self.hsclient.server_timestamp()
-        self.assertGreater(ts1, 0)
-        self.assertLessEqual(ts1, ts2)
+from .conftest import TEST_AUTH, TEST_ENDPOINT
+from .conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME
+from .conftest import start_job
+
+
+def test_default_ua(hsclient):
+    assert hsclient.user_agent == HubstorageClient.DEFAULT_USER_AGENT
+
+
+def test_custom_ua():
+    client = HubstorageClient(auth=TEST_AUTH,
+                              endpoint=TEST_ENDPOINT,
+                              user_agent='testUA')
+    assert client.user_agent == 'testUA'
+
+
+def test_push_job(hsclient, hsproject):
+    hsclient.push_job(
+        TEST_PROJECT_ID, TEST_SPIDER_NAME,
+        priority=hsproject.jobq.PRIO_LOW,
+        foo='baz',
+    )
+    job = start_job(hsproject)
+    meta = job.metadata
+    assert meta.get('state') == u'running', hsclient.auth
+    assert meta.get('foo') == u'baz'
+    hsproject.jobq.finish(job)
+    hsproject.jobq.delete(job)
+
+    # job auth token is valid only while job is running
+    meta = hsclient.get_job(job.key).metadata
+    assert meta.get('state') == u'deleted'
+    assert meta.get('foo') == u'baz'
+
+
+def test_jobsummaries(hsclient):
+    # add at least one running or pending job to ensure summary is returned
+    hsclient.push_job(TEST_PROJECT_ID, TEST_SPIDER_NAME, state='running')
+
+    def _get_summary():
+        jss = hsclient.projects.jobsummaries()
+        mjss = dict((str(js['project']), js) for js in jss)
+        return mjss.get(TEST_PROJECT_ID)
+
+    summary = apipoll(_get_summary)
+    assert summary is not None
+
+
+def test_timestamp(hsclient):
+    ts1 = hsclient.server_timestamp()
+    ts2 = hsclient.server_timestamp()
+    assert ts1 > 0
+    assert ts1 <= ts2