Update parameters for method Collection.import_bulk

Author: joowani

README.rst

@@ -40,7 +40,7 @@ Features
 
 - Clean, Pythonic interface
 - Lightweight
-- 95%+ ArangoDB REST API coverage
+- High ArangoDB REST API coverage
 
 Compatibility
 =============

arango/client.py

@@ -412,7 +412,6 @@ def log_levels(self):
         """Return the current logging levels.
 
         .. note::
-
             This method is only compatible with ArangoDB version 3.1+ only.
 
         :return: the current logging levels
@@ -438,11 +437,9 @@ def set_log_levels(self, **kwargs):
             )
 
         .. note::
-
             Keys that are not valid logger names are simply ignored.
 
         .. note::
-
             This method is only compatible with ArangoDB version 3.1+ only.
 
         :return: the new logging levels
@@ -611,7 +608,7 @@ def users(self):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         res = self._conn.get('/_api/user')
         if res.status_code not in HTTP_OK:
@@ -633,7 +630,7 @@ def user(self, username):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         res = self._conn.get('/_api/user/{}'.format(username))
         if res.status_code not in HTTP_OK:
@@ -661,7 +658,7 @@ def create_user(self, username, password, active=None, extra=None):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         data = {'user': username, 'passwd': password}
         if active is not None:
@@ -695,7 +692,7 @@ def update_user(self, username, password=None, active=None, extra=None):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         data = {}
         if password is not None:
@@ -734,7 +731,7 @@ def replace_user(self, username, password, active=None, extra=None):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         data = {'user': username, 'passwd': password}
         if active is not None:
@@ -768,7 +765,7 @@ def delete_user(self, username, ignore_missing=False):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         res = self._conn.delete('/_api/user/{user}'.format(user=username))
         if res.status_code in HTTP_OK:
@@ -788,7 +785,7 @@ def user_access(self, username):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         res = self._conn.get('/_api/user/{}/database'.format(username))
         if res.status_code in HTTP_OK:
@@ -808,7 +805,7 @@ def grant_user_access(self, username, database):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         res = self._conn.put(
             '/_api/user/{}/database/{}'.format(username, database),
@@ -831,7 +828,7 @@ def revoke_user_access(self, username, database):
 
         .. note::
             Root privileges (i.e. access to the ``_system`` database) are
-            required to use this method
+            required to use this method.
         """
         res = self._conn.put(
             '/_api/user/{}/database/{}'.format(username, database),

arango/collections/standard.py

@@ -888,36 +888,106 @@ def handler(res):
         return request, handler
 
     @api_method
-    def import_bulk(self, documents, halt_on_error=True, details=True):
+    def import_bulk(self,
+                    documents,
+                    halt_on_error=None,
+                    details=True,
+                    from_prefix=None,
+                    to_prefix=None,
+                    overwrite=None,
+                    on_duplicate=None,
+                    sync=None):
         """Insert multiple documents into the collection.
 
         This is faster than :func:`~arango.collections.Collection.insert_many`
         but does not return as much information. Any ``"_id"`` and ``"_rev"``
         fields in **documents** are ignored.
 
-        :param documents: the list of the new documents to insert
+        :param documents: the list of the new documents to insert in bulk
         :type documents: list
-        :param halt_on_error: halt the whole import on a failure
+        :param halt_on_error: halt the entire import on an error
             (default: ``True``)
         :type halt_on_error: bool
         :param details: if ``True``, the returned result will include an
-            additional list of details error messages (default: ``True``)
+            additional list of detailed error messages (default: ``True``)
         :type details: bool
+        :param from_prefix: the string prefix to prepend to the ``"_from"``
+            field of each edge document inserted. *This only works for edge
+            collections.*
+        :type from_prefix: str | unicode
+        :param to_prefix: the string prefix to prepend to the ``"_to"`` field
+            of each edge document inserted. *This only works for edge
+            collections.*
+        :type to_prefix: str | unicode
+        :param overwrite: if ``True``, all existing documents in the collection
+            are removed prior to the import. Indexes are still preserved.
+        :type overwrite: bool
+        :param on_duplicate: the action to take on unique key constraint
+            violations. Possible values are:
+
+            .. code-block:: none
+
+                "error"   : do not import the new documents and count them as
+                            errors (this is the default)
+
+                "update"  : update the existing documents while preserving any
+                            fields missing in the new ones
+
+                "replace" : replace the existing documents with the new ones
+
+                "ignore"  : do not import the new documents and count them as
+                            ignored, as opposed to counting them as errors
+
+        :type on_duplicate: str | unicode
+        :param sync: wait for the operation to sync to disk
+        :type sync: bool
         :returns: the result of the bulk import
         :rtype: dict
         :raises arango.exceptions.DocumentInsertError: if the documents cannot
             be inserted into the collection
+
+        .. note::
+            Parameters **from_prefix** and **to_prefix** only work for edge
+            collections. When the prefix is prepended, it is followed by a
+            ``"/"`` character. For example, prefix ``"foo"`` prepended to an
+            edge document with ``"_from": "bar"`` will result in a new value
+            ``"_from": "foo/bar"``.
+
+        .. note::
+            Parameter **on_duplicate** actions ``"update"``, ``"replace"``
+            and ``"ignore"`` will work only when **documents** contain the
+            ``"_key"`` fields.
+
+        .. warning::
+            Parameter **on_duplicate** actions  ``"update"`` and ``"replace"``
+            may fail on secondary unique key constraint violations.
         """
+        params = {
+            'type': 'array',
+            'collection': self._name,
+            'complete': halt_on_error,
+            'details': details,
+        }
+        if halt_on_error is not None:
+            params['complete'] = halt_on_error
+        if details is not None:
+            params['details'] = details
+        if from_prefix is not None:
+            params['fromPrefix'] = from_prefix
+        if to_prefix is not None:
+            params['toPrefix'] = to_prefix
+        if overwrite is not None:
+            params['overwrite'] = overwrite
+        if on_duplicate is not None:
+            params['onDuplicate'] = on_duplicate
+        if sync is not None:
+            params['waitForSync'] = sync
+
         request = Request(
             method='post',
             endpoint='/_api/import',
             data=documents,
-            params={
-                'type': 'array',
-                'collection': self._name,
-                'complete': halt_on_error,
-                'details': details
-            }
+            params=params
         )
 
         def handler(res):

arango/version.py

@@ -1 +1 @@
-VERSION = '3.6.0'
+VERSION = '3.7.0'

docs/index.rst

@@ -16,7 +16,7 @@ Features
 
 - Clean, Pythonic interface
 - Lightweight
-- 95%+ ArangoDB REST API coverage
+- High ArangoDB REST API coverage
 
 Compatibility
 =============

tests/test_document.py

@@ -18,6 +18,8 @@
 db = arango_client.create_database(db_name)
 col_name = generate_col_name(db)
 col = db.create_collection(col_name)
+edge_col_name = generate_col_name(db)
+edge_col = db.create_collection(edge_col_name, edge=True)
 geo_index = col.add_geo_index(['coordinates'])
 bad_col_name = generate_col_name(db)
 bad_col = db.collection(bad_col_name)
@@ -30,6 +32,13 @@
 test_docs = [doc1, doc2, doc3, doc4, doc5]
 test_doc_keys = [d['_key'] for d in test_docs]
 
+edge1 = {'_key': '1', '_to': '1', '_from': '2'}
+edge2 = {'_key': '2', '_to': '2', '_from': '3'}
+edge3 = {'_key': '3', '_to': '3', '_from': '4'}
+edge4 = {'_key': '4', '_to': '4', '_from': '5'}
+edge5 = {'_key': '5', '_to': '5', '_from': '1'}
+test_edges = [edge1, edge2, edge3, edge4, edge5]
+
 
 def teardown_module(*_):
     arango_client.delete_database(db_name, ignore_missing=True)
@@ -1000,7 +1009,7 @@ def test_has():
         bad_col.has('1')
 
     with pytest.raises(DocumentInError):
-        '1' in bad_col
+        assert '1' in bad_col
 
 
 def test_get():
@@ -1495,6 +1504,9 @@ def test_import_bulk():
     result = col.import_bulk(test_docs)
     assert result['created'] == 5
     assert result['errors'] == 0
+    assert result['empty'] == 0
+    assert result['updated'] == 0
+    assert result['ignored'] == 0
     assert 'details' in result
     assert len(col) == 5
     for doc in test_docs:
@@ -1505,10 +1517,13 @@ def test_import_bulk():
         assert col[key]['coordinates'] == doc['coordinates']
     col.truncate()
 
-    # Test import bulk without details
-    result = col.import_bulk(test_docs, details=False)
+    # Test import bulk without details and with sync
+    result = col.import_bulk(test_docs, details=False, sync=True)
     assert result['created'] == 5
     assert result['errors'] == 0
+    assert result['empty'] == 0
+    assert result['updated'] == 0
+    assert result['ignored'] == 0
     assert 'details' not in result
     assert len(col) == 5
     for doc in test_docs:
@@ -1528,9 +1543,88 @@ def test_import_bulk():
     result = col.import_bulk([doc2, doc2], halt_on_error=False)
     assert result['created'] == 1
     assert result['errors'] == 1
+    assert result['empty'] == 0
+    assert result['updated'] == 0
+    assert result['ignored'] == 0
     assert len(col) == 1
 
     # Test import bulk in missing collection
     with pytest.raises(DocumentInsertError):
         bad_col.import_bulk([doc3, doc4], halt_on_error=True)
     assert len(col) == 1
+
+    # Test import bulk with overwrite
+    result = col.import_bulk([doc3, doc4], overwrite=True)
+    assert result['created'] == 2
+    assert result['errors'] == 0
+    assert result['empty'] == 0
+    assert result['updated'] == 0
+    assert result['ignored'] == 0
+    assert '1' not in col
+    assert '2' not in col
+    assert '3' in col
+    assert '4' in col
+    col.truncate()
+
+    # Test import bulk to_prefix and from_prefix
+    result = edge_col.import_bulk(
+        test_edges, from_prefix='foo', to_prefix='bar'
+    )
+    assert result['created'] == 5
+    assert result['errors'] == 0
+    assert result['empty'] == 0
+    assert result['updated'] == 0
+    assert result['ignored'] == 0
+    for edge in test_edges:
+        key = edge['_key']
+        assert key in edge_col
+        assert edge_col[key]['_from'] == 'foo/' + edge['_from']
+        assert edge_col[key]['_to'] == 'bar/' + edge['_to']
+    edge_col.truncate()
+
+    # Test import bulk on_duplicate actions
+    old_doc = {'_key': '1', 'foo': '2'}
+    new_doc = {'_key': '1', 'bar': '3'}
+
+    col.insert(old_doc)
+    result = col.import_bulk([new_doc], on_duplicate='error')
+    assert len(col) == 1
+    assert result['created'] == 0
+    assert result['errors'] == 1
+    assert result['empty'] == 0
+    assert result['updated'] == 0
+    assert result['ignored'] == 0
+    assert col['1']['foo'] == '2'
+    assert 'bar' not in col['1']
+
+    result = col.import_bulk([new_doc], on_duplicate='ignore')
+    assert len(col) == 1
+    assert result['created'] == 0
+    assert result['errors'] == 0
+    assert result['empty'] == 0
+    assert result['updated'] == 0
+    assert result['ignored'] == 1
+    assert col['1']['foo'] == '2'
+    assert 'bar' not in col['1']
+
+    result = col.import_bulk([new_doc], on_duplicate='update')
+    assert len(col) == 1
+    assert result['created'] == 0
+    assert result['errors'] == 0
+    assert result['empty'] == 0
+    assert result['updated'] == 1
+    assert result['ignored'] == 0
+    assert col['1']['foo'] == '2'
+    assert col['1']['bar'] == '3'
+
+    col.truncate()
+    col.insert(old_doc)
+    result = col.import_bulk([new_doc], on_duplicate='replace')
+    assert len(col) == 1
+    assert result['created'] == 0
+    assert result['errors'] == 0
+    assert result['empty'] == 0
+    assert result['updated'] == 1
+    assert result['ignored'] == 0
+    assert 'foo' not in col['1']
+    assert col['1']['bar'] == '3'