From 8a0f6b325daa2a99ab8ba158d6bbe3e40ebd0fc1 Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 14 Jul 2023 17:33:44 +0300 Subject: [PATCH 1/3] Adding refill_index_caches parameter --- arango/collection.py | 74 ++++++++++++++++++++++++++++++++++++++++-- tests/test_document.py | 43 ++++++++++++++++++++---- 2 files changed, 109 insertions(+), 8 deletions(-) diff --git a/arango/collection.py b/arango/collection.py index 529ff7ec..f3e91415 100644 --- a/arango/collection.py +++ b/arango/collection.py @@ -1468,6 +1468,7 @@ def insert_many( overwrite_mode: Optional[str] = None, keep_none: Optional[bool] = None, merge: Optional[bool] = None, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Insert multiple documents. @@ -1518,8 +1519,10 @@ def insert_many( instead of the new one overwriting the old one. Applies only when **overwrite_mode** is set to "update" (update-insert). :type merge: bool | None - :return: Document metadata (e.g. document key, revision) or True if - parameter **silent** was set to True. + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: List of document metadata (e.g. document keys, revisions) and any exception, or True if parameter **silent** was set to True. :rtype: [dict | ArangoServerError] | bool @@ -1543,6 +1546,10 @@ def insert_many( if merge is not None: params["mergeObjects"] = merge + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="post", endpoint=f"/_api/document/{self.name}", @@ -1582,6 +1589,7 @@ def update_many( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Update multiple documents. @@ -1624,6 +1632,10 @@ def update_many( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: List of document metadata (e.g. document keys, revisions) and any exceptions, or True if parameter **silent** was set to True. :rtype: [dict | ArangoError] | bool @@ -1641,6 +1653,10 @@ def update_many( if sync is not None: params["waitForSync"] = sync + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + documents = [self._ensure_key_in_body(doc) for doc in documents] request = Request( @@ -1753,6 +1769,7 @@ def replace_many( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Replace multiple documents. @@ -1790,6 +1807,10 @@ def replace_many( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: List of document metadata (e.g. document keys, revisions) and any exceptions, or True if parameter **silent** was set to True. :rtype: [dict | ArangoServerError] | bool @@ -1805,6 +1826,10 @@ def replace_many( if sync is not None: params["waitForSync"] = sync + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + documents = [self._ensure_key_in_body(doc) for doc in documents] request = Request( @@ -1901,6 +1926,7 @@ def delete_many( check_rev: bool = True, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, List[Union[Json, ArangoServerError]]]]: """Delete multiple documents. @@ -1933,6 +1959,10 @@ def delete_many( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: List of document metadata (e.g. document keys, revisions) and any exceptions, or True if parameter **silent** was set to True. :rtype: [dict | ArangoServerError] | bool @@ -1947,6 +1977,10 @@ def delete_many( if sync is not None: params["waitForSync"] = sync + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillCaches"] = refill_index_caches + documents = [ self._ensure_key_in_body(doc) if isinstance(doc, dict) else doc for doc in documents @@ -2229,6 +2263,7 @@ def insert( overwrite_mode: Optional[str] = None, keep_none: Optional[bool] = None, merge: Optional[bool] = None, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, Json]]: """Insert a new document. @@ -2263,6 +2298,10 @@ def insert( instead of the new one overwriting the old one. Applies only when **overwrite_mode** is set to "update" (update-insert). :type merge: bool | None + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: Document metadata (e.g. document key, revision) or True if parameter **silent** was set to True. :rtype: bool | dict @@ -2285,6 +2324,10 @@ def insert( if merge is not None: params["mergeObjects"] = merge + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="post", endpoint=f"/_api/document/{self.name}", @@ -2317,6 +2360,7 @@ def update( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, Json]]: """Update a document. @@ -2343,6 +2387,10 @@ def update( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: Document metadata (e.g. document key, revision) or True if parameter **silent** was set to True. :rtype: bool | dict @@ -2361,6 +2409,10 @@ def update( if sync is not None: params["waitForSync"] = sync + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="patch", endpoint=f"/_api/document/{self._extract_id(document)}", @@ -2391,6 +2443,7 @@ def replace( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, Json]]: """Replace a document. @@ -2412,6 +2465,10 @@ def replace( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document insertions affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: Document metadata (e.g. document key, revision) or True if parameter **silent** was set to True. :rtype: bool | dict @@ -2428,6 +2485,10 @@ def replace( if sync is not None: params["waitForSync"] = sync + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="put", endpoint=f"/_api/document/{self._extract_id(document)}", @@ -2461,6 +2522,7 @@ def delete( return_old: bool = False, sync: Optional[bool] = None, silent: bool = False, + refill_index_caches: Optional[bool] = None, ) -> Result[Union[bool, Json]]: """Delete a document. @@ -2485,6 +2547,10 @@ def delete( :param silent: If set to True, no document metadata is returned. This can be used to save resources. :type silent: bool + :param refill_index_caches: Whether to add new entries to in-memory + index caches if document operations affect the edge index or + cache-enabled persistent indexes. + :type refill_index_caches: bool | None :return: Document metadata (e.g. document key, revision), or True if parameter **silent** was set to True, or False if document was not found and **ignore_missing** was set to True (does not apply in @@ -2504,6 +2570,10 @@ def delete( if sync is not None: params["waitForSync"] = sync + # New in 3.9.6 and 3.10.2 + if refill_index_caches is not None: + params["refillIndexCaches"] = refill_index_caches + request = Request( method="delete", endpoint=f"/_api/document/{handle}", diff --git a/tests/test_document.py b/tests/test_document.py index d310b45e..a527ba66 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -140,6 +140,13 @@ def test_document_insert(col, docs): ) assert err.value.error_code == 1210 + # Test insert with cache refilling + empty_collection(col) + doc = docs[0] + assert col.insert(doc, refill_index_caches=True) + assert col[doc["_key"]]["_key"] == doc["_key"] + assert col[doc["_key"]]["val"] == doc["val"] + def test_document_insert_many(col, bad_col, docs): # Test insert_many with default options @@ -291,6 +298,13 @@ def test_document_insert_many(col, bad_col, docs): assert results[0].error_code == 1210 assert "new" in results[1] + # Test insert with cache refilling + empty_collection(col) + assert col.insert_many(docs, refill_index_caches=True) + for doc in docs: + assert col[doc["_key"]]["_key"] == doc["_key"] + assert col[doc["_key"]]["val"] == doc["val"] + def test_document_update(col, docs): doc = docs[0] @@ -422,6 +436,11 @@ def test_document_update(col, docs): assert col.update(doc, silent=True) is True assert col[doc["_key"]]["val"] == 8 + # Test update with cache refilling + doc["val"] = 9 + assert col.update(doc, refill_index_caches=True, check_rev=False) + assert col[doc["_key"]]["val"] == 9 + def test_document_update_many(col, bad_col, docs): col.insert_many(docs) @@ -600,6 +619,13 @@ def test_document_update_many(col, bad_col, docs): for doc in docs: assert col[doc["_key"]]["val"] == 8 + # Test update_many with cache refilling + for doc in docs: + doc["val"] = 9 + assert col.update_many(docs, refill_index_caches=True, check_rev=False) + for doc in docs: + assert col[doc["_key"]]["val"] == 9 + # Test update_many with bad documents with assert_raises(DocumentParseError) as err: bad_col.update_many([{}]) @@ -734,6 +760,11 @@ def test_document_replace(col, docs): assert col.replace(doc, silent=True) is True assert col[doc["_key"]]["val"] == 8 + # Test replace with cache refilling + doc["val"] = 9 + assert col.replace(doc, refill_index_caches=True, check_rev=False) + assert col[doc["_key"]]["val"] == 9 + def test_document_replace_many(col, bad_col, docs): col.insert_many(docs) @@ -817,8 +848,8 @@ def test_document_replace_many(col, bad_col, docs): assert "foo" not in doc assert doc["baz"] == 4 - # Test replace_many with check_rev set to False - results = col.replace_many(docs, check_rev=False) + # Test replace_many with check_rev set to False and cache refilling + results = col.replace_many(docs, check_rev=False, refill_index_caches=True) for result, doc in zip(results, docs): doc_key = doc["_key"] assert result["_id"] == f"{col.name}/{doc_key}" @@ -965,9 +996,9 @@ def test_document_delete(col, docs): if col.context != "transaction": assert col.delete(bad_key, ignore_missing=True) is False - # Test delete (document) with silent set to True + # Test delete (document) with silent set to True and cache refilling doc = docs[5] - assert col.delete(doc, silent=True) is True + assert col.delete(doc, silent=True, refill_index_caches=True) is True assert doc["_key"] not in col assert len(col) == 1 @@ -1029,9 +1060,9 @@ def test_document_delete_many(col, bad_col, docs): old_revs[doc_key] = result["_rev"] assert len(col) == 0 - # Test delete_many with silent set to True + # Test delete_many with silent set to True and cache refilling col.import_bulk(docs) - assert col.delete_many(docs, silent=True) is True + assert col.delete_many(docs, silent=True, refill_index_caches=True) is True assert len(col) == 0 # Test delete_many (documents) with check_rev set to True From f4c64ff79b48d82461950cbfaa6e3352858cb56f Mon Sep 17 00:00:00 2001 From: Alex Petenchea Date: Fri, 14 Jul 2023 17:36:28 +0300 Subject: [PATCH 2/3] Adding changelog entry --- CHANGELOG.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2d62183..fda147f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,25 @@ main ---- +* Added new per-operation option `refillIndexCache` to write operations: + + - single-document write operations (insert, replace, update, delete) + - multi-document write operations (insert_many, replace_many, update_many, delete_many) + + If the option is set to `True`, new entries are added to in-memory index caches if + document operations affect the edge index or cache-enabled persistent indexes. Every + currently running transaction will keep track of which in-memory index cache entries + were invalidated by the transaction, and will try to (re-)fill them later. + + Example: + ```python + collection.insert({"foo": "bar"}, refillIndexCaches=True) + db.aql.execute("INSERT {foo: bar} INTO collection OPTIONS { refillIndexCaches: true }") + ``` + +7.5.9 +----- + * Added cache and primaryKeyCache parameters to the inverted index API. * Added allow_retry query parameter, making it possible to retry fetching the latest batch from a cursor. From 7c2bc15f6317bd6400dbca0498a276bd9cd94d97 Mon Sep 17 00:00:00 2001 From: Anthony Mahanna Date: Mon, 24 Jul 2023 12:03:15 -0400 Subject: [PATCH 3/3] update: comments --- arango/collection.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/arango/collection.py b/arango/collection.py index f3e91415..015e6607 100644 --- a/arango/collection.py +++ b/arango/collection.py @@ -1546,7 +1546,7 @@ def insert_many( if merge is not None: params["mergeObjects"] = merge - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillIndexCaches"] = refill_index_caches @@ -1653,7 +1653,7 @@ def update_many( if sync is not None: params["waitForSync"] = sync - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillIndexCaches"] = refill_index_caches @@ -1826,7 +1826,7 @@ def replace_many( if sync is not None: params["waitForSync"] = sync - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillIndexCaches"] = refill_index_caches @@ -1977,7 +1977,7 @@ def delete_many( if sync is not None: params["waitForSync"] = sync - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillCaches"] = refill_index_caches @@ -2324,7 +2324,7 @@ def insert( if merge is not None: params["mergeObjects"] = merge - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillIndexCaches"] = refill_index_caches @@ -2409,7 +2409,7 @@ def update( if sync is not None: params["waitForSync"] = sync - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillIndexCaches"] = refill_index_caches @@ -2485,7 +2485,7 @@ def replace( if sync is not None: params["waitForSync"] = sync - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillIndexCaches"] = refill_index_caches @@ -2570,7 +2570,7 @@ def delete( if sync is not None: params["waitForSync"] = sync - # New in 3.9.6 and 3.10.2 + # New in ArangoDB 3.9.6 and 3.10.2 if refill_index_caches is not None: params["refillIndexCaches"] = refill_index_caches