Merge branch 'main' into DE-756

Author: aMahanna

README.md

@@ -52,8 +52,8 @@ db = client.db("test", username="root", password="passwd")
 # Create a new collection named "students".
 students = db.create_collection("students")
 
-# Add a hash index to the collection.
-students.add_hash_index(fields=["name"], unique=True)
+# Add a persistent index to the collection.
+students.add_persistent_index(fields=["name"], unique=True)
 
 # Insert new documents into the collection.
 students.insert({"name": "jane", "age": 39})

arango/collection.py

@@ -1293,6 +1293,13 @@ def add_hash_index(
     ) -> Result[Json]:
         """Create a new hash index.
 
+        .. warning::
+
+            The index types `hash` and `skiplist` are aliases for the persistent
+            index type and should no longer be used to create new indexes. The
+            aliases will be removed in a future version. Use
+            :func:`arango.collection.Collection.add_persistent_index` instead.
+
         :param fields: Document fields to index.
         :type fields: [str]
         :param unique: Whether the index is unique.
@@ -1337,6 +1344,13 @@ def add_skiplist_index(
     ) -> Result[Json]:
         """Create a new skiplist index.
 
+        .. warning::
+
+            The index types `hash` and `skiplist` are aliases for the persistent
+            index type and should no longer be used to create new indexes. The
+            aliases will be removed in a future version. Use
+            :func:`arango.collection.Collection.add_persistent_index` instead.
+
         :param fields: Document fields to index.
         :type fields: [str]
         :param unique: Whether the index is unique.

arango/database.py

@@ -42,6 +42,8 @@
     PermissionListError,
     PermissionResetError,
     PermissionUpdateError,
+    ServerAvailableOptionsGetError,
+    ServerCurrentOptionsGetError,
     ServerDetailsError,
     ServerEchoError,
     ServerEncryptionError,
@@ -1118,6 +1120,58 @@ def response_handler(resp: Response) -> Json:
 
         return self._execute(request, response_handler)
 
+    def options(self) -> Result[Json]:
+        """Return the currently-set server options (ArangoDB 3.12+)
+
+        As this API may reveal sensitive data about the deployment, it can only
+        be accessed from inside the _system database. In addition, there is a
+        policy control startup option --server.options-api that determines if and
+        to whom the API is made available. This option can have the following
+        values:
+        - disabled: API is disabled.
+        - jwt: API can only be accessed via superuser JWT.
+        - admin: API can be accessed by admin users in the _system database only.
+        - public: everyone with access to _system database can access the API.
+
+        :return: Server options.
+        :rtype: dict
+        """
+        request = Request(method="get", endpoint="/_admin/options")
+
+        def response_handler(resp: Response) -> Json:
+            if resp.is_success:
+                result: Json = resp.body
+                return result
+            raise ServerCurrentOptionsGetError(resp, request)
+
+        return self._execute(request, response_handler)
+
+    def options_available(self) -> Result[Json]:
+        """Return a description of all available server options (ArangoDB 3.12+)
+
+        As this API may reveal sensitive data about the deployment, it can only
+        be accessed from inside the _system database. In addition, there is a
+        policy control startup option --server.options-api that determines if and
+        to whom the API is made available. This option can have the following
+        values:
+        - disabled: API is disabled.
+        - jwt: API can only be accessed via superuser JWT.
+        - admin: API can be accessed by admin users in the _system database only.
+        - public: everyone with access to _system database can access the options API.
+
+        :return: Server options.
+        :rtype: dict
+        """
+        request = Request(method="get", endpoint="/_admin/options-description")
+
+        def response_handler(resp: Response) -> Json:
+            if resp.is_success:
+                result: Json = resp.body
+                return result
+            raise ServerAvailableOptionsGetError(resp, request)
+
+        return self._execute(request, response_handler)
+
     #######################
     # Database Management #
     #######################

arango/exceptions.py

@@ -722,6 +722,14 @@ class ServerEncryptionError(ArangoServerError):
     """Failed to reload user-defined encryption keys."""
 
 
+class ServerCurrentOptionsGetError(ArangoServerError):
+    """Failed to retrieve currently-set server options."""
+
+
+class ServerAvailableOptionsGetError(ArangoServerError):
+    """Failed to retrieve available server options."""
+
+
 class ServerExecuteError(ArangoServerError):
     """Failed to execute raw JavaScript command."""
 

arango/formatter.py

@@ -104,6 +104,10 @@ def format_index(body: Json) -> Json:
     if "fieldValueTypes" in body:
         result["field_value_types"] = body["fieldValueTypes"]
 
+    # Introduced in 3.12 EE
+    if "optimizeTopK" in body:
+        result["optimizeTopK"] = body["optimizeTopK"]
+
     return verify_format(body, result)
 
 

docs/indexes.rst

@@ -27,15 +27,15 @@ on fields ``_from`` and ``_to``. For more information on indexes, refer to
     # List the indexes in the collection.
     cities.indexes()
 
-    # Add a new hash index on document fields "continent" and "country".
-    index = cities.add_hash_index(fields=['continent', 'country'], unique=True)
+    # Add a new persistent index on document fields "continent" and "country".
+    index = cities.add_persistent_index(fields=['continent', 'country'], unique=True)
 
     # Add new fulltext indexes on fields "continent" and "country".
     index = cities.add_fulltext_index(fields=['continent'])
     index = cities.add_fulltext_index(fields=['country'])
 
-    # Add a new skiplist index on field 'population'.
-    index = cities.add_skiplist_index(fields=['population'], sparse=False)
+    # Add a new persistent index on field 'population'.
+    index = cities.add_persistent_index(fields=['population'], sparse=False)
 
     # Add a new geo-spatial index on field 'coordinates'.
     index = cities.add_geo_index(fields=['coordinates'])
@@ -47,7 +47,7 @@ on fields ``_from`` and ``_to``. For more information on indexes, refer to
     index = cities.add_ttl_index(fields=['currency'], expiry_time=200)
 
     # Indexes may be added with a name that can be referred to in AQL queries.
-    index = cities.add_hash_index(fields=['country'], name='my_hash_index')
+    index = cities.add_persistent_index(fields=['country'], name='my_persistent_index')
 
     # Delete the last index from the collection.
     cities.delete_index(index['id'])

starter.sh

@@ -32,11 +32,12 @@ else
     exit 1
 fi
 
-conf_file=""
-if [[ "${version%.*}" == "3.10" ]]; then
-    conf_file="${setup}-3.10"
+if [ "$version" == "latest" ]; then
+    conf_file="${setup}-3.12"
+elif [[ "$version" == *.*.* ]]; then
+    conf_file="${setup}-${version%.*}"
 else
-    conf_file="${setup}"
+    conf_file="${setup}-${version}"
 fi
 
 docker run -d \

tests/static/cluster.conf renamed to tests/static/cluster-3.11.conf

@@ -0,0 +1,15 @@
+[starter]
+mode = cluster
+local = true
+address = 0.0.0.0
+port = 8528
+
+[auth]
+jwt-secret = /tests/static/keyfile
+
+[args]
+all.database.password = passwd
+all.database.extended-names = true
+all.log.api-enabled = true
+all.javascript.allow-admin-execute = true
+all.server.options-api = admin

tests/static/cluster-3.12.conf

@@ -0,0 +1,13 @@
+[starter]
+mode = single
+address = 0.0.0.0
+port = 8528
+
+[auth]
+jwt-secret = /tests/static/keyfile
+
+[args]
+all.database.password = passwd
+all.database.extended-names = true
+all.javascript.allow-admin-execute = true
+all.server.options-api = admin

tests/static/single.conf renamed to tests/static/single-3.11.conf

@@ -1,3 +1,5 @@
+from packaging import version
+
 from arango.exceptions import (
     AnalyzerCreateError,
     AnalyzerDeleteError,
@@ -7,18 +9,28 @@
 from tests.helpers import assert_raises, generate_analyzer_name
 
 
-def test_analyzer_management(db, bad_db, cluster, enterprise):
+def test_analyzer_management(db, bad_db, cluster, enterprise, db_version):
     analyzer_name = generate_analyzer_name()
     full_analyzer_name = db.name + "::" + analyzer_name
     bad_analyzer_name = generate_analyzer_name()
 
-    # Test create analyzer
+    # Test create identity analyzer
     result = db.create_analyzer(analyzer_name, "identity", {})
     assert result["name"] == full_analyzer_name
     assert result["type"] == "identity"
     assert result["properties"] == {}
     assert result["features"] == []
 
+    # Test create delimiter analyzer
+    result = db.create_analyzer(
+        name=generate_analyzer_name(),
+        analyzer_type="delimiter",
+        properties={"delimiter": ","},
+    )
+    assert result["type"] == "delimiter"
+    assert result["properties"] == {"delimiter": ","}
+    assert result["features"] == []
+
     # Test create duplicate with bad database
     with assert_raises(AnalyzerCreateError) as err:
         bad_db.create_analyzer(analyzer_name, "identity", {}, [])
@@ -69,3 +81,22 @@ def test_analyzer_management(db, bad_db, cluster, enterprise):
             "format": "latLngDouble",
         }
         assert db.delete_analyzer(analyzer_name)
+
+    # Test create delimieter analyzer with multiple delimiters
+    if db_version >= version.parse("3.12.0"):
+        result = db.create_analyzer(
+            name=generate_analyzer_name(),
+            analyzer_type="multi_delimiter",
+            properties={"delimiters": [",", "."]},
+        )
+
+        assert result["type"] == "multi_delimiter"
+        assert result["properties"] == {"delimiters": [",", "."]}
+        assert result["features"] == []
+
+    if db_version >= version.parse("3.12.0"):
+        analyzer_name = generate_analyzer_name()
+        result = db.create_analyzer(analyzer_name, "wildcard", {"ngramSize": 4})
+        assert result["type"] == "wildcard"
+        assert result["features"] == []
+        assert result["properties"] == {"ngramSize": 4}

tests/static/single-3.12.conf

@@ -209,7 +209,6 @@ def test_collection_management(db, bad_db, cluster):
         shard_count=2,
         shard_fields=["test_attr:"],
         replication_factor=1,
-        shard_like="",
         sync_replication=False,
         enforce_replication_factor=False,
         sharding_strategy="community-compat",
@@ -236,6 +235,14 @@ def test_collection_management(db, bad_db, cluster):
     assert properties["computedValues"] == computed_values
     col.configure(computed_values=[])
 
+    if cluster:
+        # Create distribute-shards-like collection
+        shard_like_name = col_name + "_shards_like"
+        shard_like_col = db.create_collection(name=shard_like_name, shard_like=col_name)
+        assert shard_like_col.properties()["shard_like"] == col_name
+        assert db.has_collection(shard_like_name) is True
+        assert db.delete_collection(shard_like_name, system=False) is True
+
     # Test create duplicate collection
     with assert_raises(CollectionCreateError) as err:
         db.create_collection(col_name)

tests/test_analyzer.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 
 import pytest
+from packaging import version
 
 from arango.aql import AQL
 from arango.backup import Backup
@@ -438,3 +439,12 @@ def test_license(sys_db, enterprise):
         assert license == {"license": "none"}
         with pytest.raises(ServerLicenseSetError):
             sys_db.set_license("abc")
+
+
+def test_options(sys_db, db_version):
+    # Skip if below 3.12
+    if db_version < version.parse("3.12.0"):
+        pytest.skip("Database options require ArangoDB 3.12+")
+
+    assert sys_db.options()
+    assert sys_db.options_available()

tests/test_collection.py

@@ -97,7 +97,7 @@ def test_transaction_commit(db, col, docs):
         sync=True,
         allow_implicit=False,
         lock_timeout=1000,
-        max_size=10000,
+        max_size=1024 * 1024,  # 1MB
     )
     txn_col = txn_db.collection(col.name)
 
@@ -126,7 +126,7 @@ def test_transaction_fetch_existing(db, col, docs):
         sync=True,
         allow_implicit=False,
         lock_timeout=1000,
-        max_size=10000,
+        max_size=1024 * 1024,  # 1MB
     )
     txn_col = original_txn.collection(col.name)