Merge pull request #247 from ArangoDB-Community/feature/de-593-support-latest-arangodb-versions

[DE-593] Support latest ArangoDB version

Author: CryptoNinjaGeek

.github/workflows/build.yaml

@@ -13,11 +13,11 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11.1"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - name: Checkout repository
@@ -28,14 +28,8 @@ jobs:
 
       - name: Create ArangoDB Docker container
         run: >
-          docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd
-          arangodb/arangodb:3.7.7 --server.jwt-secret-keyfile=/tmp/keyfile
-
-      - name: Copy Foxx service zip into ArangoDB Docker container
-        run: docker cp tests/static/service.zip arango:/tmp/service.zip
-
-      - name: Copy keyfile into ArangoDB Docker container
-        run: docker cp tests/static/keyfile arango:/tmp/keyfile
+          docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd -v "$(pwd)/tests/static/":/tests/static
+          arangodb/arangodb:3.10.6 --server.jwt-secret-keyfile=/tests/static/keyfile
 
       - name: Start ArangoDB Docker container
         run: docker start arango

.gitignore

@@ -121,3 +121,6 @@ node_modules/
 
 # setuptools_scm
 arango/version.py
+
+# test results
+*_results.txt

CONTRIBUTING.md

@@ -13,6 +13,12 @@ Run unit tests with coverage:
 py.test --cov=arango --cov-report=html  # Open htmlcov/index.html in your browser
 ```
 
+For a more comprehensive test suite, run:
+
+```shell
+./tester.sh  # Requires docker
+```
+
 Build and test documentation:
 
 ```shell

arango/backup.py

@@ -38,7 +38,7 @@ def get(self, backup_id: Optional[str] = None) -> Result[Json]:
         request = Request(
             method="post",
             endpoint="/_admin/backup/list",
-            data={} if backup_id is None else {"id": backup_id},
+            data=None if backup_id is None else {"id": backup_id},
         )
 
         def response_handler(resp: Response) -> Json:

arango/collection.py

@@ -863,7 +863,7 @@ def find_in_box(
         limit: Optional[int] = None,
         index: Optional[str] = None,
     ) -> Result[Cursor]:
-        """Return all documents in an rectangular area.
+        """Return all documents in a rectangular area.
 
         :param latitude1: First latitude.
         :type latitude1: int | float

arango/formatter.py

@@ -65,6 +65,8 @@ def format_index(body: Json) -> Json:
         result["cacheEnabled"] = body["cacheEnabled"]
     if "legacyPolygons" in body:
         result["legacyPolygons"] = body["legacyPolygons"]
+    if "estimates" in body:
+        result["estimates"] = body["estimates"]
 
     return verify_format(body, result)
 
@@ -227,6 +229,8 @@ def format_collection(body: Json) -> Json:
             }
             for cv in body["computedValues"]
         ]
+    if "internalValidatorType" in body:
+        result["internal_validator_type"] = body["internalValidatorType"]
 
     return verify_format(body, result)
 
@@ -393,6 +397,10 @@ def format_server_status(body: Json) -> Json:
     """
     result: Json = {}
 
+    if "agency" in body:
+        result["agency"] = body["agency"]
+    if "coordinator" in body:
+        result["coordinator"] = body["coordinator"]
     if "foxxApi" in body:
         result["foxx_api"] = body["foxxApi"]
     if "host" in body:
@@ -985,6 +993,9 @@ def format_backup(body: Json) -> Json:
     if "nrPiecesPresent" in body:
         result["pieces_present"] = body["nrPiecesPresent"]
 
+    if "countIncludesFilesOnly" in body:
+        result["count_includes_files_only"] = body["countIncludesFilesOnly"]
+
     return verify_format(body, result)
 
 
@@ -1135,6 +1146,14 @@ def format_pregel_job_data(body: Json) -> Json:
     # The detail element was introduced in 3.10
     if "detail" in body:
         result["detail"] = body["detail"]
+    if "database" in body:
+        result["database"] = body["database"]
+    if "masterContext" in body:
+        result["master_context"] = body["masterContext"]
+    if "parallelism" in body:
+        result["parallelism"] = body["parallelism"]
+    if "useMemoryMaps" in body:
+        result["use_memory_maps"] = body["useMemoryMaps"]
 
     return verify_format(body, result)
 
@@ -1177,12 +1196,18 @@ def format_graph_properties(body: Json) -> Json:
     }
     if "isSmart" in body:
         result["smart"] = body["isSmart"]
+    if "isSatellite" in body:
+        result["is_satellite"] = body["isSatellite"]
     if "smartGraphAttribute" in body:
         result["smart_field"] = body["smartGraphAttribute"]
     if "numberOfShards" in body:
         result["shard_count"] = body["numberOfShards"]
     if "replicationFactor" in body:
         result["replication_factor"] = body["replicationFactor"]
+    if "minReplicationFactor" in body:
+        result["min_replication_factor"] = body["minReplicationFactor"]
+    if "writeConcern" in body:
+        result["write_concern"] = body["writeConcern"]
 
     return verify_format(body, result)
 

docs/foxx.rst

@@ -31,7 +31,7 @@ information, refer to `ArangoDB manual`_.
     # Create a service using source on server.
     foxx.create_service(
         mount=service_mount,
-        source='/tmp/service.zip',
+        source='/tests/static/service.zip',
         config={},
         dependencies={},
         development=True,
@@ -42,7 +42,7 @@ information, refer to `ArangoDB manual`_.
     # Update (upgrade) a service.
     service = db.foxx.update_service(
         mount=service_mount,
-        source='/tmp/service.zip',
+        source='/tests/static/service.zip',
         config={},
         dependencies={},
         teardown=True,
@@ -53,7 +53,7 @@ information, refer to `ArangoDB manual`_.
     # Replace (overwrite) a service.
     service = db.foxx.replace_service(
         mount=service_mount,
-        source='/tmp/service.zip',
+        source='/tests/static/service.zip',
         config={},
         dependencies={},
         teardown=True,

tester.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# Tests python-arango driver against a local ArangoDB single server or cluster setup.
+# 1. Starts a local ArangoDB server or cluster (community).
+# 2. Runs the python-arango tests for the community edition.
+# 3. Starts a local ArangoDB server or cluster (enterprise).
+# 4. Runs all python-arango tests, including enterprise tests.
+
+# Usage:
+#   ./start.sh [all|single|cluster] [all|community|enterprise] [version] ["notest"]
+
+setup="${1:-all}"
+if [[ "$setup" != "all" && "$setup" != "single" && "$setup" != "cluster" ]]; then
+    echo "Invalid argument. Please provide either 'all', 'single' or 'cluster'."
+    exit 1
+fi
+
+tests="${2:-all}"
+if [[ "$tests" != "all" && "$tests" != "community" && "$tests" != "enterprise" ]]; then
+    echo "Invalid argument. Please provide either 'all', 'community', or 'enterprise'."
+    exit 1
+fi
+
+version="${3:-3.10.6}"
+
+if [[ -n "$4" && "$4" != "notest" ]]; then
+    echo "Invalid argument. Use 'notest' to only start the docker container, without running the tests."
+    exit 1
+fi
+mode="${4:-test}"
+
+if [ "$setup" == "all" ] || [ "$setup" == "single" ]; then
+  if [ "$tests" == "all" ] || [ "$tests" == "community" ]; then
+    echo "Starting single server community setup..."
+    docker run -d --rm \
+      --name arango \
+      -p 8529:8529 \
+      -v "$(pwd)/tests/static/":/tests/static \
+      -v /tmp:/tmp \
+      arangodb/arangodb:"$version" \
+      /bin/sh -c "arangodb --configuration=/tests/static/single.conf"
+
+    if [[ "$mode" == "notest" ]]; then
+      exit 0
+    fi
+
+    echo "Running python-arango tests for single server community setup..."
+    sleep 3
+    py.test --complete --cov=arango --cov-report=html | tee single_community_results.txt
+    echo "Stopping single server community setup..."
+    docker stop arango
+    docker wait arango
+    sleep 3
+  fi
+
+  if [ "$tests" == "all" ] || [ "$tests" == "enterprise" ]; then
+    echo "Starting single server enterprise setup..."
+    docker run -d --rm \
+      --name arango \
+      -p 8529:8529 \
+      -v "$(pwd)/tests/static/":/tests/static \
+      -v /tmp:/tmp \
+      arangodb/enterprise:"$version" \
+      /bin/sh -c "arangodb --configuration=/tests/static/single.conf"
+
+    if [[ "$mode" == "notest" ]]; then
+      exit 0
+    fi
+
+    echo "Running python-arango tests for single server enterprise setup..."
+    sleep 3
+    py.test --complete --enterprise --cov=arango --cov-report=html --cov-append | tee single_enterprise_results.txt
+    echo "Stopping single server enterprise setup..."
+    docker stop arango
+    docker wait arango
+    sleep 3
+  fi
+fi
+
+if [ "$setup" == "all" ] || [ "$setup" == "cluster" ]; then
+  if [ "$tests" == "all" ] || [ "$tests" == "community" ]; then
+    echo "Starting community cluster setup..."
+    docker run -d --rm \
+      --name arango \
+      -p 8529:8529 \
+      -v "$(pwd)/tests/static/":/tests/static \
+      -v /tmp:/tmp \
+      arangodb/arangodb:"$version" \
+      /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf"
+
+    if [[ "$mode" == "notest" ]]; then
+      exit 0
+    fi
+
+    echo "Running python-arango tests for community cluster setup..."
+    sleep 15
+    py.test --cluster --complete --cov=arango --cov-report=html | tee cluster_community_results.txt
+    echo "Stopping community cluster setup..."
+    docker stop arango
+    docker wait arango
+    sleep 3
+  fi
+
+  if [ "$tests" == "all" ] || [ "$tests" == "enterprise" ]; then
+    echo "Starting enterprise cluster setup..."
+    docker run -d --rm \
+      --name arango \
+      -p 8529:8529 \
+      -v "$(pwd)/tests/static/":/tests/static \
+      -v /tmp:/tmp \
+      arangodb/enterprise:"$version" \
+      /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf"
+
+    if [[ "$mode" == "notest" ]]; then
+      exit 0
+    fi
+
+    echo "Running python-arango tests for enterprise cluster setup..."
+    sleep 15
+    py.test --cluster --enterprise --complete --cov=arango --cov-report=html | tee cluster_enterprise_results.txt
+    echo "Stopping enterprise cluster setup..."
+    docker stop arango
+    docker wait arango
+  fi
+fi

tests/conftest.py

@@ -5,11 +5,7 @@
 from arango import ArangoClient, formatter
 from arango.database import StandardDatabase
 from arango.typings import Json
-from tests.executors import (
-    TestAsyncApiExecutor,
-    TestBatchExecutor,
-    TestTransactionApiExecutor,
-)
+from tests.executors import TestAsyncApiExecutor, TestTransactionApiExecutor
 from tests.helpers import (
     empty_collection,
     generate_col_name,
@@ -210,13 +206,15 @@ def pytest_generate_tests(metafunc):
             bad_async_db._executor = TestAsyncApiExecutor(bad_conn)
             bad_dbs.append(bad_async_db)
 
-            # Add test batch databases
+            # Skip test batch databases, as they are deprecated.
+            """
             tst_batch_db = StandardDatabase(tst_conn)
             tst_batch_db._executor = TestBatchExecutor(tst_conn)
             tst_dbs.append(tst_batch_db)
             bad_batch_bdb = StandardDatabase(bad_conn)
             bad_batch_bdb._executor = TestBatchExecutor(bad_conn)
             bad_dbs.append(bad_batch_bdb)
+            """
 
     if "db" in metafunc.fixturenames and "bad_db" in metafunc.fixturenames:
         metafunc.parametrize("db,bad_db", zip(tst_dbs, bad_dbs))
@@ -234,6 +232,12 @@ def mock_verify_format(body, result):
         body.pop("error", None)
         body.pop("code", None)
         result.pop("edge", None)
+
+        # Remove all None values
+        # Sometimes they are expected to be excluded from the body (see computedValues)
+        result = {k: v for k, v in result.items() if v is not None}
+        body = {k: v for k, v in body.items() if v is not None}
+
         if len(body) != len(result):
             before = sorted(body, key=lambda x: x.strip("_"))
             after = sorted(result, key=lambda x: x.strip("_"))

tests/static/cluster.conf

@@ -0,0 +1,11 @@
+[starter]
+mode = cluster
+local = true
+address = 0.0.0.0
+
+[auth]
+jwt-secret = /tests/static/keyfile
+
+[args]
+all.database.password = passwd
+all.log.api-enabled = true

tests/static/single.conf

@@ -0,0 +1,10 @@
+[starter]
+mode = single
+address = 0.0.0.0
+port = 8528
+
+[auth]
+jwt-secret = /tests/static/keyfile
+
+[args]
+all.database.password = passwd