diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 969d3840..44b14258 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -13,11 +13,11 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11.1"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - name: Checkout repository @@ -28,14 +28,8 @@ jobs: - name: Create ArangoDB Docker container run: > - docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd - arangodb/arangodb:3.7.7 --server.jwt-secret-keyfile=/tmp/keyfile - - - name: Copy Foxx service zip into ArangoDB Docker container - run: docker cp tests/static/service.zip arango:/tmp/service.zip - - - name: Copy keyfile into ArangoDB Docker container - run: docker cp tests/static/keyfile arango:/tmp/keyfile + docker create --name arango -p 8529:8529 -e ARANGO_ROOT_PASSWORD=passwd -v "$(pwd)/tests/static/":/tests/static + arangodb/arangodb:3.10.6 --server.jwt-secret-keyfile=/tests/static/keyfile - name: Start ArangoDB Docker container run: docker start arango diff --git a/.gitignore b/.gitignore index 19a5ba17..c6ef2445 100644 --- a/.gitignore +++ b/.gitignore @@ -121,3 +121,6 @@ node_modules/ # setuptools_scm arango/version.py + +# test results +*_results.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3913ac85..1d25746c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,12 @@ Run unit tests with coverage: py.test --cov=arango --cov-report=html # Open htmlcov/index.html in your browser ``` +For a more comprehensive test suite, run: + +```shell +./tester.sh # Requires docker +``` + Build and test documentation: ```shell diff --git a/arango/backup.py b/arango/backup.py index 5c683a97..c06e4e15 100644 --- a/arango/backup.py +++ b/arango/backup.py @@ -38,7 +38,7 @@ def get(self, backup_id: Optional[str] = None) -> Result[Json]: request = Request( method="post", endpoint="/_admin/backup/list", - data={} if backup_id is None else {"id": backup_id}, + data=None if backup_id is None else {"id": backup_id}, ) def response_handler(resp: Response) -> Json: diff --git a/arango/collection.py b/arango/collection.py index a37f140a..db2ab193 100644 --- a/arango/collection.py +++ b/arango/collection.py @@ -863,7 +863,7 @@ def find_in_box( limit: Optional[int] = None, index: Optional[str] = None, ) -> Result[Cursor]: - """Return all documents in an rectangular area. + """Return all documents in a rectangular area. :param latitude1: First latitude. :type latitude1: int | float diff --git a/arango/formatter.py b/arango/formatter.py index 27f5c172..44efc9aa 100644 --- a/arango/formatter.py +++ b/arango/formatter.py @@ -65,6 +65,8 @@ def format_index(body: Json) -> Json: result["cacheEnabled"] = body["cacheEnabled"] if "legacyPolygons" in body: result["legacyPolygons"] = body["legacyPolygons"] + if "estimates" in body: + result["estimates"] = body["estimates"] return verify_format(body, result) @@ -227,6 +229,8 @@ def format_collection(body: Json) -> Json: } for cv in body["computedValues"] ] + if "internalValidatorType" in body: + result["internal_validator_type"] = body["internalValidatorType"] return verify_format(body, result) @@ -393,6 +397,10 @@ def format_server_status(body: Json) -> Json: """ result: Json = {} + if "agency" in body: + result["agency"] = body["agency"] + if "coordinator" in body: + result["coordinator"] = body["coordinator"] if "foxxApi" in body: result["foxx_api"] = body["foxxApi"] if "host" in body: @@ -985,6 +993,9 @@ def format_backup(body: Json) -> Json: if "nrPiecesPresent" in body: result["pieces_present"] = body["nrPiecesPresent"] + if "countIncludesFilesOnly" in body: + result["count_includes_files_only"] = body["countIncludesFilesOnly"] + return verify_format(body, result) @@ -1135,6 +1146,14 @@ def format_pregel_job_data(body: Json) -> Json: # The detail element was introduced in 3.10 if "detail" in body: result["detail"] = body["detail"] + if "database" in body: + result["database"] = body["database"] + if "masterContext" in body: + result["master_context"] = body["masterContext"] + if "parallelism" in body: + result["parallelism"] = body["parallelism"] + if "useMemoryMaps" in body: + result["use_memory_maps"] = body["useMemoryMaps"] return verify_format(body, result) @@ -1177,12 +1196,18 @@ def format_graph_properties(body: Json) -> Json: } if "isSmart" in body: result["smart"] = body["isSmart"] + if "isSatellite" in body: + result["is_satellite"] = body["isSatellite"] if "smartGraphAttribute" in body: result["smart_field"] = body["smartGraphAttribute"] if "numberOfShards" in body: result["shard_count"] = body["numberOfShards"] if "replicationFactor" in body: result["replication_factor"] = body["replicationFactor"] + if "minReplicationFactor" in body: + result["min_replication_factor"] = body["minReplicationFactor"] + if "writeConcern" in body: + result["write_concern"] = body["writeConcern"] return verify_format(body, result) diff --git a/docs/foxx.rst b/docs/foxx.rst index 97c20605..4f6ce35e 100644 --- a/docs/foxx.rst +++ b/docs/foxx.rst @@ -31,7 +31,7 @@ information, refer to `ArangoDB manual`_. # Create a service using source on server. foxx.create_service( mount=service_mount, - source='/tmp/service.zip', + source='/tests/static/service.zip', config={}, dependencies={}, development=True, @@ -42,7 +42,7 @@ information, refer to `ArangoDB manual`_. # Update (upgrade) a service. service = db.foxx.update_service( mount=service_mount, - source='/tmp/service.zip', + source='/tests/static/service.zip', config={}, dependencies={}, teardown=True, @@ -53,7 +53,7 @@ information, refer to `ArangoDB manual`_. # Replace (overwrite) a service. service = db.foxx.replace_service( mount=service_mount, - source='/tmp/service.zip', + source='/tests/static/service.zip', config={}, dependencies={}, teardown=True, diff --git a/tester.sh b/tester.sh new file mode 100755 index 00000000..dab53b54 --- /dev/null +++ b/tester.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +# Tests python-arango driver against a local ArangoDB single server or cluster setup. +# 1. Starts a local ArangoDB server or cluster (community). +# 2. Runs the python-arango tests for the community edition. +# 3. Starts a local ArangoDB server or cluster (enterprise). +# 4. Runs all python-arango tests, including enterprise tests. + +# Usage: +# ./start.sh [all|single|cluster] [all|community|enterprise] [version] ["notest"] + +setup="${1:-all}" +if [[ "$setup" != "all" && "$setup" != "single" && "$setup" != "cluster" ]]; then + echo "Invalid argument. Please provide either 'all', 'single' or 'cluster'." + exit 1 +fi + +tests="${2:-all}" +if [[ "$tests" != "all" && "$tests" != "community" && "$tests" != "enterprise" ]]; then + echo "Invalid argument. Please provide either 'all', 'community', or 'enterprise'." + exit 1 +fi + +version="${3:-3.10.6}" + +if [[ -n "$4" && "$4" != "notest" ]]; then + echo "Invalid argument. Use 'notest' to only start the docker container, without running the tests." + exit 1 +fi +mode="${4:-test}" + +if [ "$setup" == "all" ] || [ "$setup" == "single" ]; then + if [ "$tests" == "all" ] || [ "$tests" == "community" ]; then + echo "Starting single server community setup..." + docker run -d --rm \ + --name arango \ + -p 8529:8529 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/arangodb:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/single.conf" + + if [[ "$mode" == "notest" ]]; then + exit 0 + fi + + echo "Running python-arango tests for single server community setup..." + sleep 3 + py.test --complete --cov=arango --cov-report=html | tee single_community_results.txt + echo "Stopping single server community setup..." + docker stop arango + docker wait arango + sleep 3 + fi + + if [ "$tests" == "all" ] || [ "$tests" == "enterprise" ]; then + echo "Starting single server enterprise setup..." + docker run -d --rm \ + --name arango \ + -p 8529:8529 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/enterprise:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/single.conf" + + if [[ "$mode" == "notest" ]]; then + exit 0 + fi + + echo "Running python-arango tests for single server enterprise setup..." + sleep 3 + py.test --complete --enterprise --cov=arango --cov-report=html --cov-append | tee single_enterprise_results.txt + echo "Stopping single server enterprise setup..." + docker stop arango + docker wait arango + sleep 3 + fi +fi + +if [ "$setup" == "all" ] || [ "$setup" == "cluster" ]; then + if [ "$tests" == "all" ] || [ "$tests" == "community" ]; then + echo "Starting community cluster setup..." + docker run -d --rm \ + --name arango \ + -p 8529:8529 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/arangodb:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" + + if [[ "$mode" == "notest" ]]; then + exit 0 + fi + + echo "Running python-arango tests for community cluster setup..." + sleep 15 + py.test --cluster --complete --cov=arango --cov-report=html | tee cluster_community_results.txt + echo "Stopping community cluster setup..." + docker stop arango + docker wait arango + sleep 3 + fi + + if [ "$tests" == "all" ] || [ "$tests" == "enterprise" ]; then + echo "Starting enterprise cluster setup..." + docker run -d --rm \ + --name arango \ + -p 8529:8529 \ + -v "$(pwd)/tests/static/":/tests/static \ + -v /tmp:/tmp \ + arangodb/enterprise:"$version" \ + /bin/sh -c "arangodb --configuration=/tests/static/cluster.conf" + + if [[ "$mode" == "notest" ]]; then + exit 0 + fi + + echo "Running python-arango tests for enterprise cluster setup..." + sleep 15 + py.test --cluster --enterprise --complete --cov=arango --cov-report=html | tee cluster_enterprise_results.txt + echo "Stopping enterprise cluster setup..." + docker stop arango + docker wait arango + fi +fi diff --git a/tests/conftest.py b/tests/conftest.py index cfad2a4c..cd670a96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,11 +5,7 @@ from arango import ArangoClient, formatter from arango.database import StandardDatabase from arango.typings import Json -from tests.executors import ( - TestAsyncApiExecutor, - TestBatchExecutor, - TestTransactionApiExecutor, -) +from tests.executors import TestAsyncApiExecutor, TestTransactionApiExecutor from tests.helpers import ( empty_collection, generate_col_name, @@ -210,13 +206,15 @@ def pytest_generate_tests(metafunc): bad_async_db._executor = TestAsyncApiExecutor(bad_conn) bad_dbs.append(bad_async_db) - # Add test batch databases + # Skip test batch databases, as they are deprecated. + """ tst_batch_db = StandardDatabase(tst_conn) tst_batch_db._executor = TestBatchExecutor(tst_conn) tst_dbs.append(tst_batch_db) bad_batch_bdb = StandardDatabase(bad_conn) bad_batch_bdb._executor = TestBatchExecutor(bad_conn) bad_dbs.append(bad_batch_bdb) + """ if "db" in metafunc.fixturenames and "bad_db" in metafunc.fixturenames: metafunc.parametrize("db,bad_db", zip(tst_dbs, bad_dbs)) @@ -234,6 +232,12 @@ def mock_verify_format(body, result): body.pop("error", None) body.pop("code", None) result.pop("edge", None) + + # Remove all None values + # Sometimes they are expected to be excluded from the body (see computedValues) + result = {k: v for k, v in result.items() if v is not None} + body = {k: v for k, v in body.items() if v is not None} + if len(body) != len(result): before = sorted(body, key=lambda x: x.strip("_")) after = sorted(result, key=lambda x: x.strip("_")) diff --git a/tests/static/cluster.conf b/tests/static/cluster.conf new file mode 100644 index 00000000..8438947e --- /dev/null +++ b/tests/static/cluster.conf @@ -0,0 +1,11 @@ +[starter] +mode = cluster +local = true +address = 0.0.0.0 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd +all.log.api-enabled = true diff --git a/tests/static/single.conf b/tests/static/single.conf new file mode 100644 index 00000000..c982303b --- /dev/null +++ b/tests/static/single.conf @@ -0,0 +1,10 @@ +[starter] +mode = single +address = 0.0.0.0 +port = 8528 + +[auth] +jwt-secret = /tests/static/keyfile + +[args] +all.database.password = passwd diff --git a/tests/test_backup.py b/tests/test_backup.py index 23ea563c..e030c5e6 100644 --- a/tests/test_backup.py +++ b/tests/test_backup.py @@ -1,6 +1,6 @@ import pytest -from arango.errno import DATABASE_NOT_FOUND, FILE_NOT_FOUND, FORBIDDEN +from arango.errno import DATABASE_NOT_FOUND, FILE_NOT_FOUND, FORBIDDEN, HTTP_NOT_FOUND from arango.exceptions import ( BackupCreateError, BackupDeleteError, @@ -12,7 +12,7 @@ from tests.helpers import assert_raises -def test_backup_management(sys_db, bad_db, enterprise): +def test_backup_management(sys_db, bad_db, enterprise, cluster): if not enterprise: pytest.skip("Only for ArangoDB enterprise edition") @@ -59,8 +59,9 @@ def test_backup_management(sys_db, bad_db, enterprise): assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} # Test upload backup. + backup_id = backup_id_foo if cluster else backup_id_bar result = sys_db.backup.upload( - backup_id=backup_id_foo, + backup_id=backup_id, repository="local://tmp/backups", config={"local": {"type": "local"}}, ) @@ -79,7 +80,7 @@ def test_backup_management(sys_db, bad_db, enterprise): # Test download backup. result = sys_db.backup.download( - backup_id=backup_id_bar, + backup_id=backup_id_foo, repository="local://tmp/backups", config={"local": {"type": "local"}}, ) @@ -112,4 +113,7 @@ def test_backup_management(sys_db, bad_db, enterprise): # Test delete missing backup. with assert_raises(BackupDeleteError) as err: sys_db.backup.delete(backup_id_foo) - assert err.value.error_code == FILE_NOT_FOUND + if cluster: + assert err.value.error_code == HTTP_NOT_FOUND + else: + assert err.value.error_code == FILE_NOT_FOUND diff --git a/tests/test_collection.py b/tests/test_collection.py index 4f081688..299df959 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -133,12 +133,13 @@ def test_collection_management(db, bad_db, cluster): "rule": { "type": "object", "properties": { - "test_attr": {"type": "string"}, + "test_attr:": {"type": "string"}, }, "required": ["test_attr"], }, "level": "moderate", "message": "Schema Validation Failed.", + "type": "json", } col = db.create_collection( @@ -151,13 +152,13 @@ def test_collection_management(db, bad_db, cluster): key_offset=100, edge=True, shard_count=2, - shard_fields=["test_attr"], + shard_fields=["test_attr:"], replication_factor=1, shard_like="", sync_replication=False, enforce_replication_factor=False, sharding_strategy="community-compat", - smart_join_attribute="test", + smart_join_attribute="test_attr", write_concern=1, schema=schema, ) diff --git a/tests/test_document.py b/tests/test_document.py index 9ae6faa5..d310b45e 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -1326,7 +1326,7 @@ def test_document_find_in_box(col, bad_col, geo, cluster): # Test find_in_box with limit of 1 result = col.find_in_box( - latitude1=0, + latitude1=2, longitude1=0, latitude2=6, longitude2=3, @@ -1359,13 +1359,13 @@ def test_document_find_in_box(col, bad_col, geo, cluster): longitude2=3, skip=1, ) - assert clean_doc(result) == [doc1] + assert clean_doc(result) in [[doc1], [doc3]] # Test find_in_box with skip 3 result = col.find_in_box( - latitude1=0, longitude1=0, latitude2=10, longitude2=10, skip=2 + latitude1=0, longitude1=0, latitude2=10, longitude2=10, skip=3 ) - assert clean_doc(result) == [doc1, doc2] + assert clean_doc(result) in [[doc1], [doc2], [doc3], [doc4]] # Test find_in_box with bad collection with assert_raises(DocumentGetError) as err: @@ -1622,7 +1622,10 @@ def test_document_get_many(col, bad_col, docs): # Test get_many in empty collection empty_collection(col) - assert col.get_many([]) == [] + + # sending an empty list returns internal error + # assert col.get_many([]) == [] + assert col.get_many(docs[:1]) == [] assert col.get_many(docs[:3]) == [] diff --git a/tests/test_foxx.py b/tests/test_foxx.py index 83a7f966..b096d2e8 100644 --- a/tests/test_foxx.py +++ b/tests/test_foxx.py @@ -29,7 +29,7 @@ from arango.foxx import Foxx from tests.helpers import assert_raises, extract, generate_service_mount -service_file = "/tmp/service.zip" +service_file = "/tests/static/service.zip" service_name = "test" diff --git a/tests/test_replication.py b/tests/test_replication.py index bf03991c..2136b97a 100644 --- a/tests/test_replication.py +++ b/tests/test_replication.py @@ -150,9 +150,12 @@ def test_replication_applier(sys_db, bad_db, url, cluster): bad_db.replication.stop_applier() assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} + # We need a tcp endpoint + tcp_endpoint = url.replace("http", "tcp") + # Test replication set applier config result = sys_db.replication.set_applier_config( - endpoint=url, + endpoint=tcp_endpoint, database="_system", username="root", password="passwd", @@ -174,7 +177,7 @@ def test_replication_applier(sys_db, bad_db, url, cluster): restrict_type="exclude", restrict_collections=["students"], ) - assert result["endpoint"] == url + assert result["endpoint"] == tcp_endpoint assert result["database"] == "_system" assert result["username"] == "root" assert result["max_connect_retries"] == 120 @@ -196,7 +199,7 @@ def test_replication_applier(sys_db, bad_db, url, cluster): assert result["restrict_collections"] == ["students"] with assert_raises(ReplicationApplierConfigSetError) as err: - bad_db.replication.set_applier_config(url) + bad_db.replication.set_applier_config(tcp_endpoint) assert err.value.error_code in {FORBIDDEN, DATABASE_NOT_FOUND} # Test replication start applier diff --git a/tests/test_user.py b/tests/test_user.py index 2e48448f..6bf8439a 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -79,7 +79,7 @@ def test_user_management(sys_db, bad_db): ) assert new_user["username"] == username assert new_user["active"] is False - assert new_user["extra"] == {"bar": "baz"} + assert new_user["extra"] == {"foo": "bar", "bar": "baz"} assert sys_db.user(username) == new_user # Update missing user