diff --git a/.evergreen/docker/Dockerfile.musl b/.evergreen/docker/Dockerfile.musl index 18daf0413d4..28f32c89ae8 100644 --- a/.evergreen/docker/Dockerfile.musl +++ b/.evergreen/docker/Dockerfile.musl @@ -12,10 +12,3 @@ git --version c++ --version g++ --version EOF - -# clone drivers evergreen tools into the same location that it lives on the host machine -# this ensures any paths configured in variables by drivers-evergreen-tools work -# in the container too. -ARG DRIVERS_TOOLS=drivers-evergreen-tools -WORKDIR ${DRIVERS_TOOLS} -RUN git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git . diff --git a/.evergreen/prepare-mongodb-aws-ecs-auth.sh b/.evergreen/prepare-mongodb-aws-ecs-auth.sh index 4940b94db88..2de4215c1a7 100755 --- a/.evergreen/prepare-mongodb-aws-ecs-auth.sh +++ b/.evergreen/prepare-mongodb-aws-ecs-auth.sh @@ -21,7 +21,7 @@ cat $ECS_SRC_DIR/.evergreen/run-mongodb-aws-ecs-test.sh # tar the file and drivers tools and do the same cd .. -tar -czf src.tgz src drivers-tools +tar -czf src.tgz src mv src.tgz $ECS_SRC_DIR/src.tgz export MONGODB_BINARIES="${MONGODB_BINARIES}" diff --git a/.evergreen/prepare-shell.sh b/.evergreen/prepare-shell.sh index af8cd4a00ee..ca0c28f4818 100644 --- a/.evergreen/prepare-shell.sh +++ b/.evergreen/prepare-shell.sh @@ -6,10 +6,9 @@ # This script prepares a shell to run the remaining scripts in this folder # It MUST be kept idempotent! It will overwrite the orchestration config and expansion.yml file upon every run -# and it will only clone drivers-tools if they do not exist one directory above our driver src PROJECT_DIRECTORY="$(pwd)" -DRIVERS_TOOLS=$(cd .. && echo "$(pwd)/drivers-tools") +DRIVERS_TOOLS="$(pwd)/drivers-evergreen-tools" MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" UPLOAD_BUCKET="${project}" @@ -30,12 +29,12 @@ export MONGODB_BINARIES export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" -if [ ! -d "$DRIVERS_TOOLS" ]; then - # Only clone driver tools if it does not exist - git clone --depth=1 "https://github.com/mongodb-labs/drivers-evergreen-tools.git" "${DRIVERS_TOOLS}" -fi - -echo "installed DRIVERS_TOOLS from commit $(git -C "${DRIVERS_TOOLS}" rev-parse HEAD)" +# Note the evergreen option on git.get_project recurse_submodules does not work, so do it here. +# We ignore errors in case we are running in a container where git doesn't trust the tmp directory. +set +e +git submodule init +git submodule update +set -e cat < "$MONGO_ORCHESTRATION_HOME/orchestration.config" { diff --git a/.evergreen/run-alpine-fle-tests.sh b/.evergreen/run-alpine-fle-tests.sh index cd518c1b4ee..076c0bc6ee9 100644 --- a/.evergreen/run-alpine-fle-tests.sh +++ b/.evergreen/run-alpine-fle-tests.sh @@ -9,5 +9,10 @@ set -o xtrace export npm_config_cache=$(pwd)/.cache npm install +# Fix to point at the drivers tools pems installed in src. +export CSFLE_TLS_CA_FILE=$(pwd)/drivers-evergreen-tools/.evergreen/x509gen/ca.pem +export CSFLE_TLS_CERT_FILE=$(pwd)/drivers-evergreen-tools/.evergreen/x509gen/server.pem +export CSFLE_TLS_CLIENT_CERT_FILE=$(pwd)/drivers-evergreen-tools/.evergreen/x509gen/client.pem + ALPINE=true \ npm run check:csfle diff --git a/.evergreen/run-deployed-azure-kms-tests.sh b/.evergreen/run-deployed-azure-kms-tests.sh index 45a596656ca..5244f665fb5 100644 --- a/.evergreen/run-deployed-azure-kms-tests.sh +++ b/.evergreen/run-deployed-azure-kms-tests.sh @@ -11,7 +11,7 @@ export AZUREKMS_PUBLICKEYPATH=/tmp/testazurekms_publickey export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey echo "compressing node driver source and tools ... begin" -tar -czf node-driver-source.tgz src drivers-tools +tar -czf node-driver-source.tgz src echo "compressing node driver source and tools ... end" export AZUREKMS_SRC=node-driver-source.tgz diff --git a/.evergreen/run-mongodb-aws-ecs-test.sh b/.evergreen/run-mongodb-aws-ecs-test.sh index bcbfff4a08f..c43776c037e 100755 --- a/.evergreen/run-mongodb-aws-ecs-test.sh +++ b/.evergreen/run-mongodb-aws-ecs-test.sh @@ -5,7 +5,7 @@ set -o errexit # Exit the script with error if any of the commands fail export MONGODB_URI="$1" tar -xzf src/src.tgz -# produces src/ and drivers-tools/ +# produces src/ cd src diff --git a/.evergreen/run-oidc-tests-azure.sh b/.evergreen/run-oidc-tests-azure.sh index 5ce21d65d91..f8a0e88ee07 100644 --- a/.evergreen/run-oidc-tests-azure.sh +++ b/.evergreen/run-oidc-tests-azure.sh @@ -4,7 +4,7 @@ set -o errexit # Exit the script with error if any of the commands fail export AZUREOIDC_DRIVERS_TAR_FILE=/tmp/node-mongodb-native.tgz cd .. -tar -czf $AZUREOIDC_DRIVERS_TAR_FILE src drivers-tools +tar -czf $AZUREOIDC_DRIVERS_TAR_FILE src cd - export AZUREOIDC_TEST_CMD="source ./env.sh && cd src && ENVIRONMENT=azure ./.evergreen/${SCRIPT}" export PROJECT_DIRECTORY=$PROJECT_DIRECTORY diff --git a/.evergreen/run-oidc-tests-gcp.sh b/.evergreen/run-oidc-tests-gcp.sh index 3ec71bfc4a8..490abf3c1f6 100644 --- a/.evergreen/run-oidc-tests-gcp.sh +++ b/.evergreen/run-oidc-tests-gcp.sh @@ -4,7 +4,7 @@ set -o errexit # Exit the script with error if any of the commands fail export GCPOIDC_DRIVERS_TAR_FILE=/tmp/node-mongodb-native.tgz cd .. -tar -czf $GCPOIDC_DRIVERS_TAR_FILE src drivers-tools +tar -czf $GCPOIDC_DRIVERS_TAR_FILE src cd - export GCPOIDC_TEST_CMD="source ./secrets-export.sh drivers/gcpoidc && cd src && ENVIRONMENT=gcp ./.evergreen/${SCRIPT}" export PROJECT_DIRECTORY=$PROJECT_DIRECTORY diff --git a/.evergreen/run-oidc-tests-k8s.sh b/.evergreen/run-oidc-tests-k8s.sh index 261c6df5f52..86a49964584 100644 --- a/.evergreen/run-oidc-tests-k8s.sh +++ b/.evergreen/run-oidc-tests-k8s.sh @@ -4,7 +4,7 @@ set -o errexit # Exit the script with error if any of the commands fail export K8S_DRIVERS_TAR_FILE=/tmp/node-mongodb-native.tgz cd .. -tar -czf $K8S_DRIVERS_TAR_FILE src drivers-tools +tar -czf $K8S_DRIVERS_TAR_FILE src cd - bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/setup-pod.sh bash $DRIVERS_TOOLS/.evergreen/auth_oidc/k8s/run-self-test.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6575d61549c..0efe40b9ac9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,10 @@ version: 2 updates: + - package-ecosystem: "gitsubmodule" # See documentation for possible values + directory: "/" + schedule: + interval: "weekly" - package-ecosystem: "github-actions" # See documentation for possible values directory: "/" # Location of package manifests schedule: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000000..d89986aa201 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "drivers-evergreen-tools"] + path = drivers-evergreen-tools + url = https://github.com/mongodb-labs/drivers-evergreen-tools.git diff --git a/drivers-evergreen-tools b/drivers-evergreen-tools new file mode 160000 index 00000000000..cec013984b2 --- /dev/null +++ b/drivers-evergreen-tools @@ -0,0 +1 @@ +Subproject commit cec013984b202e6327e40d422fcc658fda108fdd diff --git a/test/readme.md b/test/readme.md index e0f39cada0c..6744c633d44 100644 --- a/test/readme.md +++ b/test/readme.md @@ -78,10 +78,17 @@ The actual implementations of the spec tests can be unit tests or integration te The easiest way to get started running the tests locally is to start a standalone server and run all of the tests. -Start a `mongod` standalone with our [cluster_setup.sh](tools/cluster_setup.sh) script: +Ensure the drivers tools submodule is cloned: ```sh -./test/tools/cluster_setup.sh server +git submodule init +git submodule update +``` + +Start a `mongod` standalone with our [run-orchestration.sh](.evergreen/run-orchestration.sh) script with the environment set for the cluster: + +```sh +VERSION='latest' TOPOLOGY='server' AUTH='noauth' ./.evergreen/run-orchestration.sh ``` Then run the tests: @@ -98,11 +105,10 @@ In the following subsections, we'll dig into the details of running the tests. ### Testing With Authorization-Enabled -By default, the integration tests run with auth-enabled and the `cluster_setup.sh` script defaults to starting servers with auth-enabled. Tests can be run locally without auth by setting the environment -variable `AUTH` to the value of `noauth`. This must be a two-step process of starting a server without auth-enabled and then running the tests without auth-enabled. +By default, the integration tests run with auth-enabled and the mongo orchestration script will run with auth enabled when the `AUTH` variable is set to `auth`. Tests can be run locally without auth by setting the environment variable `AUTH` to the value of `noauth`. This must be a two-step process of starting a server without auth-enabled and then running the tests without auth-enabled. ```shell -AUTH='noauth' ./test/tools/cluster_setup.sh +AUTH='noauth' TOPOLOGY='server' ./.evergreen/run-orchestration.sh AUTH='noauth' npm run check:test ``` ### Testing Different MongoDB Topologies @@ -112,21 +118,21 @@ As we mentioned earlier, the tests check the topology of the MongoDB server bein In the steps above, we started a standalone server: ```sh -./test/tools/cluster_setup.sh server +TOPOLOGY='server' ./.evergreen/run-orchestration.sh ``` -You can use the same [cluster_setup.sh](tools/cluster_setup.sh) script to start a replica set or sharded cluster by passing the appropriate option: +You can use the same [run-orchestration.sh](.evergreen/run-orchestration.sh) script to start a replica set or sharded cluster by passing the appropriate option: ```sh -./test/tools/cluster_setup.sh replica_set +TOPOLOGY='replica_set' ./.evergreen/run-orchestration.sh ``` or ```sh -./test/tools/cluster_setup.sh sharded_cluster +TOPOLOGY='sharded_cluster' ./.evergreen/run-orchestration.sh ``` If you are running more than a standalone server, make sure your `ulimit` settings are in accordance with [MongoDB's recommendations][mongodb-ulimit]. Changing the settings on the latest versions of macOS can be tricky. See [this article][macos-ulimt] for tips. (You likely don't need to do the complicated `maxproc` steps.) -The [cluster_setup.sh](tools/cluster_setup.sh) script automatically stores the files associated with the MongoDB server in the `data` directory, which is stored at the top-level of this repository. -You can delete this directory if you want to ensure you're running a clean configuration. If you delete the directory, the associated database server will be stopped, and you will need to run [cluster_setup.sh](tools/cluster_setup.sh) again. +The [run-orchestration.sh](.evergreen/run-orchestration.sh) script automatically stores the files associated with the MongoDB server in the `data` directory, which is stored at the top-level of this repository. +You can delete this directory if you want to ensure you're running a clean configuration. If you delete the directory, the associated database server will be stopped, and you will need to run [run-orchestration.sh](.evergreen/run-orchestration.sh) again. You can prefix `npm test` with a `MONGODB_URI` environment variable to point the tests to a specific deployment. For example, for a standalone server, you might use: @@ -360,12 +366,17 @@ In order to test some features, you will need to generate and set a specialized We recommend using a different terminal for each specialized environment to avoid the environment variables from one specialized environment impacting the test runs for another specialized environment. -Before you begin any of the subsections below, clone the [drivers-evergreen-tools repo](https://github.com/mongodb-labs/drivers-evergreen-tools.git). +Before you begin any of the subsections below, ensure the drivers-evergreen-tools submodule is updated: + +```sh +git submodule init +git submodule update +``` -We recommend creating an environment variable named `DRIVERS_TOOLS` that stores the path to your local copy of the `driver-evergreen-tools` repo (code examples in this section will assume this has been done): +We recommend creating an environment variable named `DRIVERS_TOOLS` that stores the path to the `driver-evergreen-tools` submodule (code examples in this section will assume this has been done): ```sh -export DRIVERS_TOOLS="/path/to/your/copy/of/drivers-evergreen-tools" +export DRIVERS_TOOLS="./drivers-evergreen-tools" ``` ### Serverless diff --git a/test/tools/cluster_setup.sh b/test/tools/cluster_setup.sh deleted file mode 100755 index 65073216457..00000000000 --- a/test/tools/cluster_setup.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ "$#" -ne 1 ]; then - echo "usage: cluster_setup " - echo "override env variables to change dbPath" - exit -fi - -DATA_DIR=${DATA_DIR:-data} -SINGLE_DIR=${SINGLE_DIR:-$DATA_DIR/server} -REPLICASET_DIR=${REPLICASET_DIR:-$DATA_DIR/replica_set} -SHARDED_DIR=${SHARDED_DIR:-$DATA_DIR/sharded_cluster} - -if [[ $1 == "replica_set" ]]; then - mkdir -p $REPLICASET_DIR # user / password - mlaunch init --dir $REPLICASET_DIR --ipv6 --auth --username "bob" --password "pwd123" --replicaset --nodes 3 --arbiter --name rs --port 31000 --enableMajorityReadConcern --setParameter enableTestCommands=1 - echo "mongodb://bob:pwd123@localhost:31000,localhost:31001,localhost:31002/?replicaSet=rs" -elif [[ $1 == "sharded_cluster" ]]; then - mkdir -p $SHARDED_DIR - mlaunch init --dir $SHARDED_DIR --ipv6 --auth --username "bob" --password "pwd123" --replicaset --nodes 3 --name rs --port 51000 --enableMajorityReadConcern --setParameter enableTestCommands=1 --sharded 1 --mongos 2 - echo "mongodb://bob:pwd123@localhost:51000,localhost:51001" -elif [[ $1 == "server" ]]; then - mkdir -p $SINGLE_DIR - mlaunch init --dir $SINGLE_DIR --ipv6 --auth --username "bob" --password "pwd123" --single --setParameter enableTestCommands=1 - echo "mongodb://bob:pwd123@localhost:27017" -else - echo "unsupported topology: $1" -fi