RUBY-3262 enable QEv2 tests on Serverless (#2731)

* RUBY-3262 enable QEv2 tests on Serverless

* also address RUBY-3270

this is just removing a bandaid I added a few weeks ago to work around this very issue

* add CSE specs to serverless

also some incidental cleanup of the version parsing code, which turned out to be unnecessary

* bump spec/shared for better version parsing

* Check wire version on connection

---------

Co-authored-by: Dmitry Rybakov <dmitry.rybakov@mongodb.com>

Author: jamis

.evergreen/run-tests-serverless.sh

@@ -76,6 +76,7 @@ cd -
 echo "Running specs"
 
 bundle exec rspec \
+    spec/spec_tests/client_side_encryption_spec.rb \
     spec/spec_tests/crud_spec.rb \
     spec/spec_tests/retryable_reads_spec.rb \
     spec/spec_tests/retryable_writes_spec.rb \
@@ -91,9 +92,6 @@ bundle exec rspec \
     spec/spec_tests/sessions_unified_spec.rb \
     spec/spec_tests/transactions_unified_spec.rb
 
-# https://jira.mongodb.org/browse/RUBY-3249
-# Add when fixed: spec/spec_tests/client_side_encryption_spec.rb \
-
 kill_jruby
 # Terminate all kmip servers... and whatever else happens to be running
 # that is a python script.

lib/mongo/collection/queryable_encryption.rb

@@ -36,12 +36,14 @@ def maybe_create_qe_collections(encrypted_fields, client, session)
         encrypted_fields = encrypted_fields_from(encrypted_fields)
         return yield if encrypted_fields.empty?
 
-        check_wire_version!
-
-        emm_collections(encrypted_fields).each do |coll|
-          context = Operation::Context.new(client: client, session: session)
-          create_operation_for(coll)
-            .execute(next_primary(nil, session), context: context)
+        server = next_primary(nil, session)
+        context = Operation::Context.new(client: client, session: session)
+        server.with_connection do |connection|
+          check_wire_version!(connection)
+          emm_collections(encrypted_fields).each do |coll|
+            create_operation_for(coll)
+              .execute_with_connection(connection, context: context)
+          end
         end
 
         yield(encrypted_fields).tap do |result|
@@ -99,10 +101,13 @@ def emm_collections(encrypted_fields)
       # Creating encrypted collections is only supported on 7.0.0 and later
       # (wire version 21+).
       #
+      # @param [ Mongo::Connection ] connection The connection to check
+      #   the wire version of.
+      #
       # @raise [ Mongo::Error ] if the wire version is not
       #   recent enough
-      def check_wire_version!
-        return unless next_primary.max_wire_version < QE2_MIN_WIRE_VERSION
+      def check_wire_version!(connection)
+        return unless connection.description.max_wire_version < QE2_MIN_WIRE_VERSION
 
         raise Mongo::Error,
               'Driver support of Queryable Encryption is incompatible with server. ' \

lib/mongo/crypt/binding.rb

@@ -94,34 +94,42 @@ class Binding
       #   @return [ String ] A version string for libmongocrypt.
       attach_function :mongocrypt_version, [:pointer], :string
 
-      # Validates if provided version of libmongocrypt is valid, i.e. equal or
-      # greater than minimum required version. Raises a LoadError if not.
+      # Given a string representing a version number, parses it into a
+      # Gem::Version object. This handles the case where the string is not
+      # in a format supported by Gem::Version by doing some custom parsing.
       #
-      # @param [ String ] lmc_version String representing libmongocrypt version.
+      # @param [ String ] version String representing a version number.
       #
-      # @raise [ LoadError ] if given version is lesser than minimum required version.
+      # @return [ Gem::Version ] the version number
+      #
+      # @raise [ ArgumentError ] if the string cannot be parsed.
       #
       # @api private
-      def self.validate_version(lmc_version)
-        if (actual_version = Gem::Version.new(lmc_version)) < MIN_LIBMONGOCRYPT_VERSION
-          raise LoadError, "libmongocrypt version #{MIN_LIBMONGOCRYPT_VERSION} or above is required, " +
-            "but version #{actual_version} was found."
-        end
-      rescue ArgumentError => e
-        # Some lmc versions cannot be parsed with Gem::Version class,
-        # so we fall back to regex.
-        match = lmc_version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(-[A-Za-z\+\d]+)?\z/)
-        if match.nil?
-          raise ArgumentError.new("Malformed version number string #{lmc_version}")
-        end
-        actual_version = Gem::Version.new(
+      def self.parse_version(version)
+        Gem::Version.new(version)
+      rescue ArgumentError
+        match = version.match(/\A(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)?(-[A-Za-z\+\d]+)?\z/)
+        raise ArgumentError.new("Malformed version number string #{version}") if match.nil?
+
+        Gem::Version.new(
           [
             match[:major],
             match[:minor],
             match[:patch]
           ].join('.')
         )
-        if actual_version < MIN_LIBMONGOCRYPT_VERSION
+      end
+
+      # Validates if provided version of libmongocrypt is valid, i.e. equal or
+      # greater than minimum required version. Raises a LoadError if not.
+      #
+      # @param [ String ] lmc_version String representing libmongocrypt version.
+      #
+      # @raise [ LoadError ] if given version is lesser than minimum required version.
+      #
+      # @api private
+      def self.validate_version(lmc_version)
+        if (actual_version = parse_version(lmc_version)) < MIN_LIBMONGOCRYPT_VERSION
           raise LoadError, "libmongocrypt version #{MIN_LIBMONGOCRYPT_VERSION} or above is required, " +
             "but version #{actual_version} was found."
         end

spec/mongo/crypt/binding/version_spec.rb

@@ -43,10 +43,24 @@
     end
 
     context 'when in a non-parsable format' do
-      it 'does not raise ArgumentError' do
-        expect do
-          Mongo::Crypt::Binding.validate_version("1.5.3-dev+20220730git8f8675fa11")
-        end.not_to raise_error(ArgumentError, /Malformed version number string/)
+      let(:base_version) { Mongo::Crypt::Binding::MIN_LIBMONGOCRYPT_VERSION.to_s }
+
+      shared_examples_for 'non-standard version format' do
+        it 'does not raise an exception' do
+          expect do
+            Mongo::Crypt::Binding.validate_version(version)
+          end.not_to raise_error
+        end
+      end
+
+      context 'when the version is MAJOR.MINOR.PATH-dev+datecommit' do
+        let(:version) { "#{base_version}-dev+20220730git8f8675fa11" }
+        include_examples 'non-standard version format'
+      end
+
+      context 'when the version is MAJOR.MINOR.PATH-date+commit' do
+        let(:version) { "#{base_version}-20230601+git9b07846bef" }
+        include_examples 'non-standard version format'
       end
     end
   end

spec/shared

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-BypassQueryAnalysis.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Compact.yml

@@ -37,3 +37,25 @@ tests:
           collection: "encryptedCollection"
         result:
           errorContains: "Driver support of Queryable Encryption is incompatible with server. Upgrade server to use Queryable Encryption."
+      # Assert no collections were created.
+      - name: assertCollectionNotExists
+        object: testRunner
+        arguments:
+          database: *database_name
+          collection: &esc_collection_name "enxcol_.encryptedCollection.esc"
+      # ecc collection is no longer created for QEv2
+      - name: assertCollectionNotExists
+        object: testRunner
+        arguments:
+          database: *database_name
+          collection: &ecc_collection_name "enxcol_.encryptedCollection.ecc"
+      - name: assertCollectionNotExists
+        object: testRunner
+        arguments:
+          database: *database_name
+          collection: &ecoc_collection_name "enxcol_.encryptedCollection.ecoc"
+      - name: assertCollectionNotExists
+        object: testRunner
+        arguments:
+          database: *database_name
+          collection: encryptedCollection

spec/spec_tests/data/client_side_encryption/fle2v2-CreateCollection-OldServer.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]
@@ -101,10 +100,6 @@ tests:
             command:
               create: *encrypted_collection_name
               encryptedFields: &encrypted_fields_expectation {
-                # Expect state collections are not included in the encryptedFields sent to the server.
-                "escCollection": null,
-                "ecocCollection": null,
-                "eccCollection": null,
                 "fields": [
                     {
                         "path": "firstName",
@@ -939,4 +934,4 @@ tests:
           collection: *encrypted_collection_name
         result:
           # Expect error due to server constraints added in SERVER-74069
-          errorContains: "Encrypted State Collection name should follow"
+          errorContains: "Encrypted State Collection name should follow"

spec/spec_tests/data/client_side_encryption/fle2v2-CreateCollection.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-DecryptExistingData.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Delete.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-EncryptedFieldsMap.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFields-vs-jsonSchema.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-EncryptedFieldsMap-defaults.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-FindOneAndUpdate.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Indexed.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-InsertFind-Unindexed.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-MissingKey.yml

@@ -1,7 +1,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-NoEncryption.yml

@@ -2,7 +2,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-Aggregate.yml

@@ -4,7 +4,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-Correctness.yml

@@ -2,7 +2,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-Delete.yml

@@ -2,7 +2,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-FindOneAndUpdate.yml

@@ -2,7 +2,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]

spec/spec_tests/data/client_side_encryption/fle2v2-Range-Date-InsertFind.yml

@@ -2,7 +2,6 @@
 # Requires libmongocrypt 1.8.0.
 runOn:
   - minServerVersion: "7.0.0"
-    serverless: "forbid"
     # Skip QEv2 (also referred to as FLE2v2) tests on Serverless. Unskip once Serverless enables the QEv2 protocol.
     # FLE 2 Encrypted collections are not supported on standalone.
     topology: [ "replicaset", "sharded", "load-balanced" ]