Dev/1.30 (#275)

* Add support for defining generative config at query-time (dynamic RAG)

* Fix unit test

* Again fix unit test

* Add concurrency limit to CI on a per branch basis to cancel old runs

* Add test of string-only usage with runtime generative

* Add factory to produce user friendly gen runtime config objects

* Remove `it.only` from test

* Update CI images

* Update name of version checker method

* Update to use latest proto with optional model fix in openai

* Fix unit test

* Update CI images

* merge

* Add ability to define tenant-level perms for `data` and `tenants` perms

* Make the next publish job push a next tag (add bash script to do this automatically in future)

* 3.5.0-beta.0

* Fix action typo, add generic to catch this in future

* 3.5.0-beta.1

* Fix broken add/remove permissions method

* 3.5.0-beta.2

* Make fixes in response to user feedback

* Improvements: (#279)

- Fixes bad typing of `GenerativeConfigRuntime`
- adds mocks for this usage
- renames `generativeConfigRuntime` to `generativeParameters` to match Python naming

* Use REST instead of gRPC for `tenants.getByName` when available due to filtering issues with rbac

* Add back tests of unsupported error throwing for <1.25

* chore: update OpenAPI schema

* feat: enable dynamic user management, add db/oidc namespaces

* refactor: re-use common code snippets

* test: extend integration test suite

* refactor(test): use concise Promise.all

* feat(breaking): include user types in user assignments

* chore: format and lint

* test: activate test case for 'oidc' users

* chore: lint and format

* refactor: replace .reduce with .map where possible

Rewrote other .reduce occurences in a more succinct manner

* test: use valid tenant name

* refactor: collect optional parameters in an object

* test: add test case w/ includePermissions=true

* chore: lint and format

* Add missing modules and module params for
- `baseURL` in `generative-anthropic`
- `videoIntervalSeconds` in `multi2vec-google`
- `outputEncoding` in `Multi2VecVoyageAIConfig`
- the entire `text2vec-nvidia` module

* fix: expect no response for /activate and /deactivate

* fix: allow expected code only for instances of WeaviateUnexpectedStatusCodeError

* refactor: declare types in **/types.ts

* test: await on all expectations which should resolve/reject

* refactor: replace postNoBody with a more explicit postReturn<null, R>

* chore: add documentation for dynamic user management

* chore: lint and format

* test: use different port binding for backup/unit.test.ts

Seems like Node 18 does not automatically 'resolve'
port collisions, so 2 test running Express servers
on the same ports will interfere with one another.

* feat: add revokeKey option to /deactivate

* Remove `--tag next` from CI publish in preparation for merging

---------

Co-authored-by: dyma solovei <dyma@weaviate.io>

Author: tsmith023

.github/workflows/main.yaml

@@ -8,11 +8,16 @@ on:
 
 env:
   WEAVIATE_124: 1.24.26
-  WEAVIATE_125: 1.25.30
-  WEAVIATE_126: 1.26.14
-  WEAVIATE_127: 1.27.11
-  WEAVIATE_128: 1.28.4
-  WEAVIATE_129: 1.29.0
+  WEAVIATE_125: 1.25.34
+  WEAVIATE_126: 1.26.17
+  WEAVIATE_127: 1.27.15
+  WEAVIATE_128: 1.28.11
+  WEAVIATE_129: 1.29.1
+  WEAVIATE_130: 1.30.0-rc.0-6b9a01c
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   checks:
@@ -41,9 +46,10 @@ jobs:
           { node: "22.x", weaviate: $WEAVIATE_126},
           { node: "22.x", weaviate: $WEAVIATE_127},
           { node: "22.x", weaviate: $WEAVIATE_128},
-          { node: "18.x", weaviate: $WEAVIATE_129},
-          { node: "20.x", weaviate: $WEAVIATE_129},
-          { node: "22.x", weaviate: $WEAVIATE_129}
+          { node: "22.x", weaviate: $WEAVIATE_129},
+          { node: "18.x", weaviate: $WEAVIATE_130},
+          { node: "20.x", weaviate: $WEAVIATE_130},
+          { node: "22.x", weaviate: $WEAVIATE_130}
         ]
     steps:
     - uses: actions/checkout@v3
@@ -74,7 +80,7 @@ jobs:
       fail-fast: false
       matrix:
         versions: [
-          { node: "22.x", weaviate: $WEAVIATE_129}
+          { node: "22.x", weaviate: $WEAVIATE_130}
         ]
     steps:
     - uses: actions/checkout@v3
@@ -133,4 +139,4 @@ jobs:
         uses: softprops/action-gh-release@v1
         with:
           generate_release_notes: true
-          draft: true
+          draft: true

ci/compose.sh

@@ -28,4 +28,10 @@ services:
       AUTHORIZATION_RBAC_ENABLED: "true"
       AUTHORIZATION_ADMIN_USERS: "admin-user"
       AUTHORIZATION_VIEWER_USERS: "viewer-user"
+      AUTHENTICATION_DB_USERS_ENABLED: "true"
+      AUTHENTICATION_OIDC_ENABLED: "true"
+      AUTHENTICATION_OIDC_CLIENT_ID: "wcs"
+      AUTHENTICATION_OIDC_ISSUER: "https://auth.wcs.api.weaviate.io/auth/realms/SeMI"
+      AUTHENTICATION_OIDC_USERNAME_CLAIM: "email"
+      AUTHENTICATION_OIDC_GROUPS_CLAIM: "groups"
 ...

ci/docker-compose-rbac.yml

@@ -1,6 +1,6 @@
 {
   "name": "weaviate-client",
-  "version": "3.4.2",
+  "version": "3.5.0-beta.2",
   "description": "JS/TS client for Weaviate",
   "main": "dist/node/cjs/index.js",
   "type": "module",

package-lock.json

@@ -109,8 +109,8 @@ describe('Mock testing of backup cancellation', () => {
   let mock: CancelMock;
 
   beforeAll(async () => {
-    mock = await CancelMock.use('1.27.0', 8958, 8959);
-    client = await weaviate.connectToLocal({ port: 8958, grpcPort: 8959 });
+    mock = await CancelMock.use('1.27.0', 8912, 8913);
+    client = await weaviate.connectToLocal({ port: 8912, grpcPort: 8913 });
   });
 
   it('should throw while waiting for creation if backup is cancelled in the meantime', async () => {
@@ -133,7 +133,7 @@ describe('Mock testing of backup cancellation', () => {
   });
 
   it('should return false if creation backup does not exist', async () => {
-    const success = await client.backup.cancel({ backupId: `${BACKUP_ID}4`, backend: BACKEND });
+    const success = await client.backup.cancel({ backupId: `${BACKUP_ID}-unknown`, backend: BACKEND });
     expect(success).toBe(false);
   });
 

package.json

@@ -15,6 +15,7 @@ export type GenerativeAWSConfig = {
 };
 
 export type GenerativeAnthropicConfig = {
+  baseURL?: string;
   maxTokens?: number;
   model?: string;
   stopSequences?: string[];
@@ -58,6 +59,13 @@ export type GenerativeMistralConfig = {
   temperature?: number;
 };
 
+export type GenerativeNvidiaConfig = {
+  baseURL?: string;
+  maxTokens?: number;
+  model?: string;
+  temperature?: number;
+};
+
 export type GenerativeOllamaConfig = {
   apiEndpoint?: string;
   model?: string;

src/collections/backup/unit.test.ts

@@ -35,6 +35,7 @@ export type Vectorizer =
   | 'text2vec-gpt4all'
   | 'text2vec-huggingface'
   | 'text2vec-jinaai'
+  | 'text2vec-nvidia'
   | 'text2vec-mistral'
   | 'text2vec-ollama'
   | 'text2vec-openai'
@@ -169,6 +170,8 @@ export type Multi2VecGoogleConfig = {
   textFields?: string[];
   /** The video fields used when vectorizing. */
   videoFields?: string[];
+  /** Length of a video interval in seconds. */
+  videoIntervalSeconds?: number;
   /** The model ID in use. */
   modelId?: string;
   /** The dimensionality of the vector once embedded. */
@@ -223,6 +226,8 @@ export type Multi2VecVoyageAIConfig = {
   imageFields?: string[];
   /** The model to use. */
   model?: string;
+  /** How the output from the model should be encoded on return. */
+  outputEncoding?: string;
   /** The text fields used when vectorizing. */
   textFields?: string[];
   /** Whether the input should be truncated to fit in the context window. */
@@ -363,6 +368,22 @@ export type Text2VecJinaAIConfig = {
 /** @deprecated Use `Text2VecJinaAIConfig` instead. */
 export type Text2VecJinaConfig = Text2VecJinaAIConfig;
 
+/**
+ * The configuration for text vectorization using the Nvidia module.
+ *
+ * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/nvidia/embeddings) for detailed usage.
+ */
+export type Text2VecNvidiaConfig = {
+  /** The base URL to use where API requests should go. */
+  baseURL?: string;
+  /** The model to use. */
+  model?: string;
+  /** Whether to truncate when vectorising. */
+  truncate?: boolean;
+  /** Whether to vectorize the collection name. */
+  vectorizeCollectionName?: boolean;
+};
+
 /**
  * The configuration for text vectorization using the Mistral module.
  *
@@ -541,6 +562,8 @@ export type VectorizerConfigType<V> = V extends 'img2vec-neural'
   ? Text2VecHuggingFaceConfig | undefined
   : V extends 'text2vec-jinaai'
   ? Text2VecJinaAIConfig | undefined
+  : V extends 'text2vec-nvidia'
+  ? Text2VecNvidiaConfig | undefined
   : V extends 'text2vec-mistral'
   ? Text2VecMistralConfig | undefined
   : V extends 'text2vec-ollama'

src/collections/config/types/generative.ts

@@ -8,6 +8,7 @@ import {
   GenerativeFriendliAIConfig,
   GenerativeGoogleConfig,
   GenerativeMistralConfig,
+  GenerativeNvidiaConfig,
   GenerativeOllamaConfig,
   GenerativeOpenAIConfig,
   GenerativePaLMConfig,
@@ -22,6 +23,7 @@ import {
   GenerativeDatabricksConfigCreate,
   GenerativeFriendliAIConfigCreate,
   GenerativeMistralConfigCreate,
+  GenerativeNvidiaConfigCreate,
   GenerativeOllamaConfigCreate,
   GenerativeOpenAIConfigCreate,
   GenerativePaLMConfigCreate,
@@ -169,6 +171,22 @@ export default {
       config,
     };
   },
+  /**
+   * Create a `ModuleConfig<'generative-nvidia', GenerativeNvidiaConfig | undefined>` object for use when performing AI generation using the `generative-mistral` module.
+   *
+   * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/nvidia/generative) for detailed usage.
+   *
+   * @param {GenerativeNvidiaConfigCreate} [config] The configuration for the `generative-nvidia` module.
+   * @returns {ModuleConfig<'generative-nvidia', GenerativeNvidiaConfig | undefined>} The configuration object.
+   */
+  nvidia(
+    config?: GenerativeNvidiaConfigCreate
+  ): ModuleConfig<'generative-nvidia', GenerativeNvidiaConfig | undefined> {
+    return {
+      name: 'generative-nvidia',
+      config,
+    };
+  },
   /**
    * Create a `ModuleConfig<'generative-ollama', GenerativeOllamaConfig | undefined>` object for use when performing AI generation using the `generative-ollama` module.
    *

src/collections/config/types/vectorizer.ts

@@ -5,6 +5,7 @@ import {
   GenerativeDatabricksConfig,
   GenerativeFriendliAIConfig,
   GenerativeMistralConfig,
+  GenerativeNvidiaConfig,
   GenerativeOllamaConfig,
   GenerativePaLMConfig,
 } from '../../index.js';
@@ -44,6 +45,8 @@ export type GenerativeFriendliAIConfigCreate = GenerativeFriendliAIConfig;
 
 export type GenerativeMistralConfigCreate = GenerativeMistralConfig;
 
+export type GenerativeNvidiaConfigCreate = GenerativeNvidiaConfig;
+
 export type GenerativeOllamaConfigCreate = GenerativeOllamaConfig;
 
 export type GenerativeOpenAIConfigCreate = GenerativeOpenAIConfigBaseCreate & {
@@ -61,6 +64,7 @@ export type GenerativeConfigCreate =
   | GenerativeDatabricksConfigCreate
   | GenerativeFriendliAIConfigCreate
   | GenerativeMistralConfigCreate
+  | GenerativeNvidiaConfigCreate
   | GenerativeOllamaConfigCreate
   | GenerativeOpenAIConfigCreate
   | GenerativePaLMConfigCreate
@@ -81,6 +85,8 @@ export type GenerativeConfigCreateType<G> = G extends 'generative-anthropic'
   ? GenerativeFriendliAIConfigCreate
   : G extends 'generative-mistral'
   ? GenerativeMistralConfigCreate
+  : G extends 'generative-nvidia'
+  ? GenerativeNvidiaConfigCreate
   : G extends 'generative-ollama'
   ? GenerativeOllamaConfigCreate
   : G extends 'generative-openai'

src/collections/configure/generative.ts

@@ -13,6 +13,7 @@ import {
   Text2VecHuggingFaceConfig,
   Text2VecJinaAIConfig,
   Text2VecMistralConfig,
+  Text2VecNvidiaConfig,
   Text2VecOllamaConfig,
   Text2VecOpenAIConfig,
   Text2VecTransformersConfig,
@@ -198,6 +199,8 @@ export type Text2VecHuggingFaceConfigCreate = Text2VecHuggingFaceConfig;
 
 export type Text2VecJinaAIConfigCreate = Text2VecJinaAIConfig;
 
+export type Text2VecNvidiaConfigCreate = Text2VecNvidiaConfig;
+
 export type Text2VecMistralConfigCreate = Text2VecMistralConfig;
 
 export type Text2VecOllamaConfigCreate = Text2VecOllamaConfig;
@@ -247,6 +250,8 @@ export type VectorizerConfigCreateType<V> = V extends 'img2vec-neural'
   ? Text2VecHuggingFaceConfigCreate | undefined
   : V extends 'text2vec-jinaai'
   ? Text2VecJinaAIConfigCreate | undefined
+  : V extends 'text2vec-nvidia'
+  ? Text2VecNvidiaConfigCreate | undefined
   : V extends 'text2vec-mistral'
   ? Text2VecMistralConfigCreate | undefined
   : V extends 'text2vec-ollama'

src/collections/configure/types/generative.ts

@@ -1161,6 +1161,47 @@ describe('Unit testing of the vectorizer factory class', () => {
     });
   });
 
+  it('should create the correct Text2VecNvidiaConfig type with defaults', () => {
+    const config = configure.vectorizer.text2VecNvidia();
+    expect(config).toEqual<VectorConfigCreate<never, undefined, 'hnsw', 'text2vec-nvidia'>>({
+      name: undefined,
+      vectorIndex: {
+        name: 'hnsw',
+        config: undefined,
+      },
+      vectorizer: {
+        name: 'text2vec-nvidia',
+        config: undefined,
+      },
+    });
+  });
+
+  it('should create the correct Text2VecNvidiaConfig type with all values', () => {
+    const config = configure.vectorizer.text2VecNvidia({
+      name: 'test',
+      baseURL: 'base-url',
+      model: 'model',
+      truncate: true,
+      vectorizeCollectionName: true,
+    });
+    expect(config).toEqual<VectorConfigCreate<never, 'test', 'hnsw', 'text2vec-nvidia'>>({
+      name: 'test',
+      vectorIndex: {
+        name: 'hnsw',
+        config: undefined,
+      },
+      vectorizer: {
+        name: 'text2vec-nvidia',
+        config: {
+          baseURL: 'base-url',
+          model: 'model',
+          truncate: true,
+          vectorizeCollectionName: true,
+        },
+      },
+    });
+  });
+
   it('should create the correct Text2VecMistralConfig type with defaults', () => {
     const config = configure.vectorizer.text2VecMistral();
     expect(config).toEqual<VectorConfigCreate<never, undefined, 'hnsw', 'text2vec-mistral'>>({

src/collections/configure/types/vectorizer.ts

@@ -518,6 +518,19 @@ export const vectorizer = {
       },
     });
   },
+  text2VecNvidia: <T, N extends string | undefined = undefined, I extends VectorIndexType = 'hnsw'>(
+    opts?: ConfigureTextVectorizerOptions<T, N, I, 'text2vec-nvidia'>
+  ): VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-nvidia'> => {
+    const { name, sourceProperties, vectorIndexConfig, ...config } = opts || {};
+    return makeVectorizer(name, {
+      sourceProperties,
+      vectorIndexConfig,
+      vectorizerConfig: {
+        name: 'text2vec-nvidia',
+        config: Object.keys(config).length === 0 ? undefined : config,
+      },
+    });
+  },
   /**
    * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-mistral'`.
    *