diff --git a/llmstack/apps/runner/agent_actor.py b/llmstack/apps/runner/agent_actor.py index 5d993ba9f7c..7f7e21fa11f 100644 --- a/llmstack/apps/runner/agent_actor.py +++ b/llmstack/apps/runner/agent_actor.py @@ -5,10 +5,10 @@ from typing import Any, Dict, List from llmstack.apps.runner.agent_controller import ( - AgentController, AgentControllerConfig, AgentControllerData, AgentControllerDataType, + AgentControllerFactory, AgentMessageContent, AgentMessageContentType, AgentToolCallsMessage, @@ -60,7 +60,7 @@ def __init__( ) self._agent_output_queue = asyncio.Queue() - self._agent_controller = AgentController(self._agent_output_queue, self._controller_config) + self._agent_controller = AgentControllerFactory.create(self._agent_output_queue, self._controller_config) def _add_error_from_tool_call(self, output_index, tool_name, tool_call_id, errors): error_message = "\n".join([error for error in errors]) diff --git a/llmstack/apps/runner/agent_controller.py b/llmstack/apps/runner/agent_controller.py index a3714f71fe7..5b5e1703f9f 100644 --- a/llmstack/apps/runner/agent_controller.py +++ b/llmstack/apps/runner/agent_controller.py @@ -134,16 +134,8 @@ def __init__(self, output_queue: asyncio.Queue, config: AgentControllerConfig): self._config = config self._messages: List[AgentMessage] = [] self._llm_client = None - self._websocket = None self._provider_config = None - self._input_text_stream = None - self._input_audio_stream = None - self._input_transcript_stream = None - self._input_metadata = {} - self._output_audio_stream = None - self._output_transcript_stream = None - self._input_messages_queue = queue.Queue() self._loop = asyncio.new_event_loop() self._thread = threading.Thread(target=self._run_event_loop, daemon=True) @@ -153,87 +145,6 @@ def _run_event_loop(self): asyncio.set_event_loop(self._loop) self._loop.run_until_complete(self._process_messages_loop()) - async def _handle_websocket_messages(self): - while self._websocket.open: - response = await self._websocket.recv() - event = json.loads(response) - - if event["type"] == "session.created": - logger.info(f"Session created: {event['session']['id']}") - session = {} - session["instructions"] = self._config.agent_config.system_message - session["tools"] = [ - {"type": "function", **t["function"]} for t in self._config.tools if t["type"] == "function" - ] - session["voice"] = self._config.agent_config.backend.voice - - if self._config.agent_config.input_audio_format: - session["input_audio_format"] = self._config.agent_config.input_audio_format - if self._config.agent_config.output_audio_format: - session["output_audio_format"] = self._config.agent_config.output_audio_format - - updated_session = { - "type": "session.update", - "session": session, - } - await self._send_websocket_message(updated_session) - elif event["type"] == "session.updated": - pass - else: - await self.add_ws_event_to_output_queue(event) - - async def _init_websocket_connection(self): - from llmstack.apps.models import AppSessionFiles - from llmstack.assets.stream import AssetStream - - self._provider_config = get_matched_provider_config( - provider_configs=self._config.provider_configs, - provider_slug=self._config.agent_config.backend.provider, - model_slug=self._config.agent_config.backend.model, - ) - - # Create the output streams - self._output_audio_stream = AssetStream( - await sync_to_async(AppSessionFiles.create_streaming_asset)( - metadata={**self._config.metadata, "mime_type": "audio/wav"}, - ref_id=self._config.metadata.get("session_id"), - ) - ) - self._output_transcript_stream = AssetStream( - await sync_to_async(AppSessionFiles.create_streaming_asset)( - metadata={**self._config.metadata, "mime_type": "text/plain"}, - ref_id=self._config.metadata.get("session_id"), - ) - ) - - ssl_context = ssl.create_default_context() - ssl_context.check_hostname = False - ssl_context.verify_mode = ssl.CERT_NONE - - model_slug = ( - "gpt-4o-realtime-preview" - if self._config.agent_config.backend.model == "gpt-4o-realtime" - else self._config.agent_config.backend.model - ) - websocket_url = f"wss://api.openai.com/v1/realtime?model={model_slug}" - headers = { - "Authorization": f"Bearer {self._provider_config.api_key}", - "OpenAI-Beta": "realtime=v1", - } - - self._websocket = await websockets.connect( - websocket_url, - extra_headers=headers, - ssl=ssl_context, - ) - logger.info(f"WebSocket connection for realtime mode initialized: {self._websocket}") - - # Handle websocket messages and input streams - self._loop.create_task(self._handle_websocket_messages(), name="handle_websocket_messages") - - # Create an initial response - await self._send_websocket_message({"type": "response.create"}) - def _init_llm_client(self): self._provider_config = get_matched_provider_config( provider_configs=self._config.provider_configs, @@ -263,43 +174,6 @@ def _init_llm_client(self): ) ) - async def _process_input_audio_stream(self): - if self._input_audio_stream: - async for chunk in self._input_audio_stream.read_async(): - if len(chunk) == 0: - await self._send_websocket_message({"type": "response.create"}) - break - - # Base64 encode and send - await self._send_websocket_message( - {"type": "input_audio_buffer.append", "audio": base64.b64encode(chunk).decode("utf-8")} - ) - - async def _process_input_text_stream(self): - if self._input_text_stream: - async for chunk in self._input_text_stream.read_async(): - await self._send_websocket_message( - { - "type": "conversation.item.create", - "item": { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": chunk.decode("utf-8")}], - }, - } - ) - - # Cancel the previous response and create a new one - await self._send_websocket_message({"type": "response.cancel"}) - await self._send_websocket_message({"type": "response.create"}) - - # Let the client know that we just got a new message so it can interrupt the playing audio - self._output_queue.put_nowait( - AgentControllerData( - type=AgentControllerDataType.INPUT_STREAM, - ) - ) - async def _process_messages_loop(self): while True: try: @@ -379,79 +253,23 @@ def process(self, data: AgentControllerData): ) async def process_messages(self, data: AgentControllerData): - if self._config.is_voice_agent and self._config.agent_config.backend.backend_type == "multi_modal": - if not self._websocket: - await self._init_websocket_connection() - - # Use the data we just got from input queue when in realtime mode - if data.type == AgentControllerDataType.INPUT_STREAM: - # Use data from AssetStreams and respond accordingly - for content in data.data.content: - if content.type == AgentMessageContentType.TEXT_STREAM: - self._input_text_stream = content.data - elif content.type == AgentMessageContentType.AUDIO_STREAM: - self._input_audio_stream = content.data - elif content.type == AgentMessageContentType.TRANSCRIPT_STREAM: - self._input_transcript_stream = content.data - elif content.type == AgentMessageContentType.METADATA: - self._input_metadata = content.data - - # Process the input streams - self._input_audio_stream_task = self._loop.create_task(self._process_input_audio_stream()) - self._input_text_stream_task = self._loop.create_task(self._process_input_text_stream()) - - # Send output_stream info to the client - self._output_queue.put_nowait( - AgentControllerData( - type=AgentControllerDataType.OUTPUT_STREAM, - data=AgentSystemMessage( - content=[ - AgentMessageContent( - type=AgentMessageContentType.AUDIO_STREAM, - data=self._output_audio_stream.objref, - ), - AgentMessageContent( - type=AgentMessageContentType.TRANSCRIPT_STREAM, - data=self._output_transcript_stream.objref, - ), - ] - ), - ) - ) - elif data.type == AgentControllerDataType.TOOL_CALLS: - for tool_call_id, response in data.data.responses.items(): - await self._send_websocket_message( - { - "type": "conversation.item.create", - "item": { - "type": "function_call_output", - "call_id": tool_call_id, - "output": response, - }, - } - ) - await self._send_websocket_message({"type": "response.create"}) - else: - if not self._llm_client: - self._init_llm_client() - - client_messages = self._convert_messages_to_llm_client_format() - stream = True if self._config.agent_config.stream is None else self._config.agent_config.stream - response = self._llm_client.chat.completions.create( - model=self._config.agent_config.model, - messages=client_messages, - stream=stream, - tools=self._config.tools, - ) - - if stream: - for chunk in response: - self.add_llm_client_response_to_output_queue(chunk) - else: - self.add_llm_client_response_to_output_queue(response) + if not self._llm_client: + self._init_llm_client() + + client_messages = self._convert_messages_to_llm_client_format() + stream = True if self._config.agent_config.stream is None else self._config.agent_config.stream + response = self._llm_client.chat.completions.create( + model=self._config.agent_config.model, + messages=client_messages, + stream=stream, + tools=self._config.tools, + ) - async def _send_websocket_message(self, message): - await self._websocket.send(json.dumps(message)) + if stream: + for chunk in response: + self.add_llm_client_response_to_output_queue(chunk) + else: + self.add_llm_client_response_to_output_queue(response) def add_llm_client_response_to_output_queue(self, response: Any): """ @@ -565,6 +383,65 @@ def add_llm_client_response_to_output_queue(self, response: Any): ) ) + def terminate(self): + # Wait for thread to finish + self._thread.join(timeout=5) + logger.info("Agent controller terminated") + + +class VoiceAgentController(AgentController): + def __init__(self, output_queue: asyncio.Queue, config: AgentControllerConfig): + self._websocket = None + + self._input_text_stream = None + self._input_audio_stream = None + self._input_transcript_stream = None + self._input_metadata = {} + self._output_audio_stream = None + self._output_transcript_stream = None + + super().__init__(output_queue, config) + + async def _process_input_audio_stream(self): + if self._input_audio_stream: + async for chunk in self._input_audio_stream.read_async(): + if len(chunk) == 0: + await self._send_websocket_message({"type": "response.create"}) + break + + # Base64 encode and send + await self._send_websocket_message( + {"type": "input_audio_buffer.append", "audio": base64.b64encode(chunk).decode("utf-8")} + ) + + async def _process_input_text_stream(self): + if self._input_text_stream: + async for chunk in self._input_text_stream.read_async(): + await self._send_websocket_message( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": chunk.decode("utf-8")}], + }, + } + ) + + # Cancel the previous response and create a new one + await self._send_websocket_message({"type": "response.cancel"}) + await self._send_websocket_message({"type": "response.create"}) + + # Let the client know that we just got a new message so it can interrupt the playing audio + self._output_queue.put_nowait( + AgentControllerData( + type=AgentControllerDataType.INPUT_STREAM, + ) + ) + + async def _send_websocket_message(self, message): + await self._websocket.send(json.dumps(message)) + async def add_ws_event_to_output_queue(self, event: Any): event_type = event["type"] @@ -676,6 +553,140 @@ async def add_ws_event_to_output_queue(self, event: Any): elif event_type == "error": logger.error(f"WebSocket error: {event}") + async def _handle_websocket_messages(self): + while self._websocket.open: + response = await self._websocket.recv() + event = json.loads(response) + + if event["type"] == "session.created": + logger.info(f"Session created: {event['session']['id']}") + session = {} + session["instructions"] = self._config.agent_config.system_message + session["tools"] = [ + {"type": "function", **t["function"]} for t in self._config.tools if t["type"] == "function" + ] + session["voice"] = self._config.agent_config.backend.voice + + if self._config.agent_config.input_audio_format: + session["input_audio_format"] = self._config.agent_config.input_audio_format + if self._config.agent_config.output_audio_format: + session["output_audio_format"] = self._config.agent_config.output_audio_format + + updated_session = { + "type": "session.update", + "session": session, + } + await self._send_websocket_message(updated_session) + elif event["type"] == "session.updated": + pass + else: + await self.add_ws_event_to_output_queue(event) + + async def _init_websocket_connection(self): + from llmstack.apps.models import AppSessionFiles + from llmstack.assets.stream import AssetStream + + self._provider_config = get_matched_provider_config( + provider_configs=self._config.provider_configs, + provider_slug=self._config.agent_config.backend.provider, + model_slug=self._config.agent_config.backend.model, + ) + + # Create the output streams + self._output_audio_stream = AssetStream( + await sync_to_async(AppSessionFiles.create_streaming_asset)( + metadata={**self._config.metadata, "mime_type": "audio/wav"}, + ref_id=self._config.metadata.get("session_id"), + ) + ) + self._output_transcript_stream = AssetStream( + await sync_to_async(AppSessionFiles.create_streaming_asset)( + metadata={**self._config.metadata, "mime_type": "text/plain"}, + ref_id=self._config.metadata.get("session_id"), + ) + ) + + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + + model_slug = ( + "gpt-4o-realtime-preview" + if self._config.agent_config.backend.model == "gpt-4o-realtime" + else self._config.agent_config.backend.model + ) + websocket_url = f"wss://api.openai.com/v1/realtime?model={model_slug}" + headers = { + "Authorization": f"Bearer {self._provider_config.api_key}", + "OpenAI-Beta": "realtime=v1", + } + + self._websocket = await websockets.connect( + websocket_url, + extra_headers=headers, + ssl=ssl_context, + ) + logger.info(f"WebSocket connection for realtime mode initialized: {self._websocket}") + + # Handle websocket messages and input streams + self._loop.create_task(self._handle_websocket_messages(), name="handle_websocket_messages") + + # Create an initial response + await self._send_websocket_message({"type": "response.create"}) + + async def process_messages(self, data: AgentControllerData): + if not self._websocket: + await self._init_websocket_connection() + + # Use the data we just got from input queue when in realtime mode + if data.type == AgentControllerDataType.INPUT_STREAM: + # Use data from AssetStreams and respond accordingly + for content in data.data.content: + if content.type == AgentMessageContentType.TEXT_STREAM: + self._input_text_stream = content.data + elif content.type == AgentMessageContentType.AUDIO_STREAM: + self._input_audio_stream = content.data + elif content.type == AgentMessageContentType.TRANSCRIPT_STREAM: + self._input_transcript_stream = content.data + elif content.type == AgentMessageContentType.METADATA: + self._input_metadata = content.data + + # Process the input streams + self._input_audio_stream_task = self._loop.create_task(self._process_input_audio_stream()) + self._input_text_stream_task = self._loop.create_task(self._process_input_text_stream()) + + # Send output_stream info to the client + self._output_queue.put_nowait( + AgentControllerData( + type=AgentControllerDataType.OUTPUT_STREAM, + data=AgentSystemMessage( + content=[ + AgentMessageContent( + type=AgentMessageContentType.AUDIO_STREAM, + data=self._output_audio_stream.objref, + ), + AgentMessageContent( + type=AgentMessageContentType.TRANSCRIPT_STREAM, + data=self._output_transcript_stream.objref, + ), + ] + ), + ) + ) + elif data.type == AgentControllerDataType.TOOL_CALLS: + for tool_call_id, response in data.data.responses.items(): + await self._send_websocket_message( + { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": tool_call_id, + "output": response, + }, + } + ) + await self._send_websocket_message({"type": "response.create"}) + def terminate(self): # Create task for graceful websocket closure if hasattr(self, "_websocket") and self._websocket: @@ -697,6 +708,13 @@ def terminate(self): if hasattr(self, "_input_text_stream_task") and self._input_text_stream_task: self._input_text_stream_task.cancel() - # Wait for thread to finish - self._thread.join(timeout=5) - logger.info("Agent controller terminated") + super().terminate() + + +class AgentControllerFactory: + @staticmethod + def create(output_queue: asyncio.Queue, config: AgentControllerConfig) -> AgentController: + if config.is_voice_agent and config.agent_config.backend.backend_type == "multi_modal": + return VoiceAgentController(output_queue, config) + else: + return AgentController(output_queue, config) diff --git a/llmstack/data/apis.py b/llmstack/data/apis.py index 049017e2403..ee66ef4e63d 100644 --- a/llmstack/data/apis.py +++ b/llmstack/data/apis.py @@ -140,6 +140,7 @@ def destinations(self, request): def transformations(self, request): from llmstack.data.transformations import ( CodeSplitter, + CSVTextSplitter, SemanticDoubleMergingSplitterNodeParser, SentenceSplitter, UnstructuredIOSplitter, @@ -171,6 +172,12 @@ def transformations(self, request): "schema": UnstructuredIOSplitter.get_schema(), "ui_schema": UnstructuredIOSplitter.get_ui_schema(), }, + { + "slug": CSVTextSplitter.slug(), + "provider_slug": CSVTextSplitter.provider_slug(), + "schema": CSVTextSplitter.get_schema(), + "ui_schema": CSVTextSplitter.get_ui_schema(), + }, ] ) diff --git a/llmstack/data/destinations/stores/pandas.py b/llmstack/data/destinations/stores/pandas.py index b477fa1d54d..44fb43bc15d 100644 --- a/llmstack/data/destinations/stores/pandas.py +++ b/llmstack/data/destinations/stores/pandas.py @@ -95,7 +95,7 @@ def add(self, document): k: v for k, v in document.extra_info.get("extra_data", {}).items() if k in [m.target for m in self.mapping] } for item in self.schema: - if item.name == "id" or item.name == "text": + if item.name == "id" or item.name == "text" or item.name == "embedding": continue if item.name not in extra_data: if item.type == "string": @@ -112,7 +112,8 @@ def add(self, document): extra_data[item.name] = str(extra_data[item.name]) for node in document.nodes: - document_dict = {"text": node.text, **extra_data} + node_metadata = node.metadata + document_dict = {"text": node.text, "embedding": node.embedding, **extra_data, **node_metadata} entry_dict = { "id": node.id_, **{mapping.source: document_dict.get(mapping.target) for mapping in self.mapping}, @@ -137,10 +138,19 @@ def delete(self, document: DataDocument): self._asset.update_file(buffer.getvalue(), filename) def search(self, query: str, **kwargs): - result = self._dataframe.query(query).to_dict(orient="records") - nodes = list( - map(lambda x: TextNode(text=json.dumps(x), metadata={"query": query, "source": self._name}), result) - ) + df = self._dataframe + df = df.query(kwargs.get("search_filters") or query) + result = df.to_dict(orient="records") + nodes = [] + for entry in result: + entry.pop("embedding") + nodes.append( + TextNode( + text=json.dumps(entry), + metadata={"query": query, "source": self._name, "search_filters": kwargs.get("search_filters")}, + ) + ) + node_ids = list(map(lambda x: x["id"], result)) return VectorStoreQueryResult(nodes=nodes, ids=node_ids, similarities=[]) diff --git a/llmstack/data/pipeline.py b/llmstack/data/pipeline.py index 6af25a623f3..050ff61b651 100644 --- a/llmstack/data/pipeline.py +++ b/llmstack/data/pipeline.py @@ -99,12 +99,9 @@ def search(self, query: str, use_hybrid_search=True, **kwargs) -> List[dict]: content_key = self.datasource.destination_text_content_key query_embedding = None - if kwargs.get("search_filters", None): - raise NotImplementedError("Search filters are not supported for this data source.") - documents = [] - if self._embedding_generator: + if query and self._embedding_generator: query_embedding = self._embedding_generator.get_embedding(query) if self._destination: diff --git a/llmstack/data/transformations/__init__.py b/llmstack/data/transformations/__init__.py index ee75eea3c76..816f0c7903b 100644 --- a/llmstack/data/transformations/__init__.py +++ b/llmstack/data/transformations/__init__.py @@ -17,10 +17,7 @@ @cache def get_transformer_cls(slug, provider_slug): - for cls in [ - UnstructuredIOSplitter, - EmbeddingsGenerator, - ]: + for cls in [UnstructuredIOSplitter, EmbeddingsGenerator, CSVTextSplitter]: if cls.slug() == slug and cls.provider_slug() == provider_slug: return cls diff --git a/llmstack/data/transformations/splitters.py b/llmstack/data/transformations/splitters.py index 17e3a86122e..9ca94e81598 100644 --- a/llmstack/data/transformations/splitters.py +++ b/llmstack/data/transformations/splitters.py @@ -1,20 +1,54 @@ import csv -from io import StringIO +import json +import logging from typing import List, Optional -from llama_index.core.node_parser.interface import MetadataAwareTextSplitter -from pydantic import Field +from llama_index.core.bridge.pydantic import Field +from llama_index.core.node_parser.interface import TextSplitter +from llmstack.assets.utils import get_asset_by_objref +from llmstack.common.blocks.base.schema import get_ui_schema_from_json_schema -class CSVTextSplitter(MetadataAwareTextSplitter): - include_columns: Optional[List[str]] = Field( - default=None, - description="Columns to include in the text", - ) +logger = logging.getLogger(__name__) + + +class PromptlyTransformers: + @classmethod + def get_schema(cls): + json_schema = cls.schema() + json_schema["properties"].pop("callback_manager", None) + json_schema["properties"].pop("class_name", None) + json_schema["properties"].pop("include_metadata", None) + json_schema["properties"].pop("include_prev_next_rel", None) + return json_schema + + @classmethod + def get_ui_schema(cls): + return get_ui_schema_from_json_schema(cls.get_schema()) + + @classmethod + def get_default_data(cls): + data = cls().dict() + data.pop("callback_manager", None) + data.pop("class_name", None) + data.pop("include_metadata", None) + data.pop("include_prev_next_rel", None) + return data + + +class CSVTextSplitter(TextSplitter, PromptlyTransformers): exclude_columns: Optional[List[str]] = Field( default=None, description="Columns to exclude from the text", ) + text_columns: Optional[List[str]] = Field( + default=None, + description="Columns to include in the text", + ) + metadata_prefix: Optional[str] = Field( + default="cts_", + description="Prefix for metadata columns", + ) @classmethod def slug(cls): @@ -24,27 +58,40 @@ def slug(cls): def provider_slug(cls): return "promptly" - @classmethod - def class_name(cls) -> str: - return "CSVTextSplitter" - - def _split_text(self, text: str) -> List[str]: - chunks = [] - file_handle = StringIO(text) - csv_reader = csv.DictReader(file_handle) - for i, row in enumerate(csv_reader): - content = "" - for column_name, value in row.items(): - if self.include_columns and column_name not in self.include_columns: - continue - if self.exclude_columns and column_name in self.exclude_columns: - continue - content += f"{column_name}: {value}\n" - chunks.append(content) - return chunks - - def split_text_metadata_aware(self, text: str, metadata_str: str) -> List[str]: - return self._split_text(text) - def split_text(self, text: str) -> List[str]: - return self._split_text(text) + raise NotImplementedError + + def split_texts(self, texts: List[str]) -> List[str]: + raise NotImplementedError + + def _parse_nodes(self, nodes, show_progress: bool = False, **kwargs): + from llama_index.core.node_parser.node_utils import build_nodes_from_splits + from llama_index.core.utils import get_tqdm_iterable + + all_nodes = [] + nodes_with_progress = get_tqdm_iterable(nodes, show_progress, "Parsing nodes") + for node in nodes_with_progress: + if hasattr(node, "content"): + asset = get_asset_by_objref(node.content, None, None) + with asset.file.open(mode="r") as f: + csv_reader = csv.DictReader(f) + for row in csv_reader: + content = {} + for column_name, value in row.items(): + if self.exclude_columns and column_name in self.exclude_columns: + continue + content[column_name] = value + row_text = json.dumps(content) + if self.text_columns: + if len(self.text_columns) == 1: + row_text = content[self.text_columns[0]] + else: + text_parts = {} + for column_name in self.text_columns: + text_parts[column_name] = content.get(column_name, "") + row_text = json.dumps(text_parts) + all_nodes.extend(build_nodes_from_splits([row_text], node, id_func=self.id_func)) + for column_name, value in content.items(): + all_nodes[-1].metadata[f"{self.metadata_prefix}{column_name}"] = value + + return all_nodes diff --git a/llmstack/processors/providers/promptly/datasource_search.py b/llmstack/processors/providers/promptly/datasource_search.py index 865b1c3b875..14135679b56 100644 --- a/llmstack/processors/providers/promptly/datasource_search.py +++ b/llmstack/processors/providers/promptly/datasource_search.py @@ -18,7 +18,8 @@ class DataSourceSearchInput(ApiProcessorSchema): - query: str + query: Optional[str] = None + filters: Optional[str] = None class DocumentMetadata(BaseModel): @@ -104,6 +105,7 @@ def process(self) -> DataSourceSearchOutput: alpha=hybrid_semantic_search_ratio, limit=self._config.document_limit, use_hybrid_search=True, + search_filters=input_data.filters or self._config.search_filters, ) documents.extend(result) except BaseException: diff --git a/poetry.lock b/poetry.lock index 8784476f9e9..0d2dd608eae 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1484,20 +1484,6 @@ Django = ">=2.2" docs = ["mkdocs (>=0.17)", "mkdocs-rtd-dropdown (>=0.0.11)", "pymdown-extensions (>=4.11)"] testing = ["coverage (>=3.7.0)", "django-debug-toolbar (>=3.2,<4)", "jinja2"] -[[package]] -name = "django-jsonform" -version = "2.22.0" -description = "A user-friendly JSON editing form for Django admin." -optional = false -python-versions = ">=3.4" -files = [ - {file = "django-jsonform-2.22.0.tar.gz", hash = "sha256:0c9d50fb371938e7262a7fef7c5a60835dd288f872f87b952d5e2ea84c825221"}, - {file = "django_jsonform-2.22.0-py3-none-any.whl", hash = "sha256:c4dd1ba2b0152bd3164aacf326a83c35355c70d12de81908b5ced5f94c8263d6"}, -] - -[package.dependencies] -django = ">=2.0" - [[package]] name = "django-picklefield" version = "3.2" @@ -1667,41 +1653,36 @@ test = ["pytest (>=6)"] [[package]] name = "faiss-cpu" -version = "1.8.0.post1" +version = "1.9.0" description = "A library for efficient similarity search and clustering of dense vectors." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "faiss_cpu-1.8.0.post1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:fd84721eb599aa1da19b1b36345bb8705a60bb1d2887bbbc395a29e3d36a1a62"}, - {file = "faiss_cpu-1.8.0.post1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b78ff9079d15fd0f156bf5dd8a2975a8abffac1854a86ece263eec1500a2e836"}, - {file = "faiss_cpu-1.8.0.post1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9de25c943d1789e35fe06a20884c88cd32aedbb1a33bb8da2238cdea7bd9633f"}, - {file = "faiss_cpu-1.8.0.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:adae0f1b144e7216da696f14bc4991ca4300c94baaa59247c3d322588e661c95"}, - {file = "faiss_cpu-1.8.0.post1-cp310-cp310-win_amd64.whl", hash = "sha256:00345290680a444a4b4cb2d98a3844bb5c401a2160fee547c7631d759fd2ec3e"}, - {file = "faiss_cpu-1.8.0.post1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:8d4bade10cb63e9f9ff261751edd7eb097b1f4bf30be4d0d25d6f688559d795e"}, - {file = "faiss_cpu-1.8.0.post1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:20bd43eca3b7d77e71ea56b7a558cc28e900d8abff417eb285e2d92e95d934d4"}, - {file = "faiss_cpu-1.8.0.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8542a87743a7f94ac656fd3e9592ad57e58b04d961ad2fe654a22a8ca59defdb"}, - {file = "faiss_cpu-1.8.0.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed46928de3dc20170b10fec89c54075a11383c2aaf4f119c63e0f6ae5a507d74"}, - {file = "faiss_cpu-1.8.0.post1-cp311-cp311-win_amd64.whl", hash = "sha256:4fa5fc8ea210b919aa469e27d6687e50052db906e7fec3f2257178b1384fa18b"}, - {file = "faiss_cpu-1.8.0.post1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:96aec0d08a3099883af3a9b6356cfe736e8bd879318a940a27e9d1ae6f33d788"}, - {file = "faiss_cpu-1.8.0.post1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:92b06147fa84732ecdc965922e8ef50dc7011ef8be65821ff4abb2118cb5dce0"}, - {file = "faiss_cpu-1.8.0.post1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:709ef9394d1148aef70dbe890edbde8c282a4a2e06a8b69ab64f65e90f5ba572"}, - {file = "faiss_cpu-1.8.0.post1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:327a9c30971bf72cd8392b15eb4aff5d898c453212eae656dfaa3ba555b9ca0c"}, - {file = "faiss_cpu-1.8.0.post1-cp312-cp312-win_amd64.whl", hash = "sha256:8756f1d93faba56349883fa2f5d47fe36bb2f11f789200c6b1c691ef805485f2"}, - {file = "faiss_cpu-1.8.0.post1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f4a3045909c447bf1955b70083891e80f2c87c5427f20cae25245e08ec5c9e52"}, - {file = "faiss_cpu-1.8.0.post1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8842b7fc921ca1fafdb0845f2ba029e79df04eebae72ab135239f93478a9b7a2"}, - {file = "faiss_cpu-1.8.0.post1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d5a9799634e32c3862d5436d1e78112ed9a38f319e4523f5916e55d86adda8f"}, - {file = "faiss_cpu-1.8.0.post1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a70923b0fbbb40f647e20bcbcbfd472277e6d84bb23ff12d2a94b6841806b55"}, - {file = "faiss_cpu-1.8.0.post1-cp38-cp38-win_amd64.whl", hash = "sha256:ce652df3c4dd50c88ac9235d072f30ce60694dc422c5f523bbbcab320e8f3097"}, - {file = "faiss_cpu-1.8.0.post1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:83ef04b17b19189dd6601a941bdf4bfa9de0740dbcd80305aeba51a1b1955f80"}, - {file = "faiss_cpu-1.8.0.post1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c50c8697077470ede7f1939ef8dc8a846ec19cf1893b543f6b67f9af03b0a122"}, - {file = "faiss_cpu-1.8.0.post1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ce428a7a67fe5c64047280e5e12a8dbdecf7002f9d127b26cf1db354e9fe76"}, - {file = "faiss_cpu-1.8.0.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f3b36b80380bae523e3198cfb4a137867055945ce7bf10d18fe9f0284f2fb47"}, - {file = "faiss_cpu-1.8.0.post1-cp39-cp39-win_amd64.whl", hash = "sha256:4fcc67a2353f08a20c1ab955de3cde14ef3b447761b26244a5aa849c15cbc9b3"}, - {file = "faiss_cpu-1.8.0.post1.tar.gz", hash = "sha256:5686af34414678c3d49c4fa8d774df7156e9cb48d7029071e56230e74b01cc13"}, -] - -[package.dependencies] -numpy = ">=1.0,<2.0" + {file = "faiss_cpu-1.9.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:e415a149893629db2215776395460d0cf79ac5f56a62242de68f788a22b66818"}, + {file = "faiss_cpu-1.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:81f211896107a114450297571210684701d1fce5b998d8a06e2549f6be7af20c"}, + {file = "faiss_cpu-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf76982c45027817df7816232dad9d2f6471637ceaa76c1cc72e858c6e31d8d3"}, + {file = "faiss_cpu-1.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:091d3df18dc9ae43e47203ff0c3c8ffcd51939a6de17e851751dcc263c86b16b"}, + {file = "faiss_cpu-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:eababc154e95930045f86d2483aeb4ed8451b1bb9b97451a2633df20190f5ee2"}, + {file = "faiss_cpu-1.9.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:b0e9208a36da519dc2eb90e4c44c66a6812a5b68457582d8ed21d04e910e3d1f"}, + {file = "faiss_cpu-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6a4b2871057560020b83ad7bb5aaf3b97b64f980f9af2ca99ba34eeb4fe38bdf"}, + {file = "faiss_cpu-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f1dc3a42ea386f49a86a9d09a3e30a40fa2e678395df5c2f5706c3f26f06751"}, + {file = "faiss_cpu-1.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2baeed5f1d8b006533c71184cc29065892647774a3df9c6f6dc31c1b694f57fa"}, + {file = "faiss_cpu-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:81d8fcb0ef92c9e7af2f7104e321895462681a598aff6d526a8da8272a61c1dd"}, + {file = "faiss_cpu-1.9.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:2ed784120f6be7a7cde90f507831e670b4edc94f20cc7955eef3ae5fba70d449"}, + {file = "faiss_cpu-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:358be27446389c9df374fba17221ae5e45a7a8c943c4c675f81814d6fb7c31b1"}, + {file = "faiss_cpu-1.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a0b5ec546c7455cf526326194ace125199769ccbc90bb69b464cd4a26b7f4d"}, + {file = "faiss_cpu-1.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f03a4882e27c71ead60d84d06263d3f8592c842f0f469eeaf7883cfd4f2bfa"}, + {file = "faiss_cpu-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:39a163c2c3c33df10b82fd3b61cb6c8bd7884e2526f1393de32ed71814c5cbfb"}, + {file = "faiss_cpu-1.9.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:b04745b9b93736a7bdf18dd459a3362d154a6dae2e450de3f804f193154d79c9"}, + {file = "faiss_cpu-1.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25dd895a952b5f6dad5dcdb901f853e33359e24ee2b871f418b87af054ed06e0"}, + {file = "faiss_cpu-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0010ddfd16f7c71e1119111973fe2f34b6abc6b40492b688244e821b5a931964"}, + {file = "faiss_cpu-1.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5908e619b3ab2cd1f23f939a995cc2559408dffa9795b69ca78f89a08b993873"}, + {file = "faiss_cpu-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:bc40f1029515baa0228c0c5113b870c5d94961d3232ca25f127162945424375b"}, + {file = "faiss_cpu-1.9.0.tar.gz", hash = "sha256:587fcea9fa478e9307a388754824a032849d317894a607586c3cdd8c8aeb7233"}, +] + +[package.dependencies] +numpy = ">=1.25.0,<3.0" packaging = "*" [[package]] @@ -9146,4 +9127,4 @@ networking = ["junos-eznc"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<4" -content-hash = "31061720f1be5cf23e9ba925dc1eec0fdb68425239e36f1e5426ca15be573695" +content-hash = "7d2f5a9ec66e67c85ef7f0964026f0e0162018adad9d9755cf1bf0ce3a626a5b" diff --git a/pyproject.toml b/pyproject.toml index 0108d12ebc5..da07c41b1c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ django-picklefield = "^3.2" django-redis = "^5.4.0" djangorestframework = "^3.15.2" django-flags = "^5.0.13" -django-jsonform = {version = "^2.17.4"} django-ratelimit = {version = "^4.1.0"} croniter = {version ="^2.0.1"} pykka = "^4.0.2" @@ -75,7 +74,7 @@ django-rq = "^3.0.0" distlib = "^0.3.9" [tool.poetry.group.faiss.dependencies] -faiss-cpu = "^1.8.0" +faiss-cpu = "^1.9.0" [tool.poetry.group.processors]