From 2ce001476737754ae74076a5d09fbbb422ad7223 Mon Sep 17 00:00:00 2001 From: Mikko Korpela Date: Thu, 11 Apr 2024 15:25:20 +0300 Subject: [PATCH 01/28] generated sqlite code --- backend/app/agent.py | 4 +- backend/app/checkpoint.py | 128 +++++++++++++------------ backend/app/lifespan.py | 41 -------- backend/app/server.py | 3 +- backend/app/storage.py | 195 ++++++++++++++++++++------------------ 5 files changed, 174 insertions(+), 197 deletions(-) delete mode 100644 backend/app/lifespan.py diff --git a/backend/app/agent.py b/backend/app/agent.py index 512d85b2..d1540b29 100644 --- a/backend/app/agent.py +++ b/backend/app/agent.py @@ -11,7 +11,7 @@ from app.agent_types.tools_agent import get_tools_agent_executor from app.agent_types.xml_agent import get_xml_agent_executor from app.chatbot import get_chatbot_executor -from app.checkpoint import PostgresCheckpoint +from app.checkpoint import SQLiteCheckpoint from app.llms import ( get_anthropic_llm, get_google_llm, @@ -70,7 +70,7 @@ class AgentType(str, Enum): DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." -CHECKPOINTER = PostgresCheckpoint(at=CheckpointAt.END_OF_STEP) +CHECKPOINTER = SQLiteCheckpoint(at=CheckpointAt.END_OF_STEP) def get_agent_executor( diff --git a/backend/app/checkpoint.py b/backend/app/checkpoint.py index 7f79e85b..4bda5bd3 100644 --- a/backend/app/checkpoint.py +++ b/backend/app/checkpoint.py @@ -1,13 +1,23 @@ import pickle +import sqlite3 +from contextlib import contextmanager from datetime import datetime -from typing import AsyncIterator, Optional +from typing import Iterator, Optional from langchain_core.messages import BaseMessage from langchain_core.runnables import ConfigurableFieldSpec, RunnableConfig from langgraph.checkpoint import BaseCheckpointSaver from langgraph.checkpoint.base import Checkpoint, CheckpointThreadTs, CheckpointTuple -from app.lifespan import get_pg_pool + +@contextmanager +def _connect(): + conn = sqlite3.connect("opengpts.db") + conn.row_factory = sqlite3.Row # Enable dictionary access to row items. + try: + yield conn + finally: + conn.close() def loads(value: bytes) -> Checkpoint: @@ -18,7 +28,7 @@ def loads(value: bytes) -> Checkpoint: return loaded -class PostgresCheckpoint(BaseCheckpointSaver): +class SQLiteCheckpoint(BaseCheckpointSaver): class Config: arbitrary_types_allowed = True @@ -36,19 +46,26 @@ def config_specs(self) -> list[ConfigurableFieldSpec]: CheckpointThreadTs, ] + # Adapting the get method def get(self, config: RunnableConfig) -> Optional[Checkpoint]: + # Implementation adapted for SQLite raise NotImplementedError + # Adapting the put method def put(self, config: RunnableConfig, checkpoint: Checkpoint) -> RunnableConfig: + # Implementation adapted for SQLite raise NotImplementedError - async def alist(self, config: RunnableConfig) -> AsyncIterator[CheckpointTuple]: - async with get_pg_pool().acquire() as db, db.transaction(): - thread_id = config["configurable"]["thread_id"] - async for value in db.cursor( - "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = $1 ORDER BY thread_ts DESC", - thread_id, - ): + # Adapting the alist method + def alist(self, config: RunnableConfig) -> Iterator[CheckpointTuple]: + thread_id = config["configurable"]["thread_id"] + with _connect() as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC", + (thread_id,), + ) + for value in cursor.fetchall(): yield CheckpointTuple( { "configurable": { @@ -67,67 +84,58 @@ async def alist(self, config: RunnableConfig) -> AsyncIterator[CheckpointTuple]: else None, ) - async def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: + # Adapting the aget_tuple method + def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: thread_id = config["configurable"]["thread_id"] thread_ts = config["configurable"].get("thread_ts") - async with get_pg_pool().acquire() as conn: + with _connect() as conn: + cursor = conn.cursor() if thread_ts: - if value := await conn.fetchrow( - "SELECT checkpoint, parent_ts FROM checkpoints WHERE thread_id = $1 AND thread_ts = $2", - thread_id, - datetime.fromisoformat(thread_ts), - ): - return CheckpointTuple( - config, - loads(value[0]), - { - "configurable": { - "thread_id": thread_id, - "thread_ts": value[1], - } - } - if value[1] - else None, - ) + cursor.execute( + "SELECT checkpoint, parent_ts FROM checkpoints WHERE thread_id = ? AND thread_ts = ?", + (thread_id, datetime.fromisoformat(thread_ts)), + ) else: - if value := await conn.fetchrow( - "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = $1 ORDER BY thread_ts DESC LIMIT 1", - thread_id, - ): - return CheckpointTuple( - { - "configurable": { - "thread_id": thread_id, - "thread_ts": value[1], - } - }, - loads(value[0]), - { - "configurable": { - "thread_id": thread_id, - "thread_ts": value[2], - } + cursor.execute( + "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC LIMIT 1", + (thread_id,), + ) + value = cursor.fetchone() + if value: + return CheckpointTuple( + config, + loads(value[0]), + { + "configurable": { + "thread_id": thread_id, + "thread_ts": value[1], } - if value[2] - else None, - ) + } + if value[1] + else None, + ) + return None - async def aput(self, config: RunnableConfig, checkpoint: Checkpoint) -> None: + # Adapting the aput method + def aput(self, config: RunnableConfig, checkpoint: Checkpoint) -> None: thread_id = config["configurable"]["thread_id"] - async with get_pg_pool().acquire() as conn: - await conn.execute( + with _connect() as conn: + cursor = conn.cursor() + cursor.execute( """ INSERT INTO checkpoints (thread_id, thread_ts, parent_ts, checkpoint) - VALUES ($1, $2, $3, $4) + VALUES (?, ?, ?, ?) ON CONFLICT (thread_id, thread_ts) - DO UPDATE SET checkpoint = EXCLUDED.checkpoint;""", - thread_id, - datetime.fromisoformat(checkpoint["ts"]), - datetime.fromisoformat(checkpoint.get("parent_ts")) - if checkpoint.get("parent_ts") - else None, - pickle.dumps(checkpoint), + DO UPDATE SET checkpoint = EXCLUDED.checkpoint; + """, + ( + thread_id, + datetime.fromisoformat(checkpoint["ts"]), + datetime.fromisoformat(checkpoint.get("parent_ts", "")), + pickle.dumps(checkpoint), + ), ) + conn.commit() return { "configurable": { "thread_id": thread_id, diff --git a/backend/app/lifespan.py b/backend/app/lifespan.py deleted file mode 100644 index 8e15f139..00000000 --- a/backend/app/lifespan.py +++ /dev/null @@ -1,41 +0,0 @@ -import os -from contextlib import asynccontextmanager - -import asyncpg -import orjson -from fastapi import FastAPI - -_pg_pool = None - - -def get_pg_pool() -> asyncpg.pool.Pool: - return _pg_pool - - -async def _init_connection(conn) -> None: - await conn.set_type_codec( - "json", - encoder=lambda v: orjson.dumps(v).decode(), - decoder=orjson.loads, - schema="pg_catalog", - ) - await conn.set_type_codec( - "uuid", encoder=lambda v: str(v), decoder=lambda v: v, schema="pg_catalog" - ) - - -@asynccontextmanager -async def lifespan(app: FastAPI): - global _pg_pool - - _pg_pool = await asyncpg.create_pool( - database=os.environ["POSTGRES_DB"], - user=os.environ["POSTGRES_USER"], - password=os.environ["POSTGRES_PASSWORD"], - host=os.environ["POSTGRES_HOST"], - port=os.environ["POSTGRES_PORT"], - init=_init_connection, - ) - yield - await _pg_pool.close() - _pg_pool = None diff --git a/backend/app/server.py b/backend/app/server.py index a8978da4..8e362775 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -10,12 +10,11 @@ import app.storage as storage from app.api import router as api_router from app.auth.handlers import AuthedUser -from app.lifespan import lifespan from app.upload import ingest_runnable logger = logging.getLogger(__name__) -app = FastAPI(title="OpenGPTs API", lifespan=lifespan) +app = FastAPI(title="OpenGPTs API") # Get root of app, used to point to directory containing static files diff --git a/backend/app/storage.py b/backend/app/storage.py index f6a60e40..e7b37cac 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -1,101 +1,116 @@ +import json +import sqlite3 +from contextlib import contextmanager from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Sequence, Union +from uuid import UUID from langchain_core.messages import AnyMessage from app.agent import AgentType, get_agent_executor -from app.lifespan import get_pg_pool from app.schema import Assistant, Thread, User -async def list_assistants(user_id: str) -> List[Assistant]: +@contextmanager +def _connect(): + conn = sqlite3.connect("opengpts.db") + conn.row_factory = sqlite3.Row # Enable dictionary access to row items. + try: + yield conn + finally: + conn.close() + + +def list_assistants(user_id: str) -> List[Assistant]: """List all assistants for the current user.""" - async with get_pg_pool().acquire() as conn: - return await conn.fetch("SELECT * FROM assistant WHERE user_id = $1", user_id) + with _connect() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM assistant WHERE user_id = ?", (user_id,)) + rows = cursor.fetchall() + return [Assistant(**dict(row)) for row in rows] -async def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: +def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: """Get an assistant by ID.""" - async with get_pg_pool().acquire() as conn: - return await conn.fetchrow( - "SELECT * FROM assistant WHERE assistant_id = $1 AND (user_id = $2 OR public = true)", - assistant_id, - user_id, + with _connect() as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT * FROM assistant WHERE assistant_id = ? AND (user_id = ? OR public = 1)", + (assistant_id, user_id), ) + row = cursor.fetchone() + return Assistant(**dict(row)) if row else None -async def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: +def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: """List all the public assistants.""" - async with get_pg_pool().acquire() as conn: - return await conn.fetch( - ( - "SELECT * FROM assistant " - "WHERE assistant_id = ANY($1::uuid[]) " - "AND public = true;" - ), - assistant_ids, + assistant_ids_tuple = tuple( + assistant_ids + ) # SQL requires a tuple for the IN operator. + placeholders = ", ".join("?" for _ in assistant_ids) + with _connect() as conn: + cursor = conn.cursor() + cursor.execute( + f"SELECT * FROM assistant WHERE assistant_id IN ({placeholders}) AND public = 1", + assistant_ids_tuple, ) + rows = cursor.fetchall() + return [Assistant(**dict(row)) for row in rows] -async def put_assistant( +def put_assistant( user_id: str, assistant_id: str, *, name: str, config: dict, public: bool = False ) -> Assistant: - """Modify an assistant. - - Args: - user_id: The user ID. - assistant_id: The assistant ID. - name: The assistant name. - config: The assistant config. - public: Whether the assistant is public. - - Returns: - return the assistant model if no exception is raised. - """ + """Modify an assistant.""" updated_at = datetime.now(timezone.utc) - async with get_pg_pool().acquire() as conn: - async with conn.transaction(): - await conn.execute( - ( - "INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) VALUES ($1, $2, $3, $4, $5, $6) " - "ON CONFLICT (assistant_id) DO UPDATE SET " - "user_id = EXCLUDED.user_id, " - "name = EXCLUDED.name, " - "config = EXCLUDED.config, " - "updated_at = EXCLUDED.updated_at, " - "public = EXCLUDED.public;" - ), - assistant_id, - user_id, - name, - config, - updated_at, - public, - ) - return { - "assistant_id": assistant_id, - "user_id": user_id, - "name": name, - "config": config, - "updated_at": updated_at, - "public": public, - } + with _connect() as conn: + cursor = conn.cursor() + # Convert the config dict to a JSON string for storage. + config_str = json.dumps(config) + cursor.execute( + """ + INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(assistant_id) + DO UPDATE SET + user_id = EXCLUDED.user_id, + name = EXCLUDED.name, + config = EXCLUDED.config, + updated_at = EXCLUDED.updated_at, + public = EXCLUDED.public + """, + (assistant_id, user_id, name, config_str, updated_at.isoformat(), public), + ) + conn.commit() + return Assistant( + assistant_id=UUID(assistant_id), + user_id=user_id, + name=name, + config=config, + updated_at=updated_at, + public=public, + ) -async def list_threads(user_id: str) -> List[Thread]: +def list_threads(user_id: str) -> List[Thread]: """List all threads for the current user.""" - async with get_pg_pool().acquire() as conn: - return await conn.fetch("SELECT * FROM thread WHERE user_id = $1", user_id) + with _connect() as conn: + cursor = conn.cursor() + cursor.execute("SELECT * FROM thread WHERE user_id = ?", (user_id,)) + rows = cursor.fetchall() + return [Thread(**dict(row)) for row in rows] -async def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: +def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" - async with get_pg_pool().acquire() as conn: - return await conn.fetchrow( - "SELECT * FROM thread WHERE thread_id = $1 AND user_id = $2", - thread_id, - user_id, + with _connect() as conn: + cursor = conn.cursor() + cursor.execute( + "SELECT * FROM thread WHERE thread_id = ? AND user_id = ?", + (thread_id, user_id), ) + row = cursor.fetchone() + return Thread(**dict(row)) if row else None async def get_thread_state(user_id: str, thread_id: str): @@ -132,26 +147,23 @@ async def get_thread_history(user_id: str, thread_id: str): ] -async def put_thread( - user_id: str, thread_id: str, *, assistant_id: str, name: str -) -> Thread: +def put_thread(user_id: str, thread_id: str, *, assistant_id: str, name: str) -> Thread: """Modify a thread.""" updated_at = datetime.now(timezone.utc) - async with get_pg_pool().acquire() as conn: - await conn.execute( - ( - "INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) VALUES ($1, $2, $3, $4, $5) " - "ON CONFLICT (thread_id) DO UPDATE SET " - "user_id = EXCLUDED.user_id," - "assistant_id = EXCLUDED.assistant_id, " - "name = EXCLUDED.name, " - "updated_at = EXCLUDED.updated_at;" - ), - thread_id, - user_id, - assistant_id, - name, - updated_at, + with _connect() as conn: + cursor = conn.cursor() + cursor.execute( + """ + INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(thread_id) + DO UPDATE SET + user_id = EXCLUDED.user_id, + assistant_id = EXCLUDED.assistant_id, + name = EXCLUDED.name, + updated_at = EXCLUDED.updated_at + """, + (thread_id, user_id, assistant_id, name, updated_at), ) return { "thread_id": thread_id, @@ -162,14 +174,13 @@ async def put_thread( } -async def get_or_create_user(sub: str) -> tuple[User, bool]: +def get_or_create_user(sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" - async with get_pg_pool().acquire() as conn: - if user := await conn.fetchrow('SELECT * FROM "user" WHERE sub = $1', sub): + with _connect() as conn: + cursor = conn.cursor() + if user := cursor.execute('SELECT * FROM "user" WHERE sub = ?', sub): return user, False - user = await conn.fetchrow( - 'INSERT INTO "user" (sub) VALUES ($1) RETURNING *', sub - ) + user = conn.execute('INSERT INTO "user" (sub) VALUES (?) RETURNING *', sub) return user, True From 34c40babc02c10f2b1981c4b580bae22e8192f8c Mon Sep 17 00:00:00 2001 From: Mikko Korpela Date: Thu, 11 Apr 2024 15:57:15 +0300 Subject: [PATCH 02/28] go install -tags sqlite3 github.com/golang-migrate/migrate/v4/cmd/migrate@latest --- backend/Makefile | 2 +- ..._create_extensions_and_first_tables.up.sql | 35 ++++++++++--------- .../000002_checkpoints_update_schema.down.sql | 23 +++++++++--- .../000002_checkpoints_update_schema.up.sql | 32 ++++++++++++----- 4 files changed, 61 insertions(+), 31 deletions(-) diff --git a/backend/Makefile b/backend/Makefile index 3e8dc5b2..5030e0c0 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -17,7 +17,7 @@ start: poetry run uvicorn app.server:app --reload --port 8100 migrate: - migrate -database postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST):$(POSTGRES_PORT)/$(POSTGRES_DB)?sslmode=disable -path ./migrations up + migrate -database sqlite3://$(PWD)/opengpts.db -path ./migrations up test: # We need to update handling of env variables for tests diff --git a/backend/migrations/000001_create_extensions_and_first_tables.up.sql b/backend/migrations/000001_create_extensions_and_first_tables.up.sql index cb395a74..3ad5be24 100644 --- a/backend/migrations/000001_create_extensions_and_first_tables.up.sql +++ b/backend/migrations/000001_create_extensions_and_first_tables.up.sql @@ -1,24 +1,25 @@ -CREATE EXTENSION IF NOT EXISTS vector; -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - CREATE TABLE IF NOT EXISTS assistant ( - assistant_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - user_id VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, - config JSON NOT NULL, - updated_at TIMESTAMP WITH TIME ZONE DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), - public BOOLEAN NOT NULL + assistant_id TEXT PRIMARY KEY, -- Manually ensure this is a UUID v4 + user_id TEXT NOT NULL, + name TEXT NOT NULL, + config TEXT NOT NULL, -- Store JSON data as text + updated_at DATETIME DEFAULT (datetime('now')), -- Stores in UTC by default + public BOOLEAN NOT NULL CHECK (public IN (0,1)) -- SQLite uses 0 and 1 for BOOLEAN ); CREATE TABLE IF NOT EXISTS thread ( - thread_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - assistant_id UUID REFERENCES assistant(assistant_id) ON DELETE SET NULL, - user_id VARCHAR(255) NOT NULL, - name VARCHAR(255) NOT NULL, - updated_at TIMESTAMP WITH TIME ZONE DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + thread_id TEXT PRIMARY KEY, -- Manually ensure this is a UUID v4 + assistant_id TEXT, -- Store as text and ensure it's a UUID in your application + user_id TEXT NOT NULL, + name TEXT NOT NULL, + updated_at DATETIME DEFAULT (datetime('now')), -- Stores in UTC by default + FOREIGN KEY (assistant_id) REFERENCES assistant(assistant_id) ON DELETE SET NULL ); CREATE TABLE IF NOT EXISTS checkpoints ( - thread_id TEXT PRIMARY KEY, - checkpoint BYTEA -); \ No newline at end of file + thread_id TEXT NOT NULL, + thread_ts DATETIME NOT NULL, + parent_ts DATETIME, + checkpoint BLOB, -- BLOB for binary data, assuming pickle serialization + PRIMARY KEY (thread_id, thread_ts) +); diff --git a/backend/migrations/000002_checkpoints_update_schema.down.sql b/backend/migrations/000002_checkpoints_update_schema.down.sql index c8a249eb..3baf8e2d 100644 --- a/backend/migrations/000002_checkpoints_update_schema.down.sql +++ b/backend/migrations/000002_checkpoints_update_schema.down.sql @@ -1,5 +1,18 @@ -ALTER TABLE checkpoints - DROP CONSTRAINT IF EXISTS checkpoints_pkey, - ADD PRIMARY KEY (thread_id), - DROP COLUMN IF EXISTS thread_ts, - DROP COLUMN IF EXISTS parent_ts; +-- Step 1: Create a new temporary table that reflects the desired final structure, +-- excluding thread_ts and parent_ts columns, and setting thread_id as the primary key. +CREATE TABLE IF NOT EXISTS temp_checkpoints ( + thread_id TEXT NOT NULL, + checkpoint BLOB, + PRIMARY KEY (thread_id) +); + +-- Step 2: Copy relevant data from the original table to the temporary table. +-- Since thread_ts and parent_ts are being dropped, they are not included in the copy. +INSERT INTO temp_checkpoints (thread_id, checkpoint) +SELECT thread_id, checkpoint FROM checkpoints; + +-- Step 3: Drop the original checkpoints table. +DROP TABLE checkpoints; + +-- Step 4: Rename the temporary table to 'checkpoints', effectively recreating the original table structure. +ALTER TABLE temp_checkpoints RENAME TO checkpoints; diff --git a/backend/migrations/000002_checkpoints_update_schema.up.sql b/backend/migrations/000002_checkpoints_update_schema.up.sql index 9ddd077f..50b382c0 100644 --- a/backend/migrations/000002_checkpoints_update_schema.up.sql +++ b/backend/migrations/000002_checkpoints_update_schema.up.sql @@ -1,11 +1,27 @@ -ALTER TABLE checkpoints - ADD COLUMN IF NOT EXISTS thread_ts TIMESTAMPTZ, - ADD COLUMN IF NOT EXISTS parent_ts TIMESTAMPTZ; - +-- Step 2: Update the newly added columns with current UTC datetime where they are NULL. +-- This assumes you handle NULL values appropriately in your application if these columns are expected to have meaningful timestamps. UPDATE checkpoints - SET thread_ts = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' +SET thread_ts = datetime('now') WHERE thread_ts IS NULL; -ALTER TABLE checkpoints - DROP CONSTRAINT IF EXISTS checkpoints_pkey, - ADD PRIMARY KEY (thread_id, thread_ts) +-- Since SQLite does not allow altering a table to drop or add a primary key constraint directly, +-- you need to create a new table with the desired structure, copy the data, drop the old table, and rename the new one. + +-- Step 3: Create a new table with the correct structure and primary key. +CREATE TABLE IF NOT EXISTS new_checkpoints ( + thread_id TEXT NOT NULL, + thread_ts DATETIME NOT NULL, + parent_ts DATETIME, + checkpoint BLOB, + PRIMARY KEY (thread_id, thread_ts) +); + +-- Step 4: Copy data from the old table to the new table. +INSERT INTO new_checkpoints (thread_id, thread_ts, parent_ts, checkpoint) +SELECT thread_id, thread_ts, parent_ts, checkpoint FROM checkpoints; + +-- Step 5: Drop the old table. +DROP TABLE checkpoints; + +-- Step 6: Rename the new table to the original table's name. +ALTER TABLE new_checkpoints RENAME TO checkpoints; From f3e4cd3f7b039758c48d72a49b9944f77a3456d0 Mon Sep 17 00:00:00 2001 From: Mikko Korpela Date: Thu, 11 Apr 2024 16:06:45 +0300 Subject: [PATCH 03/28] make everything awaitable --- backend/app/storage.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/app/storage.py b/backend/app/storage.py index e7b37cac..19c55c82 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -21,7 +21,7 @@ def _connect(): conn.close() -def list_assistants(user_id: str) -> List[Assistant]: +async def list_assistants(user_id: str) -> List[Assistant]: """List all assistants for the current user.""" with _connect() as conn: cursor = conn.cursor() @@ -30,7 +30,7 @@ def list_assistants(user_id: str) -> List[Assistant]: return [Assistant(**dict(row)) for row in rows] -def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: +async def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: """Get an assistant by ID.""" with _connect() as conn: cursor = conn.cursor() @@ -42,7 +42,7 @@ def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: return Assistant(**dict(row)) if row else None -def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: +async def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: """List all the public assistants.""" assistant_ids_tuple = tuple( assistant_ids @@ -58,7 +58,7 @@ def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: return [Assistant(**dict(row)) for row in rows] -def put_assistant( +async def put_assistant( user_id: str, assistant_id: str, *, name: str, config: dict, public: bool = False ) -> Assistant: """Modify an assistant.""" @@ -92,7 +92,7 @@ def put_assistant( ) -def list_threads(user_id: str) -> List[Thread]: +async def list_threads(user_id: str) -> List[Thread]: """List all threads for the current user.""" with _connect() as conn: cursor = conn.cursor() @@ -101,7 +101,7 @@ def list_threads(user_id: str) -> List[Thread]: return [Thread(**dict(row)) for row in rows] -def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: +async def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" with _connect() as conn: cursor = conn.cursor() @@ -147,7 +147,7 @@ async def get_thread_history(user_id: str, thread_id: str): ] -def put_thread(user_id: str, thread_id: str, *, assistant_id: str, name: str) -> Thread: +async def put_thread(user_id: str, thread_id: str, *, assistant_id: str, name: str) -> Thread: """Modify a thread.""" updated_at = datetime.now(timezone.utc) with _connect() as conn: From 2c6fcde34f330c5367640b51327dce7afc2b9e7f Mon Sep 17 00:00:00 2001 From: Mikko Korpela Date: Fri, 12 Apr 2024 14:31:13 +0300 Subject: [PATCH 04/28] sqlite --- backend/app/api/assistants.py | 4 +- backend/app/auth/handlers.py | 4 +- backend/app/server.py | 2 +- backend/app/storage.py | 38 ++++++++---- ..._create_extensions_and_first_tables.up.sql | 4 +- .../migrations/000003_create_user.down.sql | 12 ++-- backend/migrations/000003_create_user.up.sql | 60 +++++++++++++------ 7 files changed, 83 insertions(+), 41 deletions(-) diff --git a/backend/app/api/assistants.py b/backend/app/api/assistants.py index dda15581..d4cb5986 100644 --- a/backend/app/api/assistants.py +++ b/backend/app/api/assistants.py @@ -25,9 +25,9 @@ class AssistantPayload(BaseModel): @router.get("/") -async def list_assistants(user: AuthedUser) -> List[Assistant]: +def list_assistants(user: AuthedUser) -> List[Assistant]: """List all assistants for the current user.""" - return await storage.list_assistants(user["user_id"]) + return storage.list_assistants(user["user_id"]) @router.get("/public/") diff --git a/backend/app/auth/handlers.py b/backend/app/auth/handlers.py index 630d45ff..31d7a797 100644 --- a/backend/app/auth/handlers.py +++ b/backend/app/auth/handlers.py @@ -23,7 +23,7 @@ class NOOPAuth(AuthHandler): async def __call__(self, request: Request) -> User: sub = request.cookies.get("opengpts_user_id") or self._default_sub - user, _ = await storage.get_or_create_user(sub) + user, _ = storage.get_or_create_user(sub) return user @@ -37,7 +37,7 @@ async def __call__(self, request: Request) -> User: except jwt.PyJWTError as e: raise HTTPException(status_code=401, detail=str(e)) - user, _ = await storage.get_or_create_user(payload["sub"]) + user, _ = storage.get_or_create_user(payload["sub"]) return user @abstractmethod diff --git a/backend/app/server.py b/backend/app/server.py index 8e362775..f877e81a 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -33,7 +33,7 @@ async def ingest_files( assistant_id = config["configurable"].get("assistant_id") if assistant_id is not None: - assistant = await storage.get_assistant(user["user_id"], assistant_id) + assistant = storage.get_assistant(user["user_id"], assistant_id) if assistant is None: raise HTTPException(status_code=404, detail="Assistant not found.") diff --git a/backend/app/storage.py b/backend/app/storage.py index 19c55c82..10504453 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -3,7 +3,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Sequence, Union -from uuid import UUID +from uuid import UUID, uuid4 from langchain_core.messages import AnyMessage @@ -21,7 +21,7 @@ def _connect(): conn.close() -async def list_assistants(user_id: str) -> List[Assistant]: +def list_assistants(user_id: str) -> List[Assistant]: """List all assistants for the current user.""" with _connect() as conn: cursor = conn.cursor() @@ -30,7 +30,7 @@ async def list_assistants(user_id: str) -> List[Assistant]: return [Assistant(**dict(row)) for row in rows] -async def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: +def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: """Get an assistant by ID.""" with _connect() as conn: cursor = conn.cursor() @@ -178,17 +178,33 @@ def get_or_create_user(sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" with _connect() as conn: cursor = conn.cursor() - if user := cursor.execute('SELECT * FROM "user" WHERE sub = ?', sub): + cursor.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) + user_row = cursor.fetchone() + + if user_row: + # Convert sqlite3.Row to a User object + user = User(user_id=user_row["user_id"], sub=user_row["sub"], created_at=user_row["created_at"]) return user, False - user = conn.execute('INSERT INTO "user" (sub) VALUES (?) RETURNING *', sub) - return user, True + + # SQLite doesn't support RETURNING *, so we need to manually fetch the created user. + cursor.execute('INSERT INTO "user" (user_id, sub, created_at) VALUES (?, ?, ?)', (str(uuid4()), sub, datetime.now())) + conn.commit() + + # Fetch the newly created user + cursor.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) + new_user_row = cursor.fetchone() + + new_user = User(user_id=new_user_row["user_id"], sub=new_user_row["sub"], + created_at=new_user_row["created_at"]) + return new_user, True async def delete_thread(user_id: str, thread_id: str): """Delete a thread by ID.""" - async with get_pg_pool().acquire() as conn: - await conn.execute( - "DELETE FROM thread WHERE thread_id = $1 AND user_id = $2", - thread_id, - user_id, + with _connect() as conn: + cursor = conn.cursor() + cursor.execute( + "DELETE FROM thread WHERE thread_id = ? AND user_id = ?", + (thread_id, user_id), ) + conn.commit() \ No newline at end of file diff --git a/backend/migrations/000001_create_extensions_and_first_tables.up.sql b/backend/migrations/000001_create_extensions_and_first_tables.up.sql index 3ad5be24..63ec1278 100644 --- a/backend/migrations/000001_create_extensions_and_first_tables.up.sql +++ b/backend/migrations/000001_create_extensions_and_first_tables.up.sql @@ -1,5 +1,5 @@ CREATE TABLE IF NOT EXISTS assistant ( - assistant_id TEXT PRIMARY KEY, -- Manually ensure this is a UUID v4 + assistant_id TEXT PRIMARY KEY NOT NULL, -- Manually ensure this is a UUID v4 user_id TEXT NOT NULL, name TEXT NOT NULL, config TEXT NOT NULL, -- Store JSON data as text @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS assistant ( ); CREATE TABLE IF NOT EXISTS thread ( - thread_id TEXT PRIMARY KEY, -- Manually ensure this is a UUID v4 + thread_id TEXT PRIMARY KEY NOT NULL, -- Manually ensure this is a UUID v4 assistant_id TEXT, -- Store as text and ensure it's a UUID in your application user_id TEXT NOT NULL, name TEXT NOT NULL, diff --git a/backend/migrations/000003_create_user.down.sql b/backend/migrations/000003_create_user.down.sql index 66c5acad..c83349bd 100644 --- a/backend/migrations/000003_create_user.down.sql +++ b/backend/migrations/000003_create_user.down.sql @@ -1,9 +1,9 @@ -ALTER TABLE assistant - DROP CONSTRAINT fk_assistant_user_id, - ALTER COLUMN user_id TYPE VARCHAR USING (user_id::text); +-- SQLite doesn't support ALTER TABLE to drop constraints or change column types directly. +-- Similar to the "up" migration, if you need to reverse the changes, +-- you would have to recreate each table without the foreign keys and with the original column types. -ALTER TABLE thread - DROP CONSTRAINT fk_thread_user_id, - ALTER COLUMN user_id TYPE VARCHAR USING (user_id::text); +-- For "assistant" and "thread", remove the foreign keys by recreating the tables without them. +-- Follow a similar process as described in the "up" migration, but omit the FOREIGN KEY definitions. +-- Drop the "user" table. DROP TABLE IF EXISTS "user"; \ No newline at end of file diff --git a/backend/migrations/000003_create_user.up.sql b/backend/migrations/000003_create_user.up.sql index 45612f9c..8a5c0a7d 100644 --- a/backend/migrations/000003_create_user.up.sql +++ b/backend/migrations/000003_create_user.up.sql @@ -1,25 +1,51 @@ +-- Create the "user" table. Use TEXT for UUID and store timestamps as TEXT. CREATE TABLE IF NOT EXISTS "user" ( - user_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - sub VARCHAR(255) UNIQUE NOT NULL, - created_at TIMESTAMP WITH TIME ZONE DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + user_id TEXT PRIMARY KEY, + sub TEXT UNIQUE NOT NULL, + created_at DATETIME DEFAULT (datetime('now')) ); -INSERT INTO "user" (user_id, sub) -SELECT DISTINCT user_id::uuid, user_id +-- Insert distinct users from the "assistant" table. +-- SQLite doesn't support ON CONFLICT DO NOTHING in the same way, so use INSERT OR IGNORE. +-- The casting (user_id::uuid) isn't needed since we treat all UUIDs as TEXT. +INSERT OR IGNORE INTO "user" (user_id, sub) +SELECT DISTINCT user_id, user_id FROM assistant -WHERE user_id IS NOT NULL -ON CONFLICT (user_id) DO NOTHING; +WHERE user_id IS NOT NULL; -INSERT INTO "user" (user_id, sub) -SELECT DISTINCT user_id::uuid, user_id +-- Insert distinct users from the "thread" table. +INSERT OR IGNORE INTO "user" (user_id, sub) +SELECT DISTINCT user_id, user_id FROM thread -WHERE user_id IS NOT NULL -ON CONFLICT (user_id) DO NOTHING; +WHERE user_id IS NOT NULL; -ALTER TABLE assistant - ALTER COLUMN user_id TYPE UUID USING (user_id::UUID), - ADD CONSTRAINT fk_assistant_user_id FOREIGN KEY (user_id) REFERENCES "user"(user_id); +-- SQLite does not support adding foreign keys via ALTER TABLE. +-- You will need to recreate tables to add foreign key constraints, as shown previously. +-- Here's a simplified approach for "assistant" assuming dropping and recreating is acceptable. -ALTER TABLE thread - ALTER COLUMN user_id TYPE UUID USING (user_id::UUID), - ADD CONSTRAINT fk_thread_user_id FOREIGN KEY (user_id) REFERENCES "user"(user_id); \ No newline at end of file +-- Example for "assistant", assuming it's acceptable to drop & recreate it: +-- 1. Rename existing table. +ALTER TABLE assistant RENAME TO assistant_old; + +-- 2. Create new table with foreign key constraint. +CREATE TABLE assistant ( + assistant_id TEXT PRIMARY KEY NOT NULL, -- Manually ensure this is a UUID v4 + user_id TEXT NOT NULL, + name TEXT NOT NULL, + config TEXT NOT NULL, -- Store JSON data as text + updated_at DATETIME DEFAULT (datetime('now')), -- Stores in UTC by default + public BOOLEAN NOT NULL CHECK (public IN (0,1)), + FOREIGN KEY (user_id) REFERENCES "user" (user_id) +); + +-- Version 3 - Create user table. +CREATE TABLE IF NOT EXISTS "user" ( + user_id TEXT PRIMARY KEY NOT NULL, + sub TEXT UNIQUE NOT NULL, + created_at DATETIME DEFAULT (datetime('now')) +); + +-- 4. Drop old table. +DROP TABLE assistant_old; + +-- Repeat similar steps for "thread" table to add the foreign key constraint. \ No newline at end of file From c584c751e933e0c8db653295ef8844a6f0758604 Mon Sep 17 00:00:00 2001 From: Mikko Korpela Date: Fri, 12 Apr 2024 14:36:23 +0300 Subject: [PATCH 05/28] listing assistants --- backend/app/api/assistants.py | 2 +- backend/app/storage.py | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/backend/app/api/assistants.py b/backend/app/api/assistants.py index d4cb5986..001f29ff 100644 --- a/backend/app/api/assistants.py +++ b/backend/app/api/assistants.py @@ -48,7 +48,7 @@ async def get_assistant( aid: AssistantID, ) -> Assistant: """Get an assistant by ID.""" - assistant = await storage.get_assistant(user["user_id"], aid) + assistant = storage.get_assistant(user["user_id"], aid) if not assistant: raise HTTPException(status_code=404, detail="Assistant not found") return assistant diff --git a/backend/app/storage.py b/backend/app/storage.py index 10504453..ead8206c 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -24,10 +24,21 @@ def _connect(): def list_assistants(user_id: str) -> List[Assistant]: """List all assistants for the current user.""" with _connect() as conn: + conn.row_factory = sqlite3.Row # Enable dictionary-like row access cursor = conn.cursor() cursor.execute("SELECT * FROM assistant WHERE user_id = ?", (user_id,)) rows = cursor.fetchall() - return [Assistant(**dict(row)) for row in rows] + + # Deserialize the 'config' field from a JSON string to a dict for each row + assistants = [] + for row in rows: + assistant_data = dict(row) # Convert sqlite3.Row to dict + assistant_data['config'] = json.loads(assistant_data['config']) if 'config' in assistant_data and \ + assistant_data['config'] else {} + assistant = Assistant(**assistant_data) + assistants.append(assistant) + + return assistants def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: @@ -39,7 +50,12 @@ def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: (assistant_id, user_id), ) row = cursor.fetchone() - return Assistant(**dict(row)) if row else None + if not row: + return None + assistant_data = dict(row) # Convert sqlite3.Row to dict + assistant_data['config'] = json.loads(assistant_data['config']) if 'config' in assistant_data and \ + assistant_data['config'] else {} + return Assistant(**assistant_data) async def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: From 6ad76e92eedcb209c01c035b93dcb5a67db289f3 Mon Sep 17 00:00:00 2001 From: Mikko Korpela Date: Mon, 15 Apr 2024 11:24:32 +0300 Subject: [PATCH 06/28] more working --- backend/app/api/assistants.py | 2 +- backend/app/api/runs.py | 4 ++-- backend/app/api/threads.py | 20 ++++++++------------ backend/app/storage.py | 9 +++++---- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/backend/app/api/assistants.py b/backend/app/api/assistants.py index 001f29ff..090b917d 100644 --- a/backend/app/api/assistants.py +++ b/backend/app/api/assistants.py @@ -60,7 +60,7 @@ async def create_assistant( payload: AssistantPayload, ) -> Assistant: """Create an assistant.""" - return await storage.put_assistant( + return storage.put_assistant( user["user_id"], str(uuid4()), name=payload.name, diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 11ab5758..baa75e70 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -31,11 +31,11 @@ class CreateRunPayload(BaseModel): async def _run_input_and_config(payload: CreateRunPayload, user_id: str): - thread = await get_thread(user_id, payload.thread_id) + thread = get_thread(user_id, payload.thread_id) if not thread: raise HTTPException(status_code=404, detail="Thread not found") - assistant = await get_assistant(user_id, str(thread["assistant_id"])) + assistant = get_assistant(user_id, str(thread["assistant_id"])) if not assistant: raise HTTPException(status_code=404, detail="Assistant not found") diff --git a/backend/app/api/threads.py b/backend/app/api/threads.py index 741ce87b..07b6ffd6 100644 --- a/backend/app/api/threads.py +++ b/backend/app/api/threads.py @@ -41,10 +41,8 @@ async def get_thread_state( tid: ThreadID, ): """Get state for a thread.""" - thread, state = await asyncio.gather( - storage.get_thread(user["user_id"], tid), - storage.get_thread_state(user["user_id"], tid), - ) + thread = storage.get_thread(user["user_id"], tid) + state = await storage.get_thread_state(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return state @@ -57,7 +55,7 @@ async def add_thread_state( payload: ThreadPostRequest, ): """Add state to a thread.""" - thread = await storage.get_thread(user["user_id"], tid) + thread = storage.get_thread(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return await storage.update_thread_state(user["user_id"], tid, payload.values) @@ -69,10 +67,8 @@ async def get_thread_history( tid: ThreadID, ): """Get all past states for a thread.""" - thread, history = await asyncio.gather( - storage.get_thread(user["user_id"], tid), - storage.get_thread_history(user["user_id"], tid), - ) + thread = storage.get_thread(user["user_id"], tid) + history = await storage.get_thread_history(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return history @@ -84,7 +80,7 @@ async def get_thread( tid: ThreadID, ) -> Thread: """Get a thread by ID.""" - thread = await storage.get_thread(user["user_id"], tid) + thread = storage.get_thread(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return thread @@ -96,7 +92,7 @@ async def create_thread( thread_put_request: ThreadPutRequest, ) -> Thread: """Create a thread.""" - return await storage.put_thread( + return storage.put_thread( user["user_id"], str(uuid4()), assistant_id=thread_put_request.assistant_id, @@ -111,7 +107,7 @@ async def upsert_thread( thread_put_request: ThreadPutRequest, ) -> Thread: """Update a thread.""" - return await storage.put_thread( + return storage.put_thread( user["user_id"], tid, assistant_id=thread_put_request.assistant_id, diff --git a/backend/app/storage.py b/backend/app/storage.py index ead8206c..3be7f2be 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -74,7 +74,7 @@ async def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant return [Assistant(**dict(row)) for row in rows] -async def put_assistant( +def put_assistant( user_id: str, assistant_id: str, *, name: str, config: dict, public: bool = False ) -> Assistant: """Modify an assistant.""" @@ -99,7 +99,7 @@ async def put_assistant( ) conn.commit() return Assistant( - assistant_id=UUID(assistant_id), + assistant_id=assistant_id, user_id=user_id, name=name, config=config, @@ -117,7 +117,7 @@ async def list_threads(user_id: str) -> List[Thread]: return [Thread(**dict(row)) for row in rows] -async def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: +def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" with _connect() as conn: cursor = conn.cursor() @@ -163,7 +163,7 @@ async def get_thread_history(user_id: str, thread_id: str): ] -async def put_thread(user_id: str, thread_id: str, *, assistant_id: str, name: str) -> Thread: +def put_thread(user_id: str, thread_id: str, *, assistant_id: str, name: str) -> Thread: """Modify a thread.""" updated_at = datetime.now(timezone.utc) with _connect() as conn: @@ -181,6 +181,7 @@ async def put_thread(user_id: str, thread_id: str, *, assistant_id: str, name: s """, (thread_id, user_id, assistant_id, name, updated_at), ) + conn.commit() return { "thread_id": thread_id, "user_id": user_id, From 775e1ccd7ece54d2efd2bc408e8b244443752e9f Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Wed, 17 Apr 2024 11:40:37 +0400 Subject: [PATCH 07/28] Add Chroma vectorstore. --- .gitignore | 4 + backend/app/ingest.py | 8 + backend/app/upload.py | 14 +- backend/poetry.lock | 1188 +++++++++++++++++++++++++++++++++++++++- backend/pyproject.toml | 1 + 5 files changed, 1202 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index ae1c1eed..c7ace088 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,7 @@ npm-debug.log* yarn-debug.log* yarn-error.log* pnpm-debug.log* + + +# Local db files +chroma_db/ \ No newline at end of file diff --git a/backend/app/ingest.py b/backend/app/ingest.py index e0708045..f4030e2f 100644 --- a/backend/app/ingest.py +++ b/backend/app/ingest.py @@ -20,6 +20,13 @@ def _update_document_metadata(document: Document, namespace: str) -> None: document.metadata["namespace"] = namespace +def _sanitize_document_metadata(document: Document) -> Document: + """Chroma doesn't accept None values in metadata, so we replace them.""" + for k, v in document.metadata.items(): + if v is None: + document.metadata[k] = "" + + def _sanitize_document_content(document: Document) -> Document: """Sanitize the document.""" # Without this, PDF ingestion fails with @@ -46,6 +53,7 @@ def ingest_blob( docs = text_splitter.split_documents([document]) for doc in docs: _sanitize_document_content(doc) + _sanitize_document_metadata(doc) _update_document_metadata(doc, namespace) docs_to_index.extend(docs) diff --git a/backend/app/upload.py b/backend/app/upload.py index 2a53c91f..da24a3a8 100644 --- a/backend/app/upload.py +++ b/backend/app/upload.py @@ -12,6 +12,7 @@ import os from typing import Any, BinaryIO, List, Optional +from langchain_chroma import Chroma from langchain_community.document_loaders.blob_loaders.schema import Blob from langchain_community.vectorstores.pgvector import PGVector from langchain_core.runnables import ( @@ -53,16 +54,14 @@ def _convert_ingestion_input_to_blob(data: BinaryIO) -> Blob: ) -def _determine_azure_or_openai_embeddings() -> PGVector: +def _determine_azure_or_openai_embeddings() -> Chroma: if os.environ.get("OPENAI_API_KEY"): - return PGVector( - connection_string=PG_CONNECTION_STRING, - embedding_function=OpenAIEmbeddings(), - use_jsonb=True, + return Chroma( + persist_directory="./chroma_db", embedding_function=OpenAIEmbeddings() ) if os.environ.get("AZURE_OPENAI_API_KEY"): - return PGVector( - connection_string=PG_CONNECTION_STRING, + return Chroma( + persist_directory="./chroma_db", embedding_function=AzureOpenAIEmbeddings( azure_endpoint=os.environ.get("AZURE_OPENAI_API_BASE"), azure_deployment=os.environ.get( @@ -70,7 +69,6 @@ def _determine_azure_or_openai_embeddings() -> PGVector: ), openai_api_version=os.environ.get("AZURE_OPENAI_API_VERSION"), ), - use_jsonb=True, ) raise ValueError( "Either OPENAI_API_KEY or AZURE_OPENAI_API_KEY needs to be set for embeddings to work." diff --git a/backend/poetry.lock b/backend/poetry.lock index 5288f3a8..60e577ac 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiohttp" @@ -170,6 +170,23 @@ files = [ feedparser = "6.0.10" requests = "2.31.0" +[[package]] +name = "asgiref" +version = "3.8.1" +description = "ASGI specs, helper code, and adapters" +optional = false +python-versions = ">=3.8" +files = [ + {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"}, + {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4", markers = "python_version < \"3.11\""} + +[package.extras] +tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] + [[package]] name = "async-timeout" version = "4.0.3" @@ -267,6 +284,46 @@ files = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] +[[package]] +name = "bcrypt" +version = "4.1.2" +description = "Modern password hashing for your software and your servers" +optional = false +python-versions = ">=3.7" +files = [ + {file = "bcrypt-4.1.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c"}, + {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5"}, + {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0"}, + {file = "bcrypt-4.1.2-cp37-abi3-win32.whl", hash = "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369"}, + {file = "bcrypt-4.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551"}, + {file = "bcrypt-4.1.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7"}, + {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c"}, + {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a"}, + {file = "bcrypt-4.1.2-cp39-abi3-win32.whl", hash = "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f"}, + {file = "bcrypt-4.1.2-cp39-abi3-win_amd64.whl", hash = "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946"}, + {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab"}, + {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb"}, + {file = "bcrypt-4.1.2.tar.gz", hash = "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258"}, +] + +[package.extras] +tests = ["pytest (>=3.2.1,!=3.3.0)"] +typecheck = ["mypy"] + [[package]] name = "beautifulsoup4" version = "4.12.3" @@ -322,13 +379,38 @@ files = [ jmespath = ">=0.7.1,<2.0.0" python-dateutil = ">=2.1,<3.0.0" urllib3 = [ - {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}, {version = ">=1.25.4,<1.27", markers = "python_version < \"3.10\""}, + {version = ">=1.25.4,<2.1", markers = "python_version >= \"3.10\""}, ] [package.extras] crt = ["awscrt (==0.19.19)"] +[[package]] +name = "build" +version = "1.2.1" +description = "A simple, correct Python build frontend" +optional = false +python-versions = ">=3.8" +files = [ + {file = "build-1.2.1-py3-none-any.whl", hash = "sha256:75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4"}, + {file = "build-1.2.1.tar.gz", hash = "sha256:526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "os_name == \"nt\""} +importlib-metadata = {version = ">=4.6", markers = "python_full_version < \"3.10.2\""} +packaging = ">=19.1" +pyproject_hooks = "*" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +docs = ["furo (>=2023.08.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] +test = ["build[uv,virtualenv]", "filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "setuptools (>=56.0.0)", "setuptools (>=67.8.0)", "wheel (>=0.36.0)"] +typing = ["build[uv]", "importlib-metadata (>=5.1)", "mypy (>=1.9.0,<1.10.0)", "tomli", "typing-extensions (>=3.7.4.3)"] +uv = ["uv (>=0.1.18)"] +virtualenv = ["virtualenv (>=20.0.35)"] + [[package]] name = "cachetools" version = "5.3.2" @@ -525,6 +607,84 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] +[[package]] +name = "chroma-hnswlib" +version = "0.7.3" +description = "Chromas fork of hnswlib" +optional = false +python-versions = "*" +files = [ + {file = "chroma-hnswlib-0.7.3.tar.gz", hash = "sha256:b6137bedde49fffda6af93b0297fe00429fc61e5a072b1ed9377f909ed95a932"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59d6a7c6f863c67aeb23e79a64001d537060b6995c3eca9a06e349ff7b0998ca"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d71a3f4f232f537b6152947006bd32bc1629a8686df22fd97777b70f416c127a"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c92dc1ebe062188e53970ba13f6b07e0ae32e64c9770eb7f7ffa83f149d4210"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49da700a6656fed8753f68d44b8cc8ae46efc99fc8a22a6d970dc1697f49b403"}, + {file = "chroma_hnswlib-0.7.3-cp310-cp310-win_amd64.whl", hash = "sha256:108bc4c293d819b56476d8f7865803cb03afd6ca128a2a04d678fffc139af029"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:11e7ca93fb8192214ac2b9c0943641ac0daf8f9d4591bb7b73be808a83835667"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6f552e4d23edc06cdeb553cdc757d2fe190cdeb10d43093d6a3319f8d4bf1c6b"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f96f4d5699e486eb1fb95849fe35ab79ab0901265805be7e60f4eaa83ce263ec"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:368e57fe9ebae05ee5844840fa588028a023d1182b0cfdb1d13f607c9ea05756"}, + {file = "chroma_hnswlib-0.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:b7dca27b8896b494456db0fd705b689ac6b73af78e186eb6a42fea2de4f71c6f"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:70f897dc6218afa1d99f43a9ad5eb82f392df31f57ff514ccf4eeadecd62f544"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aef10b4952708f5a1381c124a29aead0c356f8d7d6e0b520b778aaa62a356f4"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee2d8d1529fca3898d512079144ec3e28a81d9c17e15e0ea4665697a7923253"}, + {file = "chroma_hnswlib-0.7.3-cp37-cp37m-win_amd64.whl", hash = "sha256:a4021a70e898783cd6f26e00008b494c6249a7babe8774e90ce4766dd288c8ba"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a8f61fa1d417fda848e3ba06c07671f14806a2585272b175ba47501b066fe6b1"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7563be58bc98e8f0866907368e22ae218d6060601b79c42f59af4eccbbd2e0a"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51b8d411486ee70d7b66ec08cc8b9b6620116b650df9c19076d2d8b6ce2ae914"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d706782b628e4f43f1b8a81e9120ac486837fbd9bcb8ced70fe0d9b95c72d77"}, + {file = "chroma_hnswlib-0.7.3-cp38-cp38-win_amd64.whl", hash = "sha256:54f053dedc0e3ba657f05fec6e73dd541bc5db5b09aa8bc146466ffb734bdc86"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e607c5a71c610a73167a517062d302c0827ccdd6e259af6e4869a5c1306ffb5d"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2358a795870156af6761890f9eb5ca8cade57eb10c5f046fe94dae1faa04b9e"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cea425df2e6b8a5e201fff0d922a1cc1d165b3cfe762b1408075723c8892218"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:454df3dd3e97aa784fba7cf888ad191e0087eef0fd8c70daf28b753b3b591170"}, + {file = "chroma_hnswlib-0.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:df587d15007ca701c6de0ee7d5585dd5e976b7edd2b30ac72bc376b3c3f85882"}, +] + +[package.dependencies] +numpy = "*" + +[[package]] +name = "chromadb" +version = "0.4.24" +description = "Chroma." +optional = false +python-versions = ">=3.8" +files = [ + {file = "chromadb-0.4.24-py3-none-any.whl", hash = "sha256:3a08e237a4ad28b5d176685bd22429a03717fe09d35022fb230d516108da01da"}, + {file = "chromadb-0.4.24.tar.gz", hash = "sha256:a5c80b4e4ad9b236ed2d4899a5b9e8002b489293f2881cb2cadab5b199ee1c72"}, +] + +[package.dependencies] +bcrypt = ">=4.0.1" +build = ">=1.0.3" +chroma-hnswlib = "0.7.3" +fastapi = ">=0.95.2" +grpcio = ">=1.58.0" +importlib-resources = "*" +kubernetes = ">=28.1.0" +mmh3 = ">=4.0.1" +numpy = ">=1.22.5" +onnxruntime = ">=1.14.1" +opentelemetry-api = ">=1.2.0" +opentelemetry-exporter-otlp-proto-grpc = ">=1.2.0" +opentelemetry-instrumentation-fastapi = ">=0.41b0" +opentelemetry-sdk = ">=1.2.0" +orjson = ">=3.9.12" +overrides = ">=7.3.1" +posthog = ">=2.4.0" +pulsar-client = ">=3.1.0" +pydantic = ">=1.9" +pypika = ">=0.48.9" +PyYAML = ">=6.0.0" +requests = ">=2.28" +tenacity = ">=8.2.3" +tokenizers = ">=0.13.2" +tqdm = ">=4.65.0" +typer = ">=0.9.0" +typing-extensions = ">=4.5.0" +uvicorn = {version = ">=0.18.3", extras = ["standard"]} + [[package]] name = "click" version = "8.1.7" @@ -567,6 +727,23 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coloredlogs" +version = "15.0.1" +description = "Colored terminal output for Python's logging module" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, + {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, +] + +[package.dependencies] +humanfriendly = ">=9.1" + +[package.extras] +cron = ["capturer (>=2.4)"] + [[package]] name = "coverage" version = "7.3.2" @@ -920,6 +1097,17 @@ httpx-sse = "*" Pillow = "*" pydantic = "*" +[[package]] +name = "flatbuffers" +version = "24.3.25" +description = "The FlatBuffers serialization format for Python" +optional = false +python-versions = "*" +files = [ + {file = "flatbuffers-24.3.25-py2.py3-none-any.whl", hash = "sha256:8dbdec58f935f3765e4f7f3cf635ac3a77f83568138d6a2311f524ec96364812"}, + {file = "flatbuffers-24.3.25.tar.gz", hash = "sha256:de2ec5b203f21441716617f38443e0a8ebf3d25bf0d9c0bb0ce68fa00ad546a4"}, +] + [[package]] name = "frozenlist" version = "1.4.0" @@ -1040,12 +1228,12 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -1527,6 +1715,54 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] trio = ["trio (>=0.22.0,<0.23.0)"] +[[package]] +name = "httptools" +version = "0.6.1" +description = "A collection of framework independent HTTP protocol utils." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d2f6c3c4cb1948d912538217838f6e9960bc4a521d7f9b323b3da579cd14532f"}, + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:00d5d4b68a717765b1fabfd9ca755bd12bf44105eeb806c03d1962acd9b8e563"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:639dc4f381a870c9ec860ce5c45921db50205a37cc3334e756269736ff0aac58"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e57997ac7fb7ee43140cc03664de5f268813a481dff6245e0075925adc6aa185"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0ac5a0ae3d9f4fe004318d64b8a854edd85ab76cffbf7ef5e32920faef62f142"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3f30d3ce413088a98b9db71c60a6ada2001a08945cb42dd65a9a9fe228627658"}, + {file = "httptools-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:1ed99a373e327f0107cb513b61820102ee4f3675656a37a50083eda05dc9541b"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7a7ea483c1a4485c71cb5f38be9db078f8b0e8b4c4dc0210f531cdd2ddac1ef1"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85ed077c995e942b6f1b07583e4eb0a8d324d418954fc6af913d36db7c05a5a0"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b0bb634338334385351a1600a73e558ce619af390c2b38386206ac6a27fecfc"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9ceb2c957320def533671fc9c715a80c47025139c8d1f3797477decbc6edd2"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4f0f8271c0a4db459f9dc807acd0eadd4839934a4b9b892f6f160e94da309837"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6a4f5ccead6d18ec072ac0b84420e95d27c1cdf5c9f1bc8fbd8daf86bd94f43d"}, + {file = "httptools-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:5cceac09f164bcba55c0500a18fe3c47df29b62353198e4f37bbcc5d591172c3"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75c8022dca7935cba14741a42744eee13ba05db00b27a4b940f0d646bd4d56d0"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:48ed8129cd9a0d62cf4d1575fcf90fb37e3ff7d5654d3a5814eb3d55f36478c2"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f58e335a1402fb5a650e271e8c2d03cfa7cea46ae124649346d17bd30d59c90"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ad80d7176aa5788902f207a4e79885f0576134695dfb0fefc15b7a4648d503"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9bb68d3a085c2174c2477eb3ffe84ae9fb4fde8792edb7bcd09a1d8467e30a84"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b512aa728bc02354e5ac086ce76c3ce635b62f5fbc32ab7082b5e582d27867bb"}, + {file = "httptools-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:97662ce7fb196c785344d00d638fc9ad69e18ee4bfb4000b35a52efe5adcc949"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8e216a038d2d52ea13fdd9b9c9c7459fb80d78302b257828285eca1c773b99b3"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3e802e0b2378ade99cd666b5bffb8b2a7cc8f3d28988685dc300469ea8dd86cb"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd3e488b447046e386a30f07af05f9b38d3d368d1f7b4d8f7e10af85393db97"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe467eb086d80217b7584e61313ebadc8d187a4d95bb62031b7bab4b205c3ba3"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3c3b214ce057c54675b00108ac42bacf2ab8f85c58e3f324a4e963bbc46424f4"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8ae5b97f690badd2ca27cbf668494ee1b6d34cf1c464271ef7bfa9ca6b83ffaf"}, + {file = "httptools-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:405784577ba6540fa7d6ff49e37daf104e04f4b4ff2d1ac0469eaa6a20fde084"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:95fb92dd3649f9cb139e9c56604cc2d7c7bf0fc2e7c8d7fbd58f96e35eddd2a3"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dcbab042cc3ef272adc11220517278519adf8f53fd3056d0e68f0a6f891ba94e"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cf2372e98406efb42e93bfe10f2948e467edfd792b015f1b4ecd897903d3e8d"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:678fcbae74477a17d103b7cae78b74800d795d702083867ce160fc202104d0da"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e0b281cf5a125c35f7f6722b65d8542d2e57331be573e9e88bc8b0115c4a7a81"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:95658c342529bba4e1d3d2b1a874db16c7cca435e8827422154c9da76ac4e13a"}, + {file = "httptools-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ebaec1bf683e4bf5e9fbb49b8cc36da482033596a415b3e4ebab5a4c0d7ec5e"}, + {file = "httptools-0.6.1.tar.gz", hash = "sha256:c6e26c30455600b95d94b1b836085138e82f177351454ee841c148f93a9bad5a"}, +] + +[package.extras] +test = ["Cython (>=0.29.24,<0.30.0)"] + [[package]] name = "httpx" version = "0.25.2" @@ -1595,6 +1831,20 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jed torch = ["torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] +[[package]] +name = "humanfriendly" +version = "10.0" +description = "Human friendly output for text interfaces using Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, + {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, +] + +[package.dependencies] +pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} + [[package]] name = "idna" version = "3.6" @@ -1606,6 +1856,43 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] +[[package]] +name = "importlib-metadata" +version = "7.0.0" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-7.0.0-py3-none-any.whl", hash = "sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67"}, + {file = "importlib_metadata-7.0.0.tar.gz", hash = "sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7"}, +] + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +perf = ["ipython"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] + +[[package]] +name = "importlib-resources" +version = "6.4.0" +description = "Read resources from Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "importlib_resources-6.4.0-py3-none-any.whl", hash = "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c"}, + {file = "importlib_resources-6.4.0.tar.gz", hash = "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145"}, +] + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["jaraco.test (>=5.4)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"] + [[package]] name = "iniconfig" version = "2.0.0" @@ -1688,6 +1975,32 @@ files = [ [package.dependencies] requests = ">=2" +[[package]] +name = "kubernetes" +version = "29.0.0" +description = "Kubernetes python client" +optional = false +python-versions = ">=3.6" +files = [ + {file = "kubernetes-29.0.0-py2.py3-none-any.whl", hash = "sha256:ab8cb0e0576ccdfb71886366efb102c6a20f268d817be065ce7f9909c631e43e"}, + {file = "kubernetes-29.0.0.tar.gz", hash = "sha256:c4812e227ae74d07d53c88293e564e54b850452715a59a927e7e1bc6b9a60459"}, +] + +[package.dependencies] +certifi = ">=14.05.14" +google-auth = ">=1.0.1" +oauthlib = ">=3.2.2" +python-dateutil = ">=2.5.3" +pyyaml = ">=5.4.1" +requests = "*" +requests-oauthlib = "*" +six = ">=1.9.0" +urllib3 = ">=1.24.2" +websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0" + +[package.extras] +adal = ["adal (>=1.0.2)"] + [[package]] name = "langchain" version = "0.1.10" @@ -1745,6 +2058,23 @@ anthropic = ">=0.23.0,<1" defusedxml = ">=0.7.1,<0.8.0" langchain-core = ">=0.1.42,<0.2.0" +[[package]] +name = "langchain-chroma" +version = "0.1.0" +description = "An integration package connecting Chroma and LangChain" +optional = false +python-versions = "<3.13,>=3.8.1" +files = [ + {file = "langchain_chroma-0.1.0-py3-none-any.whl", hash = "sha256:a9b5624009f4e436f2a7273333c590594daabec6b5a36c0e31ccce29bb5cebf5"}, + {file = "langchain_chroma-0.1.0.tar.gz", hash = "sha256:4f17b0e633b2c182cc546850275420d939d587435c6ebbd821e5d03829539a7a"}, +] + +[package.dependencies] +chromadb = ">=0.4.0,<0.5.0" +fastapi = ">=0.95.2,<1" +langchain-core = ">=0.1.40,<0.2.0" +numpy = ">=1,<2" + [[package]] name = "langchain-community" version = "0.0.29" @@ -2021,6 +2351,30 @@ html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] source = ["Cython (>=3.0.7)"] +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "marshmallow" version = "3.20.1" @@ -2041,6 +2395,137 @@ docs = ["alabaster (==0.7.13)", "autodocsumm (==0.2.11)", "sphinx (==7.0.1)", "s lint = ["flake8 (==6.0.0)", "flake8-bugbear (==23.7.10)", "mypy (==1.4.1)", "pre-commit (>=2.4,<4.0)"] tests = ["pytest", "pytz", "simplejson"] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "mmh3" +version = "4.1.0" +description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." +optional = false +python-versions = "*" +files = [ + {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be5ac76a8b0cd8095784e51e4c1c9c318c19edcd1709a06eb14979c8d850c31a"}, + {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98a49121afdfab67cd80e912b36404139d7deceb6773a83620137aaa0da5714c"}, + {file = "mmh3-4.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5259ac0535874366e7d1a5423ef746e0d36a9e3c14509ce6511614bdc5a7ef5b"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5950827ca0453a2be357696da509ab39646044e3fa15cad364eb65d78797437"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dd0f652ae99585b9dd26de458e5f08571522f0402155809fd1dc8852a613a39"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d25548070942fab1e4a6f04d1626d67e66d0b81ed6571ecfca511f3edf07e6"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53db8d9bad3cb66c8f35cbc894f336273f63489ce4ac416634932e3cbe79eb5b"}, + {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75da0f615eb55295a437264cc0b736753f830b09d102aa4c2a7d719bc445ec05"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b926b07fd678ea84b3a2afc1fa22ce50aeb627839c44382f3d0291e945621e1a"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c5b053334f9b0af8559d6da9dc72cef0a65b325ebb3e630c680012323c950bb6"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bf33dc43cd6de2cb86e0aa73a1cc6530f557854bbbe5d59f41ef6de2e353d7b"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fa7eacd2b830727ba3dd65a365bed8a5c992ecd0c8348cf39a05cc77d22f4970"}, + {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42dfd6742b9e3eec599f85270617debfa0bbb913c545bb980c8a4fa7b2d047da"}, + {file = "mmh3-4.1.0-cp310-cp310-win32.whl", hash = "sha256:2974ad343f0d39dcc88e93ee6afa96cedc35a9883bc067febd7ff736e207fa47"}, + {file = "mmh3-4.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:74699a8984ded645c1a24d6078351a056f5a5f1fe5838870412a68ac5e28d865"}, + {file = "mmh3-4.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:f0dc874cedc23d46fc488a987faa6ad08ffa79e44fb08e3cd4d4cf2877c00a00"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3280a463855b0eae64b681cd5b9ddd9464b73f81151e87bb7c91a811d25619e6"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:97ac57c6c3301769e757d444fa7c973ceb002cb66534b39cbab5e38de61cd896"}, + {file = "mmh3-4.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b6502cdb4dbd880244818ab363c8770a48cdccecf6d729ade0241b736b5ec0"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52ba2da04671a9621580ddabf72f06f0e72c1c9c3b7b608849b58b11080d8f14"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a5fef4c4ecc782e6e43fbeab09cff1bac82c998a1773d3a5ee6a3605cde343e"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5135358a7e00991f73b88cdc8eda5203bf9de22120d10a834c5761dbeb07dd13"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cff9ae76a54f7c6fe0167c9c4028c12c1f6de52d68a31d11b6790bb2ae685560"}, + {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f02576a4d106d7830ca90278868bf0983554dd69183b7bbe09f2fcd51cf54f"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:073d57425a23721730d3ff5485e2da489dd3c90b04e86243dd7211f889898106"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:71e32ddec7f573a1a0feb8d2cf2af474c50ec21e7a8263026e8d3b4b629805db"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7cbb20b29d57e76a58b40fd8b13a9130db495a12d678d651b459bf61c0714cea"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a42ad267e131d7847076bb7e31050f6c4378cd38e8f1bf7a0edd32f30224d5c9"}, + {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a013979fc9390abadc445ea2527426a0e7a4495c19b74589204f9b71bcaafeb"}, + {file = "mmh3-4.1.0-cp311-cp311-win32.whl", hash = "sha256:1d3b1cdad7c71b7b88966301789a478af142bddcb3a2bee563f7a7d40519a00f"}, + {file = "mmh3-4.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0dc6dc32eb03727467da8e17deffe004fbb65e8b5ee2b502d36250d7a3f4e2ec"}, + {file = "mmh3-4.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:9ae3a5c1b32dda121c7dc26f9597ef7b01b4c56a98319a7fe86c35b8bc459ae6"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0033d60c7939168ef65ddc396611077a7268bde024f2c23bdc283a19123f9e9c"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d6af3e2287644b2b08b5924ed3a88c97b87b44ad08e79ca9f93d3470a54a41c5"}, + {file = "mmh3-4.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d82eb4defa245e02bb0b0dc4f1e7ee284f8d212633389c91f7fba99ba993f0a2"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba245e94b8d54765e14c2d7b6214e832557e7856d5183bc522e17884cab2f45d"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb04e2feeabaad6231e89cd43b3d01a4403579aa792c9ab6fdeef45cc58d4ec0"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e3b1a27def545ce11e36158ba5d5390cdbc300cfe456a942cc89d649cf7e3b2"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce0ab79ff736d7044e5e9b3bfe73958a55f79a4ae672e6213e92492ad5e734d5"}, + {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b02268be6e0a8eeb8a924d7db85f28e47344f35c438c1e149878bb1c47b1cd3"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:deb887f5fcdaf57cf646b1e062d56b06ef2f23421c80885fce18b37143cba828"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99dd564e9e2b512eb117bd0cbf0f79a50c45d961c2a02402787d581cec5448d5"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:08373082dfaa38fe97aa78753d1efd21a1969e51079056ff552e687764eafdfe"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:54b9c6a2ea571b714e4fe28d3e4e2db37abfd03c787a58074ea21ee9a8fd1740"}, + {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a7b1edf24c69e3513f879722b97ca85e52f9032f24a52284746877f6a7304086"}, + {file = "mmh3-4.1.0-cp312-cp312-win32.whl", hash = "sha256:411da64b951f635e1e2284b71d81a5a83580cea24994b328f8910d40bed67276"}, + {file = "mmh3-4.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bebc3ecb6ba18292e3d40c8712482b4477abd6981c2ebf0e60869bd90f8ac3a9"}, + {file = "mmh3-4.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:168473dd608ade6a8d2ba069600b35199a9af837d96177d3088ca91f2b3798e3"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:372f4b7e1dcde175507640679a2a8790185bb71f3640fc28a4690f73da986a3b"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:438584b97f6fe13e944faf590c90fc127682b57ae969f73334040d9fa1c7ffa5"}, + {file = "mmh3-4.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6e27931b232fc676675fac8641c6ec6b596daa64d82170e8597f5a5b8bdcd3b6"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:571a92bad859d7b0330e47cfd1850b76c39b615a8d8e7aa5853c1f971fd0c4b1"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a69d6afe3190fa08f9e3a58e5145549f71f1f3fff27bd0800313426929c7068"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afb127be0be946b7630220908dbea0cee0d9d3c583fa9114a07156f98566dc28"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940d86522f36348ef1a494cbf7248ab3f4a1638b84b59e6c9e90408bd11ad729"}, + {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3dcccc4935686619a8e3d1f7b6e97e3bd89a4a796247930ee97d35ea1a39341"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01bb9b90d61854dfc2407c5e5192bfb47222d74f29d140cb2dd2a69f2353f7cc"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bcb1b8b951a2c0b0fb8a5426c62a22557e2ffc52539e0a7cc46eb667b5d606a9"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6477a05d5e5ab3168e82e8b106e316210ac954134f46ec529356607900aea82a"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:da5892287e5bea6977364b15712a2573c16d134bc5fdcdd4cf460006cf849278"}, + {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:99180d7fd2327a6fffbaff270f760576839dc6ee66d045fa3a450f3490fda7f5"}, + {file = "mmh3-4.1.0-cp38-cp38-win32.whl", hash = "sha256:9b0d4f3949913a9f9a8fb1bb4cc6ecd52879730aab5ff8c5a3d8f5b593594b73"}, + {file = "mmh3-4.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:598c352da1d945108aee0c3c3cfdd0e9b3edef74108f53b49d481d3990402169"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:475d6d1445dd080f18f0f766277e1237fa2914e5fe3307a3b2a3044f30892103"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5ca07c41e6a2880991431ac717c2a049056fff497651a76e26fc22224e8b5732"}, + {file = "mmh3-4.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ebe052fef4bbe30c0548d12ee46d09f1b69035ca5208a7075e55adfe091be44"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaefd42e85afb70f2b855a011f7b4d8a3c7e19c3f2681fa13118e4d8627378c5"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0ae43caae5a47afe1b63a1ae3f0986dde54b5fb2d6c29786adbfb8edc9edfb"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6218666f74c8c013c221e7f5f8a693ac9cf68e5ac9a03f2373b32d77c48904de"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac59294a536ba447b5037f62d8367d7d93b696f80671c2c45645fa9f1109413c"}, + {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:086844830fcd1e5c84fec7017ea1ee8491487cfc877847d96f86f68881569d2e"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e42b38fad664f56f77f6fbca22d08450f2464baa68acdbf24841bf900eb98e87"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d08b790a63a9a1cde3b5d7d733ed97d4eb884bfbc92f075a091652d6bfd7709a"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:73ea4cc55e8aea28c86799ecacebca09e5f86500414870a8abaedfcbaf74d288"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f90938ff137130e47bcec8dc1f4ceb02f10178c766e2ef58a9f657ff1f62d124"}, + {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:aa1f13e94b8631c8cd53259250556edcf1de71738936b60febba95750d9632bd"}, + {file = "mmh3-4.1.0-cp39-cp39-win32.whl", hash = "sha256:a3b680b471c181490cf82da2142029edb4298e1bdfcb67c76922dedef789868d"}, + {file = "mmh3-4.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:fefef92e9c544a8dbc08f77a8d1b6d48006a750c4375bbcd5ff8199d761e263b"}, + {file = "mmh3-4.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:8e2c1f6a2b41723a4f82bd5a762a777836d29d664fc0095f17910bea0adfd4a6"}, + {file = "mmh3-4.1.0.tar.gz", hash = "sha256:a1cf25348b9acd229dda464a094d6170f47d2850a1fcb762a3b6172d2ce6ca4a"}, +] + +[package.extras] +test = ["mypy (>=1.0)", "pytest (>=7.0.0)"] + +[[package]] +name = "monotonic" +version = "1.6" +description = "An implementation of time.monotonic() for Python 2 & < 3.3" +optional = false +python-versions = "*" +files = [ + {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"}, + {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +optional = false +python-versions = "*" +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + [[package]] name = "multidict" version = "6.0.4" @@ -2197,6 +2682,64 @@ files = [ {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + +[[package]] +name = "onnxruntime" +version = "1.17.3" +description = "ONNX Runtime is a runtime accelerator for Machine Learning models" +optional = false +python-versions = "*" +files = [ + {file = "onnxruntime-1.17.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:d86dde9c0bb435d709e51bd25991c9fe5b9a5b168df45ce119769edc4d198b15"}, + {file = "onnxruntime-1.17.3-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d87b68bf931ac527b2d3c094ead66bb4381bac4298b65f46c54fe4d1e255865"}, + {file = "onnxruntime-1.17.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26e950cf0333cf114a155f9142e71da344d2b08dfe202763a403ae81cc02ebd1"}, + {file = "onnxruntime-1.17.3-cp310-cp310-win32.whl", hash = "sha256:0962a4d0f5acebf62e1f0bf69b6e0adf16649115d8de854c1460e79972324d68"}, + {file = "onnxruntime-1.17.3-cp310-cp310-win_amd64.whl", hash = "sha256:468ccb8a0faa25c681a41787b1594bf4448b0252d3efc8b62fd8b2411754340f"}, + {file = "onnxruntime-1.17.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e8cd90c1c17d13d47b89ab076471e07fb85467c01dcd87a8b8b5cdfbcb40aa51"}, + {file = "onnxruntime-1.17.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a058b39801baefe454eeb8acf3ada298c55a06a4896fafc224c02d79e9037f60"}, + {file = "onnxruntime-1.17.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f823d5eb4807007f3da7b27ca972263df6a1836e6f327384eb266274c53d05d"}, + {file = "onnxruntime-1.17.3-cp311-cp311-win32.whl", hash = "sha256:b66b23f9109e78ff2791628627a26f65cd335dcc5fbd67ff60162733a2f7aded"}, + {file = "onnxruntime-1.17.3-cp311-cp311-win_amd64.whl", hash = "sha256:570760ca53a74cdd751ee49f13de70d1384dcf73d9888b8deac0917023ccda6d"}, + {file = "onnxruntime-1.17.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:77c318178d9c16e9beadd9a4070d8aaa9f57382c3f509b01709f0f010e583b99"}, + {file = "onnxruntime-1.17.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23da8469049b9759082e22c41a444f44a520a9c874b084711b6343672879f50b"}, + {file = "onnxruntime-1.17.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2949730215af3f9289008b2e31e9bbef952012a77035b911c4977edea06f3f9e"}, + {file = "onnxruntime-1.17.3-cp312-cp312-win32.whl", hash = "sha256:6c7555a49008f403fb3b19204671efb94187c5085976ae526cb625f6ede317bc"}, + {file = "onnxruntime-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:58672cf20293a1b8a277a5c6c55383359fcdf6119b2f14df6ce3b140f5001c39"}, + {file = "onnxruntime-1.17.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:4395ba86e3c1e93c794a00619ef1aec597ab78f5a5039f3c6d2e9d0695c0a734"}, + {file = "onnxruntime-1.17.3-cp38-cp38-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdf354c04344ec38564fc22394e1fe08aa6d70d790df00159205a0055c4a4d3f"}, + {file = "onnxruntime-1.17.3-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a94b600b7af50e922d44b95a57981e3e35103c6e3693241a03d3ca204740bbda"}, + {file = "onnxruntime-1.17.3-cp38-cp38-win32.whl", hash = "sha256:5a335c76f9c002a8586c7f38bc20fe4b3725ced21f8ead835c3e4e507e42b2ab"}, + {file = "onnxruntime-1.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:8f56a86fbd0ddc8f22696ddeda0677b041381f4168a2ca06f712ef6ec6050d6d"}, + {file = "onnxruntime-1.17.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:e0ae39f5452278cd349520c296e7de3e90d62dc5b0157c6868e2748d7f28b871"}, + {file = "onnxruntime-1.17.3-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ff2dc012bd930578aff5232afd2905bf16620815f36783a941aafabf94b3702"}, + {file = "onnxruntime-1.17.3-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf6c37483782e4785019b56e26224a25e9b9a35b849d0169ce69189867a22bb1"}, + {file = "onnxruntime-1.17.3-cp39-cp39-win32.whl", hash = "sha256:351bf5a1140dcc43bfb8d3d1a230928ee61fcd54b0ea664c8e9a889a8e3aa515"}, + {file = "onnxruntime-1.17.3-cp39-cp39-win_amd64.whl", hash = "sha256:57a3de15778da8d6cc43fbf6cf038e1e746146300b5f0b1fbf01f6f795dc6440"}, +] + +[package.dependencies] +coloredlogs = "*" +flatbuffers = "*" +numpy = ">=1.21.6" +packaging = "*" +protobuf = "*" +sympy = "*" + [[package]] name = "openai" version = "1.10.0" @@ -2220,6 +2763,168 @@ typing-extensions = ">=4.7,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +[[package]] +name = "opentelemetry-api" +version = "1.24.0" +description = "OpenTelemetry Python API" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_api-1.24.0-py3-none-any.whl", hash = "sha256:0f2c363d98d10d1ce93330015ca7fd3a65f60be64e05e30f557c61de52c80ca2"}, + {file = "opentelemetry_api-1.24.0.tar.gz", hash = "sha256:42719f10ce7b5a9a73b10a4baf620574fb8ad495a9cbe5c18d76b75d8689c67e"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +importlib-metadata = ">=6.0,<=7.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.24.0" +description = "OpenTelemetry Protobuf encoding" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_exporter_otlp_proto_common-1.24.0-py3-none-any.whl", hash = "sha256:e51f2c9735054d598ad2df5d3eca830fecfb5b0bda0a2fa742c9c7718e12f641"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.24.0.tar.gz", hash = "sha256:5d31fa1ff976cacc38be1ec4e3279a3f88435c75b38b1f7a099a1faffc302461"}, +] + +[package.dependencies] +opentelemetry-proto = "1.24.0" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.24.0" +description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.24.0-py3-none-any.whl", hash = "sha256:f40d62aa30a0a43cc1657428e59fcf82ad5f7ea8fff75de0f9d9cb6f739e0a3b"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.24.0.tar.gz", hash = "sha256:217c6e30634f2c9797999ea9da29f7300479a94a610139b9df17433f915e7baa"}, +] + +[package.dependencies] +deprecated = ">=1.2.6" +googleapis-common-protos = ">=1.52,<2.0" +grpcio = ">=1.0.0,<2.0.0" +opentelemetry-api = ">=1.15,<2.0" +opentelemetry-exporter-otlp-proto-common = "1.24.0" +opentelemetry-proto = "1.24.0" +opentelemetry-sdk = ">=1.24.0,<1.25.0" + +[package.extras] +test = ["pytest-grpc"] + +[[package]] +name = "opentelemetry-instrumentation" +version = "0.45b0" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation-0.45b0-py3-none-any.whl", hash = "sha256:06c02e2c952c1b076e8eaedf1b82f715e2937ba7eeacab55913dd434fbcec258"}, + {file = "opentelemetry_instrumentation-0.45b0.tar.gz", hash = "sha256:6c47120a7970bbeb458e6a73686ee9ba84b106329a79e4a4a66761f933709c7e"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +setuptools = ">=16.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-asgi" +version = "0.45b0" +description = "ASGI instrumentation for OpenTelemetry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_asgi-0.45b0-py3-none-any.whl", hash = "sha256:8be1157ed62f0db24e45fdf7933c530c4338bd025c5d4af7830e903c0756021b"}, + {file = "opentelemetry_instrumentation_asgi-0.45b0.tar.gz", hash = "sha256:97f55620f163fd3d20323e9fd8dc3aacc826c03397213ff36b877e0f4b6b08a6"}, +] + +[package.dependencies] +asgiref = ">=3.0,<4.0" +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.45b0" +opentelemetry-semantic-conventions = "0.45b0" +opentelemetry-util-http = "0.45b0" + +[package.extras] +instruments = ["asgiref (>=3.0,<4.0)"] + +[[package]] +name = "opentelemetry-instrumentation-fastapi" +version = "0.45b0" +description = "OpenTelemetry FastAPI Instrumentation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_instrumentation_fastapi-0.45b0-py3-none-any.whl", hash = "sha256:77d9c123a363129148f5f66d44094f3d67aaaa2b201396d94782b4a7f9ce4314"}, + {file = "opentelemetry_instrumentation_fastapi-0.45b0.tar.gz", hash = "sha256:5a6b91e1c08a01601845fcfcfdefd0a2aecdb3c356d4a436a3210cb58c21487e"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.45b0" +opentelemetry-instrumentation-asgi = "0.45b0" +opentelemetry-semantic-conventions = "0.45b0" +opentelemetry-util-http = "0.45b0" + +[package.extras] +instruments = ["fastapi (>=0.58,<1.0)"] + +[[package]] +name = "opentelemetry-proto" +version = "1.24.0" +description = "OpenTelemetry Python Proto" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_proto-1.24.0-py3-none-any.whl", hash = "sha256:bcb80e1e78a003040db71ccf83f2ad2019273d1e0828089d183b18a1476527ce"}, + {file = "opentelemetry_proto-1.24.0.tar.gz", hash = "sha256:ff551b8ad63c6cabb1845ce217a6709358dfaba0f75ea1fa21a61ceddc78cab8"}, +] + +[package.dependencies] +protobuf = ">=3.19,<5.0" + +[[package]] +name = "opentelemetry-sdk" +version = "1.24.0" +description = "OpenTelemetry Python SDK" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_sdk-1.24.0-py3-none-any.whl", hash = "sha256:fa731e24efe832e98bcd90902085b359dcfef7d9c9c00eb5b9a18587dae3eb59"}, + {file = "opentelemetry_sdk-1.24.0.tar.gz", hash = "sha256:75bc0563affffa827700e0f4f4a68e1e257db0df13372344aebc6f8a64cde2e5"}, +] + +[package.dependencies] +opentelemetry-api = "1.24.0" +opentelemetry-semantic-conventions = "0.45b0" +typing-extensions = ">=3.7.4" + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.45b0" +description = "OpenTelemetry Semantic Conventions" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_semantic_conventions-0.45b0-py3-none-any.whl", hash = "sha256:a4a6fb9a7bacd9167c082aa4681009e9acdbfa28ffb2387af50c2fef3d30c864"}, + {file = "opentelemetry_semantic_conventions-0.45b0.tar.gz", hash = "sha256:7c84215a44ac846bc4b8e32d5e78935c5c43482e491812a0bb8aaf87e4d92118"}, +] + +[[package]] +name = "opentelemetry-util-http" +version = "0.45b0" +description = "Web util for OpenTelemetry" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opentelemetry_util_http-0.45b0-py3-none-any.whl", hash = "sha256:6628868b501b3004e1860f976f410eeb3d3499e009719d818000f24ce17b6e33"}, + {file = "opentelemetry_util_http-0.45b0.tar.gz", hash = "sha256:4ce08b6a7d52dd7c96b7705b5b4f06fdb6aa3eac1233b3b0bfef8a0cab9a92cd"}, +] + [[package]] name = "orjson" version = "3.10.0" @@ -2280,6 +2985,17 @@ files = [ {file = "orjson-3.10.0.tar.gz", hash = "sha256:ba4d8cac5f2e2cff36bea6b6481cdb92b38c202bcec603d6f5ff91960595a1ed"}, ] +[[package]] +name = "overrides" +version = "7.7.0" +description = "A decorator to automatically detect mismatch when overriding a method." +optional = false +python-versions = ">=3.6" +files = [ + {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, + {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, +] + [[package]] name = "packaging" version = "23.2" @@ -2424,6 +3140,29 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "posthog" +version = "3.5.0" +description = "Integrate PostHog into any python application." +optional = false +python-versions = "*" +files = [ + {file = "posthog-3.5.0-py2.py3-none-any.whl", hash = "sha256:3c672be7ba6f95d555ea207d4486c171d06657eb34b3ce25eb043bfe7b6b5b76"}, + {file = "posthog-3.5.0.tar.gz", hash = "sha256:8f7e3b2c6e8714d0c0c542a2109b83a7549f63b7113a133ab2763a89245ef2ef"}, +] + +[package.dependencies] +backoff = ">=1.10.0" +monotonic = ">=1.5" +python-dateutil = ">2.1" +requests = ">=2.7,<3.0" +six = ">=1.5" + +[package.extras] +dev = ["black", "flake8", "flake8-print", "isort", "pre-commit"] +sentry = ["django", "sentry-sdk"] +test = ["coverage", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)", "pylint", "pytest", "pytest-timeout"] + [[package]] name = "proto-plus" version = "1.23.0" @@ -2542,6 +3281,53 @@ files = [ {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, ] +[[package]] +name = "pulsar-client" +version = "3.5.0" +description = "Apache Pulsar Python client library" +optional = false +python-versions = "*" +files = [ + {file = "pulsar_client-3.5.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:c18552edb2f785de85280fe624bc507467152bff810fc81d7660fa2dfa861f38"}, + {file = "pulsar_client-3.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18d438e456c146f01be41ef146f649dedc8f7bc714d9eaef94cff2e34099812b"}, + {file = "pulsar_client-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18a26a0719841103c7a89eb1492c4a8fedf89adaa386375baecbb4fa2707e88f"}, + {file = "pulsar_client-3.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ab0e1605dc5f44a126163fd06cd0a768494ad05123f6e0de89a2c71d6e2d2319"}, + {file = "pulsar_client-3.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cdef720891b97656fdce3bf5913ea7729b2156b84ba64314f432c1e72c6117fa"}, + {file = "pulsar_client-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:a42544e38773191fe550644a90e8050579476bb2dcf17ac69a4aed62a6cb70e7"}, + {file = "pulsar_client-3.5.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:fd94432ea5d398ea78f8f2e09a217ec5058d26330c137a22690478c031e116da"}, + {file = "pulsar_client-3.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6252ae462e07ece4071213fdd9c76eab82ca522a749f2dc678037d4cbacd40b"}, + {file = "pulsar_client-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03b4d440b2d74323784328b082872ee2f206c440b5d224d7941eb3c083ec06c6"}, + {file = "pulsar_client-3.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f60af840b8d64a2fac5a0c1ce6ae0ddffec5f42267c6ded2c5e74bad8345f2a1"}, + {file = "pulsar_client-3.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2277a447c3b7f6571cb1eb9fc5c25da3fdd43d0b2fb91cf52054adfadc7d6842"}, + {file = "pulsar_client-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:f20f3e9dd50db2a37059abccad42078b7a4754b8bc1d3ae6502e71c1ad2209f0"}, + {file = "pulsar_client-3.5.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:d61f663d85308e12f44033ba95af88730f581a7e8da44f7a5c080a3aaea4878d"}, + {file = "pulsar_client-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1ba0be25b6f747bcb28102b7d906ec1de48dc9f1a2d9eacdcc6f44ab2c9e17"}, + {file = "pulsar_client-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a181e3e60ac39df72ccb3c415d7aeac61ad0286497a6e02739a560d5af28393a"}, + {file = "pulsar_client-3.5.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3c72895ff7f51347e4f78b0375b2213fa70dd4790bbb78177b4002846f1fd290"}, + {file = "pulsar_client-3.5.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:547dba1b185a17eba915e51d0a3aca27c80747b6187e5cd7a71a3ca33921decc"}, + {file = "pulsar_client-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:443b786eed96bc86d2297a6a42e79f39d1abf217ec603e0bd303f3488c0234af"}, + {file = "pulsar_client-3.5.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:15b58f5d759dd6166db8a2d90ed05a38063b05cda76c36d190d86ef5c9249397"}, + {file = "pulsar_client-3.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af34bfe813dddf772a8a298117fa0a036ee963595d8bc8f00d969a0329ae6ed9"}, + {file = "pulsar_client-3.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27a0fec1dd74e1367d3742ce16679c1807994df60f5e666f440cf39323938fad"}, + {file = "pulsar_client-3.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbcd26ef9c03f96fb9cd91baec3bbd3c4b997834eb3556670d31f41cc25b5f64"}, + {file = "pulsar_client-3.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:afea1d0b6e793fd56e56463145751ff3aa79fdcd5b26e90d0da802a1bbabe07e"}, + {file = "pulsar_client-3.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:da1ab2fb1bef64b966e9403a0a186ebc90368d99e054ce2cae5b1128478f4ef4"}, + {file = "pulsar_client-3.5.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:9ad5dcc0eb8d2a7c0fb8e1fa146a0c6d4bdaf934f1169080b2c64b2f0573e086"}, + {file = "pulsar_client-3.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5870c6805b1a57962ed908d1173e97e13470415998393925c86a43694420389"}, + {file = "pulsar_client-3.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29cb5fedb969895b78301dc00a979133e69940812b8332e4de948bb0ad3db7cb"}, + {file = "pulsar_client-3.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e53c74bfa59b20c66adea95023169060f5048dd8d843e6ef9cd3b8ee2d23e93b"}, + {file = "pulsar_client-3.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99dbadb13967f1add57010971ed36b5a77d24afcdaea01960d0e55e56cf4ba6f"}, + {file = "pulsar_client-3.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:058887661d438796f42307dcc8054c84dea88a37683dae36498b95d7e1c39b37"}, +] + +[package.dependencies] +certifi = "*" + +[package.extras] +all = ["apache-bookkeeper-client (>=4.16.1)", "fastavro (>=1.9.2)", "grpcio (>=1.60.0)", "prometheus-client", "protobuf (>=3.6.1,<=3.20.3)", "ratelimit"] +avro = ["fastavro (>=1.9.2)"] +functions = ["apache-bookkeeper-client (>=4.16.1)", "grpcio (>=1.60.0)", "prometheus-client", "protobuf (>=3.6.1,<=3.20.3)", "ratelimit"] + [[package]] name = "pyasn1" version = "0.5.1" @@ -2650,6 +3436,21 @@ requests = ">=2.14.0" typing-extensions = ">=4.0.0" urllib3 = ">=1.26.0" +[[package]] +name = "pygments" +version = "2.17.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, +] + +[package.extras] +plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "pyjwt" version = "2.8.0" @@ -2696,6 +3497,41 @@ cffi = ">=1.4.1" docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] +[[package]] +name = "pypika" +version = "0.48.9" +description = "A SQL query builder API for Python" +optional = false +python-versions = "*" +files = [ + {file = "PyPika-0.48.9.tar.gz", hash = "sha256:838836a61747e7c8380cd1b7ff638694b7a7335345d0f559b04b2cd832ad5378"}, +] + +[[package]] +name = "pyproject-hooks" +version = "1.0.0" +description = "Wrappers to call pyproject.toml-based build backend hooks." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, + {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, +] + +[package.dependencies] +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} + +[[package]] +name = "pyreadline3" +version = "3.4.1" +description = "A python implementation of GNU readline." +optional = false +python-versions = "*" +files = [ + {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, + {file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"}, +] + [[package]] name = "pytest" version = "7.4.3" @@ -2844,6 +3680,20 @@ files = [ lxml = ">=3.1.0" typing-extensions = "*" +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "python-iso639" version = "2024.2.7" @@ -3163,6 +4013,42 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=3.4" +files = [ + {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"}, + {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "rich" +version = "13.7.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, + {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "rsa" version = "4.9" @@ -3303,6 +4189,17 @@ numpy = ">=1.14" docs = ["matplotlib", "numpydoc (==1.1.*)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"] test = ["pytest", "pytest-cov"] +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] + [[package]] name = "six" version = "1.16.0" @@ -3469,6 +4366,20 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] +[[package]] +name = "sympy" +version = "1.12" +description = "Computer algebra system (CAS) in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, + {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, +] + +[package.dependencies] +mpmath = ">=0.19" + [[package]] name = "tabulate" version = "0.9.0" @@ -3718,6 +4629,23 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] +[[package]] +name = "typer" +version = "0.12.3" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.7" +files = [ + {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, + {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, +] + +[package.dependencies] +click = ">=8.0.0" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + [[package]] name = "types-protobuf" version = "4.24.0.20240106" @@ -3962,12 +4890,63 @@ files = [ [package.dependencies] click = ">=7.0" +colorama = {version = ">=0.4", optional = true, markers = "sys_platform == \"win32\" and extra == \"standard\""} h11 = ">=0.8" +httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""} +python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} +uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} +watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} +websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} [package.extras] standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] +[[package]] +name = "uvloop" +version = "0.19.0" +description = "Fast implementation of asyncio event loop on top of libuv" +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de4313d7f575474c8f5a12e163f6d89c0a878bc49219641d49e6f1444369a90e"}, + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5588bd21cf1fcf06bded085f37e43ce0e00424197e7c10e77afd4bbefffef428"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b1fd71c3843327f3bbc3237bedcdb6504fd50368ab3e04d0410e52ec293f5b8"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a05128d315e2912791de6088c34136bfcdd0c7cbc1cf85fd6fd1bb321b7c849"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cd81bdc2b8219cb4b2556eea39d2e36bfa375a2dd021404f90a62e44efaaf957"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f17766fb6da94135526273080f3455a112f82570b2ee5daa64d682387fe0dcd"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ce6b0af8f2729a02a5d1575feacb2a94fc7b2e983868b009d51c9a9d2149bef"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:31e672bb38b45abc4f26e273be83b72a0d28d074d5b370fc4dcf4c4eb15417d2"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:570fc0ed613883d8d30ee40397b79207eedd2624891692471808a95069a007c1"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5138821e40b0c3e6c9478643b4660bd44372ae1e16a322b8fc07478f92684e24"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:91ab01c6cd00e39cde50173ba4ec68a1e578fee9279ba64f5221810a9e786533"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:47bf3e9312f63684efe283f7342afb414eea4d3011542155c7e625cd799c3b12"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:da8435a3bd498419ee8c13c34b89b5005130a476bda1d6ca8cfdde3de35cd650"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:02506dc23a5d90e04d4f65c7791e65cf44bd91b37f24cfc3ef6cf2aff05dc7ec"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693049be9d36fef81741fddb3f441673ba12a34a704e7b4361efb75cf30befc"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7010271303961c6f0fe37731004335401eb9075a12680738731e9c92ddd96ad6"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5daa304d2161d2918fa9a17d5635099a2f78ae5b5960e742b2fcfbb7aefaa593"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7207272c9520203fea9b93843bb775d03e1cf88a80a936ce760f60bb5add92f3"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:78ab247f0b5671cc887c31d33f9b3abfb88d2614b84e4303f1a63b46c046c8bd"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:472d61143059c84947aa8bb74eabbace30d577a03a1805b77933d6bd13ddebbd"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45bf4c24c19fb8a50902ae37c5de50da81de4922af65baf760f7c0c42e1088be"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271718e26b3e17906b28b67314c45d19106112067205119dddbd834c2b7ce797"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:34175c9fd2a4bc3adc1380e1261f60306344e3407c20a4d684fd5f3be010fa3d"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e27f100e1ff17f6feeb1f33968bc185bf8ce41ca557deee9d9bbbffeb72030b7"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13dfdf492af0aa0a0edf66807d2b465607d11c4fa48f4a1fd41cbea5b18e8e8b"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e3d4e85ac060e2342ff85e90d0c04157acb210b9ce508e784a944f852a40e67"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca4956c9ab567d87d59d49fa3704cf29e37109ad348f2d5223c9bf761a332e7"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f467a5fd23b4fc43ed86342641f3936a68ded707f4627622fa3f82a120e18256"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:492e2c32c2af3f971473bc22f086513cedfc66a130756145a931a90c3958cb17"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2df95fca285a9f5bfe730e51945ffe2fa71ccbfdde3b0da5772b4ee4f2e770d5"}, + {file = "uvloop-0.19.0.tar.gz", hash = "sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd"}, +] + +[package.extras] +docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] +test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] + [[package]] name = "watchdog" version = "3.0.0" @@ -4007,6 +4986,190 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] +[[package]] +name = "watchfiles" +version = "0.21.0" +description = "Simple, modern and high performance file watching and code reload in python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "watchfiles-0.21.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:27b4035013f1ea49c6c0b42d983133b136637a527e48c132d368eb19bf1ac6aa"}, + {file = "watchfiles-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c81818595eff6e92535ff32825f31c116f867f64ff8cdf6562cd1d6b2e1e8f3e"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c107ea3cf2bd07199d66f156e3ea756d1b84dfd43b542b2d870b77868c98c03"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d9ac347653ebd95839a7c607608703b20bc07e577e870d824fa4801bc1cb124"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5eb86c6acb498208e7663ca22dbe68ca2cf42ab5bf1c776670a50919a56e64ab"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f564bf68404144ea6b87a78a3f910cc8de216c6b12a4cf0b27718bf4ec38d303"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d0f32ebfaa9c6011f8454994f86108c2eb9c79b8b7de00b36d558cadcedaa3d"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d45d9b699ecbac6c7bd8e0a2609767491540403610962968d258fd6405c17c"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:aff06b2cac3ef4616e26ba17a9c250c1fe9dd8a5d907d0193f84c499b1b6e6a9"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d9792dff410f266051025ecfaa927078b94cc7478954b06796a9756ccc7e14a9"}, + {file = "watchfiles-0.21.0-cp310-none-win32.whl", hash = "sha256:214cee7f9e09150d4fb42e24919a1e74d8c9b8a9306ed1474ecaddcd5479c293"}, + {file = "watchfiles-0.21.0-cp310-none-win_amd64.whl", hash = "sha256:1ad7247d79f9f55bb25ab1778fd47f32d70cf36053941f07de0b7c4e96b5d235"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:668c265d90de8ae914f860d3eeb164534ba2e836811f91fecc7050416ee70aa7"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a23092a992e61c3a6a70f350a56db7197242f3490da9c87b500f389b2d01eef"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e7941bbcfdded9c26b0bf720cb7e6fd803d95a55d2c14b4bd1f6a2772230c586"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11cd0c3100e2233e9c53106265da31d574355c288e15259c0d40a4405cbae317"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d78f30cbe8b2ce770160d3c08cff01b2ae9306fe66ce899b73f0409dc1846c1b"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6674b00b9756b0af620aa2a3346b01f8e2a3dc729d25617e1b89cf6af4a54eb1"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd7ac678b92b29ba630d8c842d8ad6c555abda1b9ef044d6cc092dacbfc9719d"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c873345680c1b87f1e09e0eaf8cf6c891b9851d8b4d3645e7efe2ec20a20cc7"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49f56e6ecc2503e7dbe233fa328b2be1a7797d31548e7a193237dcdf1ad0eee0"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:02d91cbac553a3ad141db016e3350b03184deaafeba09b9d6439826ee594b365"}, + {file = "watchfiles-0.21.0-cp311-none-win32.whl", hash = "sha256:ebe684d7d26239e23d102a2bad2a358dedf18e462e8808778703427d1f584400"}, + {file = "watchfiles-0.21.0-cp311-none-win_amd64.whl", hash = "sha256:4566006aa44cb0d21b8ab53baf4b9c667a0ed23efe4aaad8c227bfba0bf15cbe"}, + {file = "watchfiles-0.21.0-cp311-none-win_arm64.whl", hash = "sha256:c550a56bf209a3d987d5a975cdf2063b3389a5d16caf29db4bdddeae49f22078"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:51ddac60b96a42c15d24fbdc7a4bfcd02b5a29c047b7f8bf63d3f6f5a860949a"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:511f0b034120cd1989932bf1e9081aa9fb00f1f949fbd2d9cab6264916ae89b1"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cfb92d49dbb95ec7a07511bc9efb0faff8fe24ef3805662b8d6808ba8409a71a"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f92944efc564867bbf841c823c8b71bb0be75e06b8ce45c084b46411475a915"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:642d66b75eda909fd1112d35c53816d59789a4b38c141a96d62f50a3ef9b3360"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d23bcd6c8eaa6324fe109d8cac01b41fe9a54b8c498af9ce464c1aeeb99903d6"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18d5b4da8cf3e41895b34e8c37d13c9ed294954907929aacd95153508d5d89d7"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b8d1eae0f65441963d805f766c7e9cd092f91e0c600c820c764a4ff71a0764c"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1fd9a5205139f3c6bb60d11f6072e0552f0a20b712c85f43d42342d162be1235"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a1e3014a625bcf107fbf38eece0e47fa0190e52e45dc6eee5a8265ddc6dc5ea7"}, + {file = "watchfiles-0.21.0-cp312-none-win32.whl", hash = "sha256:9d09869f2c5a6f2d9df50ce3064b3391d3ecb6dced708ad64467b9e4f2c9bef3"}, + {file = "watchfiles-0.21.0-cp312-none-win_amd64.whl", hash = "sha256:18722b50783b5e30a18a8a5db3006bab146d2b705c92eb9a94f78c72beb94094"}, + {file = "watchfiles-0.21.0-cp312-none-win_arm64.whl", hash = "sha256:a3b9bec9579a15fb3ca2d9878deae789df72f2b0fdaf90ad49ee389cad5edab6"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:4ea10a29aa5de67de02256a28d1bf53d21322295cb00bd2d57fcd19b850ebd99"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:40bca549fdc929b470dd1dbfcb47b3295cb46a6d2c90e50588b0a1b3bd98f429"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9b37a7ba223b2f26122c148bb8d09a9ff312afca998c48c725ff5a0a632145f7"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec8c8900dc5c83650a63dd48c4d1d245343f904c4b64b48798c67a3767d7e165"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ad3fe0a3567c2f0f629d800409cd528cb6251da12e81a1f765e5c5345fd0137"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d353c4cfda586db2a176ce42c88f2fc31ec25e50212650c89fdd0f560ee507b"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:83a696da8922314ff2aec02987eefb03784f473281d740bf9170181829133765"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a03651352fc20975ee2a707cd2d74a386cd303cc688f407296064ad1e6d1562"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3ad692bc7792be8c32918c699638b660c0de078a6cbe464c46e1340dadb94c19"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06247538e8253975bdb328e7683f8515ff5ff041f43be6c40bff62d989b7d0b0"}, + {file = "watchfiles-0.21.0-cp38-none-win32.whl", hash = "sha256:9a0aa47f94ea9a0b39dd30850b0adf2e1cd32a8b4f9c7aa443d852aacf9ca214"}, + {file = "watchfiles-0.21.0-cp38-none-win_amd64.whl", hash = "sha256:8d5f400326840934e3507701f9f7269247f7c026d1b6cfd49477d2be0933cfca"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7f762a1a85a12cc3484f77eee7be87b10f8c50b0b787bb02f4e357403cad0c0e"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6e9be3ef84e2bb9710f3f777accce25556f4a71e15d2b73223788d528fcc2052"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4c48a10d17571d1275701e14a601e36959ffada3add8cdbc9e5061a6e3579a5d"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c889025f59884423428c261f212e04d438de865beda0b1e1babab85ef4c0f01"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66fac0c238ab9a2e72d026b5fb91cb902c146202bbd29a9a1a44e8db7b710b6f"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4a21f71885aa2744719459951819e7bf5a906a6448a6b2bbce8e9cc9f2c8128"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c9198c989f47898b2c22201756f73249de3748e0fc9de44adaf54a8b259cc0c"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f57c4461cd24fda22493109c45b3980863c58a25b8bec885ca8bea6b8d4b28"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:853853cbf7bf9408b404754b92512ebe3e3a83587503d766d23e6bf83d092ee6"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d5b1dc0e708fad9f92c296ab2f948af403bf201db8fb2eb4c8179db143732e49"}, + {file = "watchfiles-0.21.0-cp39-none-win32.whl", hash = "sha256:59137c0c6826bd56c710d1d2bda81553b5e6b7c84d5a676747d80caf0409ad94"}, + {file = "watchfiles-0.21.0-cp39-none-win_amd64.whl", hash = "sha256:6cb8fdc044909e2078c248986f2fc76f911f72b51ea4a4fbbf472e01d14faa58"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:ab03a90b305d2588e8352168e8c5a1520b721d2d367f31e9332c4235b30b8994"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:927c589500f9f41e370b0125c12ac9e7d3a2fd166b89e9ee2828b3dda20bfe6f"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd467213195e76f838caf2c28cd65e58302d0254e636e7c0fca81efa4a2e62c"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02b73130687bc3f6bb79d8a170959042eb56eb3a42df3671c79b428cd73f17cc"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:08dca260e85ffae975448e344834d765983237ad6dc308231aa16e7933db763e"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3ccceb50c611c433145502735e0370877cced72a6c70fd2410238bcbc7fe51d8"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57d430f5fb63fea141ab71ca9c064e80de3a20b427ca2febcbfcef70ff0ce895"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dd5fad9b9c0dd89904bbdea978ce89a2b692a7ee8a0ce19b940e538c88a809c"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:be6dd5d52b73018b21adc1c5d28ac0c68184a64769052dfeb0c5d9998e7f56a2"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b3cab0e06143768499384a8a5efb9c4dc53e19382952859e4802f294214f36ec"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6ed10c2497e5fedadf61e465b3ca12a19f96004c15dcffe4bd442ebadc2d85"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43babacef21c519bc6631c5fce2a61eccdfc011b4bcb9047255e9620732c8097"}, + {file = "watchfiles-0.21.0.tar.gz", hash = "sha256:c76c635fabf542bb78524905718c39f736a98e5ab25b23ec6d4abede1a85a6a3"}, +] + +[package.dependencies] +anyio = ">=3.0.0" + +[[package]] +name = "websocket-client" +version = "1.7.0" +description = "WebSocket client for Python with low level API options" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, + {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, +] + +[package.extras] +docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] +optional = ["python-socks", "wsaccel"] +test = ["websockets"] + +[[package]] +name = "websockets" +version = "12.0" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, + {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, + {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, + {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, + {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, + {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, + {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, + {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, + {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, + {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, + {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, + {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, + {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, + {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, + {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, + {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, + {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, + {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, + {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, + {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, + {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, + {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, + {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, + {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, + {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, +] + [[package]] name = "wikipedia" version = "1.4.0" @@ -4214,7 +5377,22 @@ files = [ idna = ">=2.0" multidict = ">=4.0" +[[package]] +name = "zipp" +version = "3.18.1" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.8" +files = [ + {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"}, + {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + [metadata] lock-version = "2.0" python-versions = "^3.9.0,<3.12" -content-hash = "2d8bfc1948b224006ffb1bf4dd36ab9aff037088d24648514a069d3f78a356dd" +content-hash = "d173115dd8349b20d60b475c229549fdfc9083ecc643f43a46a770223d04cdab" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index d6b9a1bd..e8e1aaf8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -44,6 +44,7 @@ asyncpg = "^0.29.0" langchain-core = "^0.1.42" pyjwt = {extras = ["crypto"], version = "^2.8.0"} langchain-anthropic = "^0.1.8" +langchain-chroma = "^0.1.0" [tool.poetry.group.dev.dependencies] uvicorn = "^0.23.2" From 1df107610703977f445e6b59d735bba3b54ef9f9 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Wed, 17 Apr 2024 12:14:31 +0400 Subject: [PATCH 08/28] Update checkpointer. --- .gitignore | 3 +- backend/app/api/assistants.py | 4 +- backend/app/api/threads.py | 11 ++- backend/app/checkpoint.py | 152 +++++++++++++++++----------------- backend/app/server.py | 2 +- backend/app/storage.py | 56 ++++++++----- 6 files changed, 122 insertions(+), 106 deletions(-) diff --git a/.gitignore b/.gitignore index c7ace088..373a8669 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,5 @@ pnpm-debug.log* # Local db files -chroma_db/ \ No newline at end of file +chroma_db/ +opengpts.db \ No newline at end of file diff --git a/backend/app/api/assistants.py b/backend/app/api/assistants.py index 090b917d..04b6c82e 100644 --- a/backend/app/api/assistants.py +++ b/backend/app/api/assistants.py @@ -37,7 +37,7 @@ async def list_public_assistants( ] = None, ) -> List[Assistant]: """List all public assistants.""" - return await storage.list_public_assistants( + return storage.list_public_assistants( FEATURED_PUBLIC_ASSISTANTS + ([shared_id] if shared_id else []) ) @@ -76,7 +76,7 @@ async def upsert_assistant( payload: AssistantPayload, ) -> Assistant: """Create or update an assistant.""" - return await storage.put_assistant( + return storage.put_assistant( user["user_id"], aid, name=payload.name, diff --git a/backend/app/api/threads.py b/backend/app/api/threads.py index 07b6ffd6..fee713c5 100644 --- a/backend/app/api/threads.py +++ b/backend/app/api/threads.py @@ -1,4 +1,3 @@ -import asyncio from typing import Annotated, Any, Dict, List, Sequence, Union from uuid import uuid4 @@ -32,7 +31,7 @@ class ThreadPostRequest(BaseModel): @router.get("/") async def list_threads(user: AuthedUser) -> List[Thread]: """List all threads for the current user.""" - return await storage.list_threads(user["user_id"]) + return storage.list_threads(user["user_id"]) @router.get("/{tid}/state") @@ -42,7 +41,7 @@ async def get_thread_state( ): """Get state for a thread.""" thread = storage.get_thread(user["user_id"], tid) - state = await storage.get_thread_state(user["user_id"], tid) + state = storage.get_thread_state(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return state @@ -58,7 +57,7 @@ async def add_thread_state( thread = storage.get_thread(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") - return await storage.update_thread_state(user["user_id"], tid, payload.values) + return storage.update_thread_state(user["user_id"], tid, payload.values) @router.get("/{tid}/history") @@ -68,7 +67,7 @@ async def get_thread_history( ): """Get all past states for a thread.""" thread = storage.get_thread(user["user_id"], tid) - history = await storage.get_thread_history(user["user_id"], tid) + history = storage.get_thread_history(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return history @@ -121,5 +120,5 @@ async def delete_thread( tid: ThreadID, ): """Delete a thread by ID.""" - await storage.delete_thread(user["user_id"], tid) + storage.delete_thread(user["user_id"], tid) return {"status": "ok"} diff --git a/backend/app/checkpoint.py b/backend/app/checkpoint.py index 4bda5bd3..90f7d872 100644 --- a/backend/app/checkpoint.py +++ b/backend/app/checkpoint.py @@ -1,13 +1,16 @@ import pickle import sqlite3 from contextlib import contextmanager -from datetime import datetime from typing import Iterator, Optional from langchain_core.messages import BaseMessage from langchain_core.runnables import ConfigurableFieldSpec, RunnableConfig from langgraph.checkpoint import BaseCheckpointSaver -from langgraph.checkpoint.base import Checkpoint, CheckpointThreadTs, CheckpointTuple +from langgraph.checkpoint.base import ( + Checkpoint, + CheckpointThreadTs, + CheckpointTuple, +) @contextmanager @@ -46,99 +49,98 @@ def config_specs(self) -> list[ConfigurableFieldSpec]: CheckpointThreadTs, ] - # Adapting the get method - def get(self, config: RunnableConfig) -> Optional[Checkpoint]: - # Implementation adapted for SQLite - raise NotImplementedError - - # Adapting the put method - def put(self, config: RunnableConfig, checkpoint: Checkpoint) -> RunnableConfig: - # Implementation adapted for SQLite - raise NotImplementedError - - # Adapting the alist method - def alist(self, config: RunnableConfig) -> Iterator[CheckpointTuple]: - thread_id = config["configurable"]["thread_id"] + @contextmanager + def cursor(self, transaction: bool = True): with _connect() as conn: - cursor = conn.cursor() - cursor.execute( - "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC", - (thread_id,), - ) - for value in cursor.fetchall(): - yield CheckpointTuple( - { - "configurable": { - "thread_id": thread_id, - "thread_ts": value[1], - } - }, - loads(value[0]), - { - "configurable": { - "thread_id": thread_id, - "thread_ts": value[2], - } - } - if value[2] - else None, - ) + cur = conn.cursor() + try: + yield cur + finally: + if transaction: + conn.commit() + cur.close() - # Adapting the aget_tuple method - def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: - thread_id = config["configurable"]["thread_id"] - thread_ts = config["configurable"].get("thread_ts") - with _connect() as conn: - cursor = conn.cursor() - if thread_ts: - cursor.execute( + def get_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: + with self.cursor(transaction=False) as cur: + if config["configurable"].get("thread_ts"): + cur.execute( "SELECT checkpoint, parent_ts FROM checkpoints WHERE thread_id = ? AND thread_ts = ?", - (thread_id, datetime.fromisoformat(thread_ts)), + ( + config["configurable"]["thread_id"], + config["configurable"]["thread_ts"], + ), ) + if value := cur.fetchone(): + return CheckpointTuple( + config, + loads(value[0]), + { + "configurable": { + "thread_id": config["configurable"]["thread_id"], + "thread_ts": value[1], + } + } + if value[1] + else None, + ) else: - cursor.execute( - "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC LIMIT 1", - (thread_id,), + cur.execute( + "SELECT thread_id, thread_ts, parent_ts, checkpoint FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC LIMIT 1", + (config["configurable"]["thread_id"],), ) - value = cursor.fetchone() - if value: - return CheckpointTuple( - config, - loads(value[0]), + if value := cur.fetchone(): + return CheckpointTuple( + { + "configurable": { + "thread_id": value[0], + "thread_ts": value[1], + } + }, + loads(value[3]), + { + "configurable": { + "thread_id": value[0], + "thread_ts": value[2], + } + } + if value[2] + else None, + ) + + def list(self, config: RunnableConfig) -> Iterator[CheckpointTuple]: + with self.cursor(transaction=False) as cur: + cur.execute( + "SELECT thread_id, thread_ts, parent_ts, checkpoint FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC", + (config["configurable"]["thread_id"],), + ) + for thread_id, thread_ts, parent_ts, value in cur: + yield CheckpointTuple( + {"configurable": {"thread_id": thread_id, "thread_ts": thread_ts}}, + loads(value), { "configurable": { "thread_id": thread_id, - "thread_ts": value[1], + "thread_ts": parent_ts, } } - if value[1] + if parent_ts else None, ) - return None - # Adapting the aput method - def aput(self, config: RunnableConfig, checkpoint: Checkpoint) -> None: - thread_id = config["configurable"]["thread_id"] - with _connect() as conn: - cursor = conn.cursor() - cursor.execute( - """ - INSERT INTO checkpoints (thread_id, thread_ts, parent_ts, checkpoint) - VALUES (?, ?, ?, ?) - ON CONFLICT (thread_id, thread_ts) - DO UPDATE SET checkpoint = EXCLUDED.checkpoint; - """, + def put(self, config: RunnableConfig, checkpoint: Checkpoint) -> RunnableConfig: + with self.cursor() as cur: + cur.execute( + "INSERT OR REPLACE INTO checkpoints (thread_id, thread_ts, parent_ts, checkpoint) VALUES (?, ?, ?, ?)", ( - thread_id, - datetime.fromisoformat(checkpoint["ts"]), - datetime.fromisoformat(checkpoint.get("parent_ts", "")), + config["configurable"]["thread_id"], + checkpoint["ts"], + config["configurable"].get("thread_ts"), pickle.dumps(checkpoint), ), ) - conn.commit() return { "configurable": { - "thread_id": thread_id, + "thread_id": config["configurable"]["thread_id"], "thread_ts": checkpoint["ts"], } } diff --git a/backend/app/server.py b/backend/app/server.py index f877e81a..124df908 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -39,7 +39,7 @@ async def ingest_files( thread_id = config["configurable"].get("thread_id") if thread_id is not None: - thread = await storage.get_thread(user["user_id"], thread_id) + thread = storage.get_thread(user["user_id"], thread_id) if thread is None: raise HTTPException(status_code=404, detail="Thread not found.") diff --git a/backend/app/storage.py b/backend/app/storage.py index 3be7f2be..902f30b4 100644 --- a/backend/app/storage.py +++ b/backend/app/storage.py @@ -3,7 +3,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Sequence, Union -from uuid import UUID, uuid4 +from uuid import uuid4 from langchain_core.messages import AnyMessage @@ -33,8 +33,11 @@ def list_assistants(user_id: str) -> List[Assistant]: assistants = [] for row in rows: assistant_data = dict(row) # Convert sqlite3.Row to dict - assistant_data['config'] = json.loads(assistant_data['config']) if 'config' in assistant_data and \ - assistant_data['config'] else {} + assistant_data["config"] = ( + json.loads(assistant_data["config"]) + if "config" in assistant_data and assistant_data["config"] + else {} + ) assistant = Assistant(**assistant_data) assistants.append(assistant) @@ -53,12 +56,15 @@ def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: if not row: return None assistant_data = dict(row) # Convert sqlite3.Row to dict - assistant_data['config'] = json.loads(assistant_data['config']) if 'config' in assistant_data and \ - assistant_data['config'] else {} + assistant_data["config"] = ( + json.loads(assistant_data["config"]) + if "config" in assistant_data and assistant_data["config"] + else {} + ) return Assistant(**assistant_data) -async def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: +def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: """List all the public assistants.""" assistant_ids_tuple = tuple( assistant_ids @@ -108,7 +114,7 @@ def put_assistant( ) -async def list_threads(user_id: str) -> List[Thread]: +def list_threads(user_id: str) -> List[Thread]: """List all threads for the current user.""" with _connect() as conn: cursor = conn.cursor() @@ -129,25 +135,25 @@ def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: return Thread(**dict(row)) if row else None -async def get_thread_state(user_id: str, thread_id: str): +def get_thread_state(user_id: str, thread_id: str): """Get state for a thread.""" app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) - state = await app.aget_state({"configurable": {"thread_id": thread_id}}) + state = app.get_state({"configurable": {"thread_id": thread_id}}) return { "values": state.values, "next": state.next, } -async def update_thread_state( +def update_thread_state( user_id: str, thread_id: str, values: Union[Sequence[AnyMessage], Dict[str, Any]] ): """Add state to a thread.""" app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) - await app.aupdate_state({"configurable": {"thread_id": thread_id}}, values) + app.update_state({"configurable": {"thread_id": thread_id}}, values) -async def get_thread_history(user_id: str, thread_id: str): +def get_thread_history(user_id: str, thread_id: str): """Get the history of a thread.""" app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) return [ @@ -157,9 +163,7 @@ async def get_thread_history(user_id: str, thread_id: str): "config": c.config, "parent": c.parent_config, } - async for c in app.aget_state_history( - {"configurable": {"thread_id": thread_id}} - ) + for c in app.get_state_history({"configurable": {"thread_id": thread_id}}) ] @@ -200,23 +204,33 @@ def get_or_create_user(sub: str) -> tuple[User, bool]: if user_row: # Convert sqlite3.Row to a User object - user = User(user_id=user_row["user_id"], sub=user_row["sub"], created_at=user_row["created_at"]) + user = User( + user_id=user_row["user_id"], + sub=user_row["sub"], + created_at=user_row["created_at"], + ) return user, False # SQLite doesn't support RETURNING *, so we need to manually fetch the created user. - cursor.execute('INSERT INTO "user" (user_id, sub, created_at) VALUES (?, ?, ?)', (str(uuid4()), sub, datetime.now())) + cursor.execute( + 'INSERT INTO "user" (user_id, sub, created_at) VALUES (?, ?, ?)', + (str(uuid4()), sub, datetime.now()), + ) conn.commit() # Fetch the newly created user cursor.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) new_user_row = cursor.fetchone() - new_user = User(user_id=new_user_row["user_id"], sub=new_user_row["sub"], - created_at=new_user_row["created_at"]) + new_user = User( + user_id=new_user_row["user_id"], + sub=new_user_row["sub"], + created_at=new_user_row["created_at"], + ) return new_user, True -async def delete_thread(user_id: str, thread_id: str): +def delete_thread(user_id: str, thread_id: str): """Delete a thread by ID.""" with _connect() as conn: cursor = conn.cursor() @@ -224,4 +238,4 @@ async def delete_thread(user_id: str, thread_id: str): "DELETE FROM thread WHERE thread_id = ? AND user_id = ?", (thread_id, user_id), ) - conn.commit() \ No newline at end of file + conn.commit() From d251e84a7342785c9602239b542d2ef031487b73 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Sun, 21 Apr 2024 20:27:48 +0400 Subject: [PATCH 09/28] Add aiosqlite dependency. --- backend/poetry.lock | 20 +++++++++++++++++++- backend/pyproject.toml | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/backend/poetry.lock b/backend/poetry.lock index 60e577ac..78459944 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -110,6 +110,24 @@ files = [ [package.dependencies] frozenlist = ">=1.1.0" +[[package]] +name = "aiosqlite" +version = "0.20.0" +description = "asyncio bridge to the standard sqlite3 module" +optional = false +python-versions = ">=3.8" +files = [ + {file = "aiosqlite-0.20.0-py3-none-any.whl", hash = "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6"}, + {file = "aiosqlite-0.20.0.tar.gz", hash = "sha256:6d35c8c256637f4672f843c31021464090805bf925385ac39473fb16eaaca3d7"}, +] + +[package.dependencies] +typing_extensions = ">=4.0" + +[package.extras] +dev = ["attribution (==1.7.0)", "black (==24.2.0)", "coverage[toml] (==7.4.1)", "flake8 (==7.0.0)", "flake8-bugbear (==24.2.6)", "flit (==3.9.0)", "mypy (==1.8.0)", "ufmt (==2.3.0)", "usort (==1.0.8.post1)"] +docs = ["sphinx (==7.2.6)", "sphinx-mdinclude (==0.5.3)"] + [[package]] name = "anthropic" version = "0.25.2" @@ -5395,4 +5413,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9.0,<3.12" -content-hash = "d173115dd8349b20d60b475c229549fdfc9083ecc643f43a46a770223d04cdab" +content-hash = "65dbe314143f753e0d8ccf71130aa1162037c5c538320aa8a9bf52854ef974d5" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index e8e1aaf8..1d4b86cc 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -45,6 +45,7 @@ langchain-core = "^0.1.42" pyjwt = {extras = ["crypto"], version = "^2.8.0"} langchain-anthropic = "^0.1.8" langchain-chroma = "^0.1.0" +aiosqlite = "^0.20.0" [tool.poetry.group.dev.dependencies] uvicorn = "^0.23.2" From 0a29bfb8410c7ea321f4818175564c78185e55c4 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Sun, 21 Apr 2024 21:03:43 +0400 Subject: [PATCH 10/28] Separete storage methods for pg and sqlite. --- backend/app/agent.py | 4 +- backend/app/api/assistants.py | 14 +- backend/app/api/runs.py | 6 +- backend/app/api/threads.py | 29 ++-- backend/app/auth/handlers.py | 6 +- backend/app/checkpoint.py | 178 +++++++++++++---------- backend/app/lifespan.py | 74 ++++++++++ backend/app/server.py | 9 +- backend/app/storage.py | 241 -------------------------------- backend/app/storage/__init__.py | 12 ++ backend/app/storage/base.py | 75 ++++++++++ backend/app/storage/postgres.py | 189 +++++++++++++++++++++++++ backend/app/storage/settings.py | 30 ++++ backend/app/storage/sqlite.py | 241 ++++++++++++++++++++++++++++++++ backend/app/upload.py | 57 ++++---- docker-compose.pg.yml | 58 ++++++++ docker-compose.yml | 32 +---- tools/sqlite_migrate/Dockerfile | 10 ++ 18 files changed, 870 insertions(+), 395 deletions(-) create mode 100644 backend/app/lifespan.py delete mode 100644 backend/app/storage.py create mode 100644 backend/app/storage/__init__.py create mode 100644 backend/app/storage/base.py create mode 100644 backend/app/storage/postgres.py create mode 100644 backend/app/storage/settings.py create mode 100644 backend/app/storage/sqlite.py create mode 100644 docker-compose.pg.yml create mode 100644 tools/sqlite_migrate/Dockerfile diff --git a/backend/app/agent.py b/backend/app/agent.py index d1540b29..49c4303d 100644 --- a/backend/app/agent.py +++ b/backend/app/agent.py @@ -11,7 +11,7 @@ from app.agent_types.tools_agent import get_tools_agent_executor from app.agent_types.xml_agent import get_xml_agent_executor from app.chatbot import get_chatbot_executor -from app.checkpoint import SQLiteCheckpoint +from app.checkpoint import Checkpointer from app.llms import ( get_anthropic_llm, get_google_llm, @@ -70,7 +70,7 @@ class AgentType(str, Enum): DEFAULT_SYSTEM_MESSAGE = "You are a helpful assistant." -CHECKPOINTER = SQLiteCheckpoint(at=CheckpointAt.END_OF_STEP) +CHECKPOINTER = Checkpointer(at=CheckpointAt.END_OF_STEP) def get_agent_executor( diff --git a/backend/app/api/assistants.py b/backend/app/api/assistants.py index 04b6c82e..7c914423 100644 --- a/backend/app/api/assistants.py +++ b/backend/app/api/assistants.py @@ -4,9 +4,9 @@ from fastapi import APIRouter, HTTPException, Path, Query from pydantic import BaseModel, Field -import app.storage as storage from app.auth.handlers import AuthedUser from app.schema import Assistant +from app.storage import storage router = APIRouter() @@ -25,9 +25,9 @@ class AssistantPayload(BaseModel): @router.get("/") -def list_assistants(user: AuthedUser) -> List[Assistant]: +async def list_assistants(user: AuthedUser) -> List[Assistant]: """List all assistants for the current user.""" - return storage.list_assistants(user["user_id"]) + return await storage.list_assistants(user["user_id"]) @router.get("/public/") @@ -37,7 +37,7 @@ async def list_public_assistants( ] = None, ) -> List[Assistant]: """List all public assistants.""" - return storage.list_public_assistants( + return await storage.list_public_assistants( FEATURED_PUBLIC_ASSISTANTS + ([shared_id] if shared_id else []) ) @@ -48,7 +48,7 @@ async def get_assistant( aid: AssistantID, ) -> Assistant: """Get an assistant by ID.""" - assistant = storage.get_assistant(user["user_id"], aid) + assistant = await storage.get_assistant(user["user_id"], aid) if not assistant: raise HTTPException(status_code=404, detail="Assistant not found") return assistant @@ -60,7 +60,7 @@ async def create_assistant( payload: AssistantPayload, ) -> Assistant: """Create an assistant.""" - return storage.put_assistant( + return await storage.put_assistant( user["user_id"], str(uuid4()), name=payload.name, @@ -76,7 +76,7 @@ async def upsert_assistant( payload: AssistantPayload, ) -> Assistant: """Create or update an assistant.""" - return storage.put_assistant( + return await storage.put_assistant( user["user_id"], aid, name=payload.name, diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index baa75e70..0c8cd3e1 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -14,7 +14,7 @@ from app.agent import agent from app.auth.handlers import AuthedUser -from app.storage import get_assistant, get_thread +from app.storage import storage from app.stream import astream_messages, to_sse router = APIRouter() @@ -31,11 +31,11 @@ class CreateRunPayload(BaseModel): async def _run_input_and_config(payload: CreateRunPayload, user_id: str): - thread = get_thread(user_id, payload.thread_id) + thread = await storage.get_thread(user_id, payload.thread_id) if not thread: raise HTTPException(status_code=404, detail="Thread not found") - assistant = get_assistant(user_id, str(thread["assistant_id"])) + assistant = await storage.get_assistant(user_id, str(thread["assistant_id"])) if not assistant: raise HTTPException(status_code=404, detail="Assistant not found") diff --git a/backend/app/api/threads.py b/backend/app/api/threads.py index fee713c5..0d4ae967 100644 --- a/backend/app/api/threads.py +++ b/backend/app/api/threads.py @@ -1,3 +1,4 @@ +import asyncio from typing import Annotated, Any, Dict, List, Sequence, Union from uuid import uuid4 @@ -5,9 +6,9 @@ from langchain.schema.messages import AnyMessage from pydantic import BaseModel, Field -import app.storage as storage from app.auth.handlers import AuthedUser from app.schema import Thread +from app.storage import storage router = APIRouter() @@ -31,7 +32,7 @@ class ThreadPostRequest(BaseModel): @router.get("/") async def list_threads(user: AuthedUser) -> List[Thread]: """List all threads for the current user.""" - return storage.list_threads(user["user_id"]) + return await storage.list_threads(user["user_id"]) @router.get("/{tid}/state") @@ -40,8 +41,10 @@ async def get_thread_state( tid: ThreadID, ): """Get state for a thread.""" - thread = storage.get_thread(user["user_id"], tid) - state = storage.get_thread_state(user["user_id"], tid) + thread, state = await asyncio.gather( + storage.get_thread(user["user_id"], tid), + storage.get_thread_state(user["user_id"], tid), + ) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return state @@ -54,10 +57,10 @@ async def add_thread_state( payload: ThreadPostRequest, ): """Add state to a thread.""" - thread = storage.get_thread(user["user_id"], tid) + thread = await storage.get_thread(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") - return storage.update_thread_state(user["user_id"], tid, payload.values) + return await storage.update_thread_state(user["user_id"], tid, payload.values) @router.get("/{tid}/history") @@ -66,8 +69,10 @@ async def get_thread_history( tid: ThreadID, ): """Get all past states for a thread.""" - thread = storage.get_thread(user["user_id"], tid) - history = storage.get_thread_history(user["user_id"], tid) + thread, history = await asyncio.gather( + storage.get_thread(user["user_id"], tid), + storage.get_thread_history(user["user_id"], tid), + ) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return history @@ -79,7 +84,7 @@ async def get_thread( tid: ThreadID, ) -> Thread: """Get a thread by ID.""" - thread = storage.get_thread(user["user_id"], tid) + thread = await storage.get_thread(user["user_id"], tid) if not thread: raise HTTPException(status_code=404, detail="Thread not found") return thread @@ -91,7 +96,7 @@ async def create_thread( thread_put_request: ThreadPutRequest, ) -> Thread: """Create a thread.""" - return storage.put_thread( + return await storage.put_thread( user["user_id"], str(uuid4()), assistant_id=thread_put_request.assistant_id, @@ -106,7 +111,7 @@ async def upsert_thread( thread_put_request: ThreadPutRequest, ) -> Thread: """Update a thread.""" - return storage.put_thread( + return await storage.put_thread( user["user_id"], tid, assistant_id=thread_put_request.assistant_id, @@ -120,5 +125,5 @@ async def delete_thread( tid: ThreadID, ): """Delete a thread by ID.""" - storage.delete_thread(user["user_id"], tid) + await storage.delete_thread(user["user_id"], tid) return {"status": "ok"} diff --git a/backend/app/auth/handlers.py b/backend/app/auth/handlers.py index 31d7a797..81177e7e 100644 --- a/backend/app/auth/handlers.py +++ b/backend/app/auth/handlers.py @@ -7,9 +7,9 @@ from fastapi import Depends, HTTPException, Request from fastapi.security.http import HTTPBearer -import app.storage as storage from app.auth.settings import AuthType, settings from app.schema import User +from app.storage import storage class AuthHandler(ABC): @@ -23,7 +23,7 @@ class NOOPAuth(AuthHandler): async def __call__(self, request: Request) -> User: sub = request.cookies.get("opengpts_user_id") or self._default_sub - user, _ = storage.get_or_create_user(sub) + user, _ = await storage.get_or_create_user(sub) return user @@ -37,7 +37,7 @@ async def __call__(self, request: Request) -> User: except jwt.PyJWTError as e: raise HTTPException(status_code=401, detail=str(e)) - user, _ = storage.get_or_create_user(payload["sub"]) + user, _ = await storage.get_or_create_user(payload["sub"]) return user @abstractmethod diff --git a/backend/app/checkpoint.py b/backend/app/checkpoint.py index 90f7d872..c7419a2f 100644 --- a/backend/app/checkpoint.py +++ b/backend/app/checkpoint.py @@ -1,26 +1,21 @@ import pickle -import sqlite3 -from contextlib import contextmanager -from typing import Iterator, Optional +from datetime import datetime +from typing import AsyncIterator, Optional +import aiosqlite from langchain_core.messages import BaseMessage from langchain_core.runnables import ConfigurableFieldSpec, RunnableConfig from langgraph.checkpoint import BaseCheckpointSaver +from langgraph.checkpoint.aiosqlite import AsyncSqliteSaver from langgraph.checkpoint.base import ( Checkpoint, CheckpointThreadTs, CheckpointTuple, ) - -@contextmanager -def _connect(): - conn = sqlite3.connect("opengpts.db") - conn.row_factory = sqlite3.Row # Enable dictionary access to row items. - try: - yield conn - finally: - conn.close() +from app.lifespan import create_sqlite_conn, get_pg_pool +from app.storage.settings import StorageType +from app.storage.settings import settings as storage_settings def loads(value: bytes) -> Checkpoint: @@ -31,7 +26,7 @@ def loads(value: bytes) -> Checkpoint: return loaded -class SQLiteCheckpoint(BaseCheckpointSaver): +class PostgresCheckpointer(BaseCheckpointSaver): class Config: arbitrary_types_allowed = True @@ -49,34 +44,53 @@ def config_specs(self) -> list[ConfigurableFieldSpec]: CheckpointThreadTs, ] - @contextmanager - def cursor(self, transaction: bool = True): - with _connect() as conn: - cur = conn.cursor() - try: - yield cur - finally: - if transaction: - conn.commit() - cur.close() - - def get_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: - with self.cursor(transaction=False) as cur: - if config["configurable"].get("thread_ts"): - cur.execute( - "SELECT checkpoint, parent_ts FROM checkpoints WHERE thread_id = ? AND thread_ts = ?", - ( - config["configurable"]["thread_id"], - config["configurable"]["thread_ts"], - ), + def get(self, config: RunnableConfig) -> Optional[Checkpoint]: + raise NotImplementedError + + def put(self, config: RunnableConfig, checkpoint: Checkpoint) -> RunnableConfig: + raise NotImplementedError + + async def alist(self, config: RunnableConfig) -> AsyncIterator[CheckpointTuple]: + async with get_pg_pool().acquire() as db, db.transaction(): + thread_id = config["configurable"]["thread_id"] + async for value in db.cursor( + "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = $1 ORDER BY thread_ts DESC", + thread_id, + ): + yield CheckpointTuple( + { + "configurable": { + "thread_id": thread_id, + "thread_ts": value[1], + } + }, + loads(value[0]), + { + "configurable": { + "thread_id": thread_id, + "thread_ts": value[2], + } + } + if value[2] + else None, ) - if value := cur.fetchone(): + + async def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: + thread_id = config["configurable"]["thread_id"] + thread_ts = config["configurable"].get("thread_ts") + async with get_pg_pool().acquire() as conn: + if thread_ts: + if value := await conn.fetchrow( + "SELECT checkpoint, parent_ts FROM checkpoints WHERE thread_id = $1 AND thread_ts = $2", + thread_id, + datetime.fromisoformat(thread_ts), + ): return CheckpointTuple( config, loads(value[0]), { "configurable": { - "thread_id": config["configurable"]["thread_id"], + "thread_id": thread_id, "thread_ts": value[1], } } @@ -84,22 +98,21 @@ def get_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: else None, ) else: - cur.execute( - "SELECT thread_id, thread_ts, parent_ts, checkpoint FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC LIMIT 1", - (config["configurable"]["thread_id"],), - ) - if value := cur.fetchone(): + if value := await conn.fetchrow( + "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = $1 ORDER BY thread_ts DESC LIMIT 1", + thread_id, + ): return CheckpointTuple( { "configurable": { - "thread_id": value[0], + "thread_id": thread_id, "thread_ts": value[1], } }, - loads(value[3]), + loads(value[0]), { "configurable": { - "thread_id": value[0], + "thread_id": thread_id, "thread_ts": value[2], } } @@ -107,40 +120,57 @@ def get_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: else None, ) - def list(self, config: RunnableConfig) -> Iterator[CheckpointTuple]: - with self.cursor(transaction=False) as cur: - cur.execute( - "SELECT thread_id, thread_ts, parent_ts, checkpoint FROM checkpoints WHERE thread_id = ? ORDER BY thread_ts DESC", - (config["configurable"]["thread_id"],), - ) - for thread_id, thread_ts, parent_ts, value in cur: - yield CheckpointTuple( - {"configurable": {"thread_id": thread_id, "thread_ts": thread_ts}}, - loads(value), - { - "configurable": { - "thread_id": thread_id, - "thread_ts": parent_ts, - } - } - if parent_ts - else None, - ) - - def put(self, config: RunnableConfig, checkpoint: Checkpoint) -> RunnableConfig: - with self.cursor() as cur: - cur.execute( - "INSERT OR REPLACE INTO checkpoints (thread_id, thread_ts, parent_ts, checkpoint) VALUES (?, ?, ?, ?)", - ( - config["configurable"]["thread_id"], - checkpoint["ts"], - config["configurable"].get("thread_ts"), - pickle.dumps(checkpoint), - ), + async def aput(self, config: RunnableConfig, checkpoint: Checkpoint) -> None: + thread_id = config["configurable"]["thread_id"] + async with get_pg_pool().acquire() as conn: + await conn.execute( + """ + INSERT INTO checkpoints (thread_id, thread_ts, parent_ts, checkpoint) + VALUES ($1, $2, $3, $4) + ON CONFLICT (thread_id, thread_ts) + DO UPDATE SET checkpoint = EXCLUDED.checkpoint;""", + thread_id, + datetime.fromisoformat(checkpoint["ts"]), + datetime.fromisoformat(checkpoint.get("parent_ts")) + if checkpoint.get("parent_ts") + else None, + pickle.dumps(checkpoint), ) return { "configurable": { - "thread_id": config["configurable"]["thread_id"], + "thread_id": thread_id, "thread_ts": checkpoint["ts"], } } + + +class SqliteCheckpointer(AsyncSqliteSaver): + conn: aiosqlite.Connection = None + + @property + def config_specs(self) -> list[ConfigurableFieldSpec]: + return [ + ConfigurableFieldSpec( + id="thread_id", + annotation=Optional[str], + name="Thread ID", + description=None, + default=None, + is_shared=True, + ), + CheckpointThreadTs, + ] + + async def setup(self) -> None: + if self.is_setup: + return + self.conn = await create_sqlite_conn(global_=True) + self.is_setup = True + + +if storage_settings.storage_type == StorageType.POSTGRES: + Checkpointer = PostgresCheckpointer +elif storage_settings.storage_type == StorageType.SQLITE: + Checkpointer = SqliteCheckpointer +else: + raise NotImplementedError() diff --git a/backend/app/lifespan.py b/backend/app/lifespan.py new file mode 100644 index 00000000..45f33afa --- /dev/null +++ b/backend/app/lifespan.py @@ -0,0 +1,74 @@ +from contextlib import asynccontextmanager +from typing import AsyncGenerator + +import aiosqlite +import asyncpg +import orjson +from fastapi import FastAPI + +from app.storage.settings import StorageType +from app.storage.settings import settings as storage_settings + +_pg_pool = None +_global_sqlite_connections = [] + + +async def create_sqlite_conn(global_: bool = False, **kwargs) -> aiosqlite.Connection: + conn = await aiosqlite.connect("opengpts.db", **kwargs) + conn.row_factory = aiosqlite.Row + if global_: + _global_sqlite_connections.append(conn) + return conn + + +@asynccontextmanager +async def sqlite_conn(**kwargs) -> AsyncGenerator[aiosqlite.Connection, None]: + conn = await create_sqlite_conn(**kwargs) + try: + yield conn + finally: + await conn.close() + + +async def _close_global_sqlite_connections() -> None: + for conn in _global_sqlite_connections: + await conn.close() + _global_sqlite_connections.clear() + + +def get_pg_pool() -> asyncpg.pool.Pool: + return _pg_pool + + +async def _init_connection(conn) -> None: + await conn.set_type_codec( + "json", + encoder=lambda v: orjson.dumps(v).decode(), + decoder=orjson.loads, + schema="pg_catalog", + ) + await conn.set_type_codec( + "uuid", encoder=lambda v: str(v), decoder=lambda v: v, schema="pg_catalog" + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + if storage_settings.storage_type == StorageType.POSTGRES: + global _pg_pool + _pg_pool = await asyncpg.create_pool( + database=storage_settings.postgres.db, + user=storage_settings.postgres.user, + password=storage_settings.postgres.password, + host=storage_settings.postgres.host, + port=storage_settings.postgres.port, + init=_init_connection, + ) + + yield + + if storage_settings.storage_type == StorageType.POSTGRES: + await _pg_pool.close() + _pg_pool = None + elif storage_settings.storage_type == StorageType.SQLITE: + await _close_global_sqlite_connections() diff --git a/backend/app/server.py b/backend/app/server.py index 124df908..e96803d4 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -7,14 +7,15 @@ from fastapi.exceptions import HTTPException from fastapi.staticfiles import StaticFiles -import app.storage as storage from app.api import router as api_router from app.auth.handlers import AuthedUser +from app.lifespan import lifespan +from app.storage import storage from app.upload import ingest_runnable logger = logging.getLogger(__name__) -app = FastAPI(title="OpenGPTs API") +app = FastAPI(title="OpenGPTs API", lifespan=lifespan) # Get root of app, used to point to directory containing static files @@ -33,13 +34,13 @@ async def ingest_files( assistant_id = config["configurable"].get("assistant_id") if assistant_id is not None: - assistant = storage.get_assistant(user["user_id"], assistant_id) + assistant = await storage.get_assistant(user["user_id"], assistant_id) if assistant is None: raise HTTPException(status_code=404, detail="Assistant not found.") thread_id = config["configurable"].get("thread_id") if thread_id is not None: - thread = storage.get_thread(user["user_id"], thread_id) + thread = await storage.get_thread(user["user_id"], thread_id) if thread is None: raise HTTPException(status_code=404, detail="Thread not found.") diff --git a/backend/app/storage.py b/backend/app/storage.py deleted file mode 100644 index 902f30b4..00000000 --- a/backend/app/storage.py +++ /dev/null @@ -1,241 +0,0 @@ -import json -import sqlite3 -from contextlib import contextmanager -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Sequence, Union -from uuid import uuid4 - -from langchain_core.messages import AnyMessage - -from app.agent import AgentType, get_agent_executor -from app.schema import Assistant, Thread, User - - -@contextmanager -def _connect(): - conn = sqlite3.connect("opengpts.db") - conn.row_factory = sqlite3.Row # Enable dictionary access to row items. - try: - yield conn - finally: - conn.close() - - -def list_assistants(user_id: str) -> List[Assistant]: - """List all assistants for the current user.""" - with _connect() as conn: - conn.row_factory = sqlite3.Row # Enable dictionary-like row access - cursor = conn.cursor() - cursor.execute("SELECT * FROM assistant WHERE user_id = ?", (user_id,)) - rows = cursor.fetchall() - - # Deserialize the 'config' field from a JSON string to a dict for each row - assistants = [] - for row in rows: - assistant_data = dict(row) # Convert sqlite3.Row to dict - assistant_data["config"] = ( - json.loads(assistant_data["config"]) - if "config" in assistant_data and assistant_data["config"] - else {} - ) - assistant = Assistant(**assistant_data) - assistants.append(assistant) - - return assistants - - -def get_assistant(user_id: str, assistant_id: str) -> Optional[Assistant]: - """Get an assistant by ID.""" - with _connect() as conn: - cursor = conn.cursor() - cursor.execute( - "SELECT * FROM assistant WHERE assistant_id = ? AND (user_id = ? OR public = 1)", - (assistant_id, user_id), - ) - row = cursor.fetchone() - if not row: - return None - assistant_data = dict(row) # Convert sqlite3.Row to dict - assistant_data["config"] = ( - json.loads(assistant_data["config"]) - if "config" in assistant_data and assistant_data["config"] - else {} - ) - return Assistant(**assistant_data) - - -def list_public_assistants(assistant_ids: Sequence[str]) -> List[Assistant]: - """List all the public assistants.""" - assistant_ids_tuple = tuple( - assistant_ids - ) # SQL requires a tuple for the IN operator. - placeholders = ", ".join("?" for _ in assistant_ids) - with _connect() as conn: - cursor = conn.cursor() - cursor.execute( - f"SELECT * FROM assistant WHERE assistant_id IN ({placeholders}) AND public = 1", - assistant_ids_tuple, - ) - rows = cursor.fetchall() - return [Assistant(**dict(row)) for row in rows] - - -def put_assistant( - user_id: str, assistant_id: str, *, name: str, config: dict, public: bool = False -) -> Assistant: - """Modify an assistant.""" - updated_at = datetime.now(timezone.utc) - with _connect() as conn: - cursor = conn.cursor() - # Convert the config dict to a JSON string for storage. - config_str = json.dumps(config) - cursor.execute( - """ - INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT(assistant_id) - DO UPDATE SET - user_id = EXCLUDED.user_id, - name = EXCLUDED.name, - config = EXCLUDED.config, - updated_at = EXCLUDED.updated_at, - public = EXCLUDED.public - """, - (assistant_id, user_id, name, config_str, updated_at.isoformat(), public), - ) - conn.commit() - return Assistant( - assistant_id=assistant_id, - user_id=user_id, - name=name, - config=config, - updated_at=updated_at, - public=public, - ) - - -def list_threads(user_id: str) -> List[Thread]: - """List all threads for the current user.""" - with _connect() as conn: - cursor = conn.cursor() - cursor.execute("SELECT * FROM thread WHERE user_id = ?", (user_id,)) - rows = cursor.fetchall() - return [Thread(**dict(row)) for row in rows] - - -def get_thread(user_id: str, thread_id: str) -> Optional[Thread]: - """Get a thread by ID.""" - with _connect() as conn: - cursor = conn.cursor() - cursor.execute( - "SELECT * FROM thread WHERE thread_id = ? AND user_id = ?", - (thread_id, user_id), - ) - row = cursor.fetchone() - return Thread(**dict(row)) if row else None - - -def get_thread_state(user_id: str, thread_id: str): - """Get state for a thread.""" - app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) - state = app.get_state({"configurable": {"thread_id": thread_id}}) - return { - "values": state.values, - "next": state.next, - } - - -def update_thread_state( - user_id: str, thread_id: str, values: Union[Sequence[AnyMessage], Dict[str, Any]] -): - """Add state to a thread.""" - app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) - app.update_state({"configurable": {"thread_id": thread_id}}, values) - - -def get_thread_history(user_id: str, thread_id: str): - """Get the history of a thread.""" - app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) - return [ - { - "values": c.values, - "next": c.next, - "config": c.config, - "parent": c.parent_config, - } - for c in app.get_state_history({"configurable": {"thread_id": thread_id}}) - ] - - -def put_thread(user_id: str, thread_id: str, *, assistant_id: str, name: str) -> Thread: - """Modify a thread.""" - updated_at = datetime.now(timezone.utc) - with _connect() as conn: - cursor = conn.cursor() - cursor.execute( - """ - INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(thread_id) - DO UPDATE SET - user_id = EXCLUDED.user_id, - assistant_id = EXCLUDED.assistant_id, - name = EXCLUDED.name, - updated_at = EXCLUDED.updated_at - """, - (thread_id, user_id, assistant_id, name, updated_at), - ) - conn.commit() - return { - "thread_id": thread_id, - "user_id": user_id, - "assistant_id": assistant_id, - "name": name, - "updated_at": updated_at, - } - - -def get_or_create_user(sub: str) -> tuple[User, bool]: - """Returns a tuple of the user and a boolean indicating whether the user was created.""" - with _connect() as conn: - cursor = conn.cursor() - cursor.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) - user_row = cursor.fetchone() - - if user_row: - # Convert sqlite3.Row to a User object - user = User( - user_id=user_row["user_id"], - sub=user_row["sub"], - created_at=user_row["created_at"], - ) - return user, False - - # SQLite doesn't support RETURNING *, so we need to manually fetch the created user. - cursor.execute( - 'INSERT INTO "user" (user_id, sub, created_at) VALUES (?, ?, ?)', - (str(uuid4()), sub, datetime.now()), - ) - conn.commit() - - # Fetch the newly created user - cursor.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) - new_user_row = cursor.fetchone() - - new_user = User( - user_id=new_user_row["user_id"], - sub=new_user_row["sub"], - created_at=new_user_row["created_at"], - ) - return new_user, True - - -def delete_thread(user_id: str, thread_id: str): - """Delete a thread by ID.""" - with _connect() as conn: - cursor = conn.cursor() - cursor.execute( - "DELETE FROM thread WHERE thread_id = ? AND user_id = ?", - (thread_id, user_id), - ) - conn.commit() diff --git a/backend/app/storage/__init__.py b/backend/app/storage/__init__.py new file mode 100644 index 00000000..702c012e --- /dev/null +++ b/backend/app/storage/__init__.py @@ -0,0 +1,12 @@ +from app.storage.postgres import PostgresStorage +from app.storage.settings import StorageType, settings +from app.storage.sqlite import SqliteStorage + +if settings.storage_type == StorageType.SQLITE: + storage = SqliteStorage() +elif settings.storage_type == StorageType.POSTGRES: + storage = PostgresStorage() +else: + raise NotImplementedError() + +__all__ = ["storage"] diff --git a/backend/app/storage/base.py b/backend/app/storage/base.py new file mode 100644 index 00000000..22b69659 --- /dev/null +++ b/backend/app/storage/base.py @@ -0,0 +1,75 @@ +from abc import ABC, abstractmethod +from typing import Any, Optional, Sequence, Union + +from langchain_core.messages import AnyMessage + +from app.schema import Assistant, Thread, User + + +class BaseStorage(ABC): + @abstractmethod + async def list_assistants(self, user_id: str) -> list[Assistant]: + """List all assistants for the current user.""" + + @abstractmethod + async def get_assistant( + self, user_id: str, assistant_id: str + ) -> Optional[Assistant]: + """Get an assistant by ID.""" + + @abstractmethod + async def list_public_assistants( + self, assistant_ids: Sequence[str] + ) -> list[Assistant]: + """List all the public assistants.""" + + @abstractmethod + async def put_assistant( + self, + user_id: str, + assistant_id: str, + *, + name: str, + config: dict, + public: bool = False, + ) -> Assistant: + """Modify an assistant.""" + + @abstractmethod + async def list_threads(self, user_id: str) -> list[Thread]: + """List all threads for the current user.""" + + @abstractmethod + async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: + """Get a thread by ID.""" + + @abstractmethod + async def get_thread_state(self, user_id: str, thread_id: str): + """Get state for a thread.""" + + @abstractmethod + async def update_thread_state( + self, + user_id: str, + thread_id: str, + values: Union[Sequence[AnyMessage], dict[str, Any]], + ): + """Add state to a thread.""" + + @abstractmethod + async def get_thread_history(self, user_id: str, thread_id: str): + """Get the history of a thread.""" + + @abstractmethod + async def put_thread( + self, user_id: str, thread_id: str, *, assistant_id: str, name: str + ) -> Thread: + """Modify a thread.""" + + @abstractmethod + async def get_or_create_user(self, sub: str) -> tuple[User, bool]: + """Returns a tuple of the user and a boolean indicating whether the user was created.""" + + @abstractmethod + async def delete_thread(self, user_id: str, thread_id: str) -> None: + """Delete a thread by ID.""" diff --git a/backend/app/storage/postgres.py b/backend/app/storage/postgres.py new file mode 100644 index 00000000..8e20f99f --- /dev/null +++ b/backend/app/storage/postgres.py @@ -0,0 +1,189 @@ +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Sequence, Union + +from langchain_core.messages import AnyMessage + +from app.agent import AgentType, get_agent_executor +from app.lifespan import get_pg_pool +from app.schema import Assistant, Thread, User +from app.storage.base import BaseStorage + + +class PostgresStorage(BaseStorage): + async def list_assistants(self, user_id: str) -> List[Assistant]: + """List all assistants for the current user.""" + async with get_pg_pool().acquire() as conn: + return await conn.fetch( + "SELECT * FROM assistant WHERE user_id = $1", user_id + ) + + async def get_assistant( + self, user_id: str, assistant_id: str + ) -> Optional[Assistant]: + """Get an assistant by ID.""" + async with get_pg_pool().acquire() as conn: + return await conn.fetchrow( + "SELECT * FROM assistant WHERE assistant_id = $1 AND (user_id = $2 OR public = true)", + assistant_id, + user_id, + ) + + async def list_public_assistants( + self, assistant_ids: Sequence[str] + ) -> List[Assistant]: + """List all the public assistants.""" + async with get_pg_pool().acquire() as conn: + return await conn.fetch( + ( + "SELECT * FROM assistant " + "WHERE assistant_id = ANY($1::uuid[]) " + "AND public = true;" + ), + assistant_ids, + ) + + async def put_assistant( + self, + user_id: str, + assistant_id: str, + *, + name: str, + config: dict, + public: bool = False, + ) -> Assistant: + """Modify an assistant. + + Args: + user_id: The user ID. + assistant_id: The assistant ID. + name: The assistant name. + config: The assistant config. + public: Whether the assistant is public. + + Returns: + return the assistant model if no exception is raised. + """ + updated_at = datetime.now(timezone.utc) + async with get_pg_pool().acquire() as conn: + async with conn.transaction(): + await conn.execute( + ( + "INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) VALUES ($1, $2, $3, $4, $5, $6) " + "ON CONFLICT (assistant_id) DO UPDATE SET " + "user_id = EXCLUDED.user_id, " + "name = EXCLUDED.name, " + "config = EXCLUDED.config, " + "updated_at = EXCLUDED.updated_at, " + "public = EXCLUDED.public;" + ), + assistant_id, + user_id, + name, + config, + updated_at, + public, + ) + return { + "assistant_id": assistant_id, + "user_id": user_id, + "name": name, + "config": config, + "updated_at": updated_at, + "public": public, + } + + async def list_threads(self, user_id: str) -> List[Thread]: + """List all threads for the current user.""" + async with get_pg_pool().acquire() as conn: + return await conn.fetch("SELECT * FROM thread WHERE user_id = $1", user_id) + + async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: + """Get a thread by ID.""" + async with get_pg_pool().acquire() as conn: + return await conn.fetchrow( + "SELECT * FROM thread WHERE thread_id = $1 AND user_id = $2", + thread_id, + user_id, + ) + + async def get_thread_state(self, user_id: str, thread_id: str): + """Get state for a thread.""" + app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) + state = await app.aget_state({"configurable": {"thread_id": thread_id}}) + return { + "values": state.values, + "next": state.next, + } + + async def update_thread_state( + self, + user_id: str, + thread_id: str, + values: Union[Sequence[AnyMessage], Dict[str, Any]], + ): + """Add state to a thread.""" + app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) + await app.aupdate_state({"configurable": {"thread_id": thread_id}}, values) + + async def get_thread_history(self, user_id: str, thread_id: str): + """Get the history of a thread.""" + app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) + return [ + { + "values": c.values, + "next": c.next, + "config": c.config, + "parent": c.parent_config, + } + async for c in app.aget_state_history( + {"configurable": {"thread_id": thread_id}} + ) + ] + + async def put_thread( + self, user_id: str, thread_id: str, *, assistant_id: str, name: str + ) -> Thread: + """Modify a thread.""" + updated_at = datetime.now(timezone.utc) + async with get_pg_pool().acquire() as conn: + await conn.execute( + ( + "INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) VALUES ($1, $2, $3, $4, $5) " + "ON CONFLICT (thread_id) DO UPDATE SET " + "user_id = EXCLUDED.user_id," + "assistant_id = EXCLUDED.assistant_id, " + "name = EXCLUDED.name, " + "updated_at = EXCLUDED.updated_at;" + ), + thread_id, + user_id, + assistant_id, + name, + updated_at, + ) + return { + "thread_id": thread_id, + "user_id": user_id, + "assistant_id": assistant_id, + "name": name, + "updated_at": updated_at, + } + + async def get_or_create_user(self, sub: str) -> tuple[User, bool]: + """Returns a tuple of the user and a boolean indicating whether the user was created.""" + async with get_pg_pool().acquire() as conn: + if user := await conn.fetchrow('SELECT * FROM "user" WHERE sub = $1', sub): + return user, False + user = await conn.fetchrow( + 'INSERT INTO "user" (sub) VALUES ($1) RETURNING *', sub + ) + return user, True + + async def delete_thread(self, user_id: str, thread_id: str) -> None: + """Delete a thread by ID.""" + async with get_pg_pool().acquire() as conn: + await conn.execute( + "DELETE FROM thread WHERE thread_id = $1 AND user_id = $2", + thread_id, + user_id, + ) diff --git a/backend/app/storage/settings.py b/backend/app/storage/settings.py new file mode 100644 index 00000000..71770d27 --- /dev/null +++ b/backend/app/storage/settings.py @@ -0,0 +1,30 @@ +from enum import Enum +from typing import Optional + +from pydantic import BaseSettings + + +class StorageType(Enum): + POSTGRES = "postgres" + SQLITE = "sqlite" + + +class PostgresSettings(BaseSettings): + host: str + port: int + db: str + user: str + password: str + + class Config: + env_prefix = "postgres_" + + +class Settings(BaseSettings): + storage_type: StorageType = StorageType.SQLITE + postgres: Optional[PostgresSettings] = None + + +settings = Settings() +if settings.storage_type == StorageType.POSTGRES: + settings.postgres = PostgresSettings() diff --git a/backend/app/storage/sqlite.py b/backend/app/storage/sqlite.py new file mode 100644 index 00000000..7f92fe98 --- /dev/null +++ b/backend/app/storage/sqlite.py @@ -0,0 +1,241 @@ +import json +from datetime import datetime, timezone +from typing import Any, Optional, Sequence, Union +from uuid import uuid4 + +from langchain_core.messages import AnyMessage + +from app.agent import AgentType, get_agent_executor +from app.lifespan import sqlite_conn +from app.schema import Assistant, Thread, User +from app.storage.base import BaseStorage + + +class SqliteStorage(BaseStorage): + async def list_assistants(self, user_id: str) -> list[Assistant]: + """List all assistants for the current user.""" + async with sqlite_conn() as conn, conn.cursor() as cur: + await cur.execute("SELECT * FROM assistant WHERE user_id = ?", (user_id,)) + rows = await cur.fetchall() + + # Deserialize the 'config' field from a JSON string to a dict for each row + assistants = [] + for row in rows: + assistant_data = dict(row) # Convert sqlite3.Row to dict + assistant_data["config"] = ( + json.loads(assistant_data["config"]) + if "config" in assistant_data and assistant_data["config"] + else {} + ) + assistant = Assistant(**assistant_data) + assistants.append(assistant) + + return assistants + + async def get_assistant( + self, user_id: str, assistant_id: str + ) -> Optional[Assistant]: + """Get an assistant by ID.""" + async with sqlite_conn() as conn, conn.cursor() as cur: + await cur.execute( + "SELECT * FROM assistant WHERE assistant_id = ? AND (user_id = ? OR public = 1)", + (assistant_id, user_id), + ) + row = await cur.fetchone() + if not row: + return None + assistant_data = dict(row) # Convert sqlite3.Row to dict + assistant_data["config"] = ( + json.loads(assistant_data["config"]) + if "config" in assistant_data and assistant_data["config"] + else {} + ) + return Assistant(**assistant_data) + + async def list_public_assistants( + self, assistant_ids: Sequence[str] + ) -> list[Assistant]: + """List all the public assistants.""" + assistant_ids_tuple = tuple( + assistant_ids + ) # SQL requires a tuple for the IN operator. + placeholders = ", ".join("?" for _ in assistant_ids) + async with sqlite_conn() as conn, conn.cursor() as cur: + await cur.execute( + f"SELECT * FROM assistant WHERE assistant_id IN ({placeholders}) AND public = 1", + assistant_ids_tuple, + ) + rows = await cur.fetchall() + return [Assistant(**dict(row)) for row in rows] + + async def put_assistant( + self, + user_id: str, + assistant_id: str, + *, + name: str, + config: dict, + public: bool = False, + ) -> Assistant: + """Modify an assistant.""" + updated_at = datetime.now(timezone.utc) + async with sqlite_conn() as conn, conn.cursor() as cur: + # Convert the config dict to a JSON string for storage. + config_str = json.dumps(config) + await cur.execute( + """ + INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(assistant_id) + DO UPDATE SET + user_id = EXCLUDED.user_id, + name = EXCLUDED.name, + config = EXCLUDED.config, + updated_at = EXCLUDED.updated_at, + public = EXCLUDED.public + """, + ( + assistant_id, + user_id, + name, + config_str, + updated_at.isoformat(), + public, + ), + ) + await conn.commit() + return Assistant( + assistant_id=assistant_id, + user_id=user_id, + name=name, + config=config, + updated_at=updated_at, + public=public, + ) + + async def list_threads(self, user_id: str) -> list[Thread]: + """List all threads for the current user.""" + async with sqlite_conn() as conn, conn.cursor() as cur: + await cur.execute("SELECT * FROM thread WHERE user_id = ?", (user_id,)) + rows = await cur.fetchall() + return [Thread(**dict(row)) for row in rows] + + async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: + """Get a thread by ID.""" + async with sqlite_conn() as conn, conn.cursor() as cur: + await cur.execute( + "SELECT * FROM thread WHERE thread_id = ? AND user_id = ?", + (thread_id, user_id), + ) + row = await cur.fetchone() + return Thread(**dict(row)) if row else None + + async def get_thread_state(self, user_id: str, thread_id: str): + """Get state for a thread.""" + app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) + state = await app.aget_state({"configurable": {"thread_id": thread_id}}) + return { + "values": state.values, + "next": state.next, + } + + async def update_thread_state( + self, + user_id: str, + thread_id: str, + values: Union[Sequence[AnyMessage], dict[str, Any]], + ): + """Add state to a thread.""" + app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) + await app.aupdate_state({"configurable": {"thread_id": thread_id}}, values) + + async def get_thread_history(self, user_id: str, thread_id: str): + """Get the history of a thread.""" + app = get_agent_executor([], AgentType.GPT_35_TURBO, "", False) + return [ + { + "values": c.values, + "next": c.next, + "config": c.config, + "parent": c.parent_config, + } + async for c in app.aget_state_history( + {"configurable": {"thread_id": thread_id}} + ) + ] + + async def put_thread( + self, user_id: str, thread_id: str, *, assistant_id: str, name: str + ) -> Thread: + """Modify a thread.""" + updated_at = datetime.now(timezone.utc) + async with sqlite_conn() as conn, conn.cursor() as cur: + await cur.execute( + """ + INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(thread_id) + DO UPDATE SET + user_id = EXCLUDED.user_id, + assistant_id = EXCLUDED.assistant_id, + name = EXCLUDED.name, + updated_at = EXCLUDED.updated_at + """, + (thread_id, user_id, assistant_id, name, updated_at), + ) + await conn.commit() + return { + "thread_id": thread_id, + "user_id": user_id, + "assistant_id": assistant_id, + "name": name, + "updated_at": updated_at, + } + + async def get_or_create_user(self, sub: str) -> tuple[User, bool]: + """Returns a tuple of the user and a boolean indicating whether the user was created.""" + async with sqlite_conn() as conn, conn.cursor() as cur: + # start a write transaction to avoid the unique contraint error due to + # concurrent inserts + await cur.execute("BEGIN EXCLUSIVE") + await cur.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) + user_row = await cur.fetchone() + + if user_row: + # Convert sqlite3.Row to a User object + user = User( + user_id=user_row["user_id"], + sub=user_row["sub"], + created_at=user_row["created_at"], + ) + return user, False + + # SQLite doesn't support RETURNING *, so we need to manually fetch the created user. + await cur.execute( + 'INSERT INTO "user" (user_id, sub, created_at) VALUES (?, ?, ?)', + (str(uuid4()), sub, datetime.now()), + ) + await conn.commit() + + # Fetch the newly created user + await cur.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) + new_user_row = await cur.fetchone() + + new_user = User( + user_id=new_user_row["user_id"], + sub=new_user_row["sub"], + created_at=new_user_row["created_at"], + ) + return new_user, True + + async def delete_thread(self, user_id: str, thread_id: str) -> None: + """Delete a thread by ID.""" + async with sqlite_conn() as conn, conn.cursor() as cur: + await cur.execute( + "DELETE FROM thread WHERE thread_id = ? AND user_id = ?", + (thread_id, user_id), + ) + await conn.commit() + + +storage = SqliteStorage() diff --git a/backend/app/upload.py b/backend/app/upload.py index da24a3a8..41345068 100644 --- a/backend/app/upload.py +++ b/backend/app/upload.py @@ -10,7 +10,7 @@ from __future__ import annotations import os -from typing import Any, BinaryIO, List, Optional +from typing import Any, BinaryIO, List, Optional, Union from langchain_chroma import Chroma from langchain_community.document_loaders.blob_loaders.schema import Blob @@ -26,6 +26,7 @@ from app.ingest import ingest_blob from app.parsing import MIMETYPE_BASED_PARSER +from app.storage.settings import StorageType, settings def _guess_mimetype(file_bytes: bytes) -> str: @@ -54,27 +55,43 @@ def _convert_ingestion_input_to_blob(data: BinaryIO) -> Blob: ) -def _determine_azure_or_openai_embeddings() -> Chroma: +def _get_embedding_function() -> Union[OpenAIEmbeddings, AzureOpenAIEmbeddings]: if os.environ.get("OPENAI_API_KEY"): - return Chroma( - persist_directory="./chroma_db", embedding_function=OpenAIEmbeddings() - ) - if os.environ.get("AZURE_OPENAI_API_KEY"): - return Chroma( - persist_directory="./chroma_db", - embedding_function=AzureOpenAIEmbeddings( - azure_endpoint=os.environ.get("AZURE_OPENAI_API_BASE"), - azure_deployment=os.environ.get( - "AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME" - ), - openai_api_version=os.environ.get("AZURE_OPENAI_API_VERSION"), - ), + return OpenAIEmbeddings() + elif os.environ.get("AZURE_OPENAI_API_KEY"): + return AzureOpenAIEmbeddings( + azure_endpoint=os.environ.get("AZURE_OPENAI_API_BASE"), + azure_deployment=os.environ.get("AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"), + openai_api_version=os.environ.get("AZURE_OPENAI_API_VERSION"), ) raise ValueError( "Either OPENAI_API_KEY or AZURE_OPENAI_API_KEY needs to be set for embeddings to work." ) +def _get_vstore() -> VectorStore: + if settings.storage_type == StorageType.POSTGRES: + PG_CONNECTION_STRING = PGVector.connection_string_from_db_params( + driver="psycopg2", + host=settings.postgres.host, + port=settings.postgres.port, + database=settings.postgres.db, + user=settings.postgres.user, + password=settings.postgres.password, + ) + return PGVector( + connection_string=PG_CONNECTION_STRING, + embedding_function=_get_embedding_function(), + use_jsonb=True, + ) + elif settings.storage_type == StorageType.SQLITE: + return Chroma( + persist_directory="./chroma_db", + embedding_function=_get_embedding_function(), + ) + raise NotImplementedError() + + class IngestRunnable(RunnableSerializable[BinaryIO, List[str]]): """Runnable for ingesting files into a vectorstore.""" @@ -131,15 +148,7 @@ def batch( return ids -PG_CONNECTION_STRING = PGVector.connection_string_from_db_params( - driver="psycopg2", - host=os.environ["POSTGRES_HOST"], - port=int(os.environ["POSTGRES_PORT"]), - database=os.environ["POSTGRES_DB"], - user=os.environ["POSTGRES_USER"], - password=os.environ["POSTGRES_PASSWORD"], -) -vstore = _determine_azure_or_openai_embeddings() +vstore = _get_vstore() ingest_runnable = IngestRunnable( diff --git a/docker-compose.pg.yml b/docker-compose.pg.yml new file mode 100644 index 00000000..079c4874 --- /dev/null +++ b/docker-compose.pg.yml @@ -0,0 +1,58 @@ +version: "3" + +services: + postgres: + image: pgvector/pgvector:pg16 + healthcheck: + test: pg_isready -U $POSTGRES_USER + start_interval: 1s + start_period: 5s + interval: 5s + retries: 5 + ports: + - "5433:5432" + env_file: + - .env + volumes: + - ./postgres-volume:/var/lib/postgresql/data + postgres-setup: + image: migrate/migrate + depends_on: + postgres: + condition: service_healthy + volumes: + - ./backend/migrations:/migrations + env_file: + - .env + command: ["-path", "/migrations", "-database", "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", "up"] + backend: + container_name: opengpts-backend + build: + context: backend + ports: + - "8100:8000" # Backend is accessible on localhost:8100 + depends_on: + postgres-setup: + condition: service_completed_successfully + env_file: + - .env + volumes: + - ./backend:/backend + environment: + STORAGE_TYPE: "postgres" + POSTGRES_HOST: "postgres" + command: + - --reload + frontend: + container_name: opengpts-frontend + build: + context: frontend + depends_on: + backend: + condition: service_healthy + volumes: + - ./frontend/src:/frontend/src + ports: + - "5173:5173" # Frontend is accessible on localhost:5173 + environment: + VITE_BACKEND_URL: "http://backend:8000" diff --git a/docker-compose.yml b/docker-compose.yml index 6d59cb42..2b5e4f42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,30 +1,12 @@ version: "3" services: - postgres: - image: pgvector/pgvector:pg16 - healthcheck: - test: pg_isready -U $POSTGRES_USER - start_interval: 1s - start_period: 5s - interval: 5s - retries: 5 - ports: - - "5433:5432" - env_file: - - .env - volumes: - - ./postgres-volume:/var/lib/postgresql/data - postgres-setup: - image: migrate/migrate - depends_on: - postgres: - condition: service_healthy + sqlite-setup: + build: + context: tools/sqlite_migrate volumes: - - ./backend/migrations:/migrations - env_file: - - .env - command: ["-path", "/migrations", "-database", "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", "up"] + - ./backend:/backend + command: ["migrate", "-path", "/backend/migrations", "-database", "sqlite3://opengpts.db", "up"] backend: container_name: opengpts-backend build: @@ -32,14 +14,14 @@ services: ports: - "8100:8000" # Backend is accessible on localhost:8100 depends_on: - postgres-setup: + sqlite-setup: condition: service_completed_successfully env_file: - .env volumes: - ./backend:/backend environment: - POSTGRES_HOST: "postgres" + STORAGE_TYPE: "sqlite" command: - --reload frontend: diff --git a/tools/sqlite_migrate/Dockerfile b/tools/sqlite_migrate/Dockerfile new file mode 100644 index 00000000..94114c0c --- /dev/null +++ b/tools/sqlite_migrate/Dockerfile @@ -0,0 +1,10 @@ +# Can't use the migrate/migrate image like we do for Postgres because it doesn't +# support sqlite3 driver by default. We need to install it via Go toolchain: +# https://github.com/golang-migrate/migrate/issues/899#issuecomment-1483741684 +# +# Using the alpine3.18 tag because of: https://github.com/nkanaev/yarr/issues/187 + +FROM golang:alpine3.18 +RUN apk add build-base +RUN go install -tags 'sqlite3' github.com/golang-migrate/migrate/v4/cmd/migrate@v4.17.1 +WORKDIR /backend \ No newline at end of file From 827b96fe05f6aadfee076ed79703ccc44ffd07a2 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Sun, 21 Apr 2024 23:42:21 +0400 Subject: [PATCH 11/28] Separate postgres and sqlite migrations. --- backend/Makefile | 8 +++++- ...reate_extensions_and_first_tables.down.sql | 0 ..._create_extensions_and_first_tables.up.sql | 24 ++++++++++++++++++ .../000002_checkpoints_update_schema.down.sql | 5 ++++ .../000002_checkpoints_update_schema.up.sql | 11 ++++++++ .../postgres/000003_create_user.down.sql | 9 +++++++ .../postgres/000003_create_user.up.sql | 25 +++++++++++++++++++ ...reate_extensions_and_first_tables.down.sql | 3 +++ ..._create_extensions_and_first_tables.up.sql | 0 .../000002_checkpoints_update_schema.down.sql | 0 .../000002_checkpoints_update_schema.up.sql | 0 .../{ => sqlite}/000003_create_user.down.sql | 0 .../{ => sqlite}/000003_create_user.up.sql | 0 docker-compose.pg.yml | 2 +- docker-compose.yml | 2 +- 15 files changed, 86 insertions(+), 3 deletions(-) rename backend/migrations/{ => postgres}/000001_create_extensions_and_first_tables.down.sql (100%) create mode 100644 backend/migrations/postgres/000001_create_extensions_and_first_tables.up.sql create mode 100644 backend/migrations/postgres/000002_checkpoints_update_schema.down.sql create mode 100644 backend/migrations/postgres/000002_checkpoints_update_schema.up.sql create mode 100644 backend/migrations/postgres/000003_create_user.down.sql create mode 100644 backend/migrations/postgres/000003_create_user.up.sql create mode 100644 backend/migrations/sqlite/000001_create_extensions_and_first_tables.down.sql rename backend/migrations/{ => sqlite}/000001_create_extensions_and_first_tables.up.sql (100%) rename backend/migrations/{ => sqlite}/000002_checkpoints_update_schema.down.sql (100%) rename backend/migrations/{ => sqlite}/000002_checkpoints_update_schema.up.sql (100%) rename backend/migrations/{ => sqlite}/000003_create_user.down.sql (100%) rename backend/migrations/{ => sqlite}/000003_create_user.up.sql (100%) diff --git a/backend/Makefile b/backend/Makefile index 5030e0c0..d86a3b54 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -17,7 +17,13 @@ start: poetry run uvicorn app.server:app --reload --port 8100 migrate: - migrate -database sqlite3://$(PWD)/opengpts.db -path ./migrations up + ifeq ($(STORAGE_TYPE),postgres) + @echo "Running Postgres migrations..." + migrate -database postgres://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@$(POSTGRES_HOST):$(POSTGRES_PORT)/$(POSTGRES_DB)?sslmode=disable -path ./migrations/postgres up + else + @echo "Running SQLite migrations..." + migrate -database sqlite3://$(PWD)/opengpts.db -path ./migrations/sqlite up + endif test: # We need to update handling of env variables for tests diff --git a/backend/migrations/000001_create_extensions_and_first_tables.down.sql b/backend/migrations/postgres/000001_create_extensions_and_first_tables.down.sql similarity index 100% rename from backend/migrations/000001_create_extensions_and_first_tables.down.sql rename to backend/migrations/postgres/000001_create_extensions_and_first_tables.down.sql diff --git a/backend/migrations/postgres/000001_create_extensions_and_first_tables.up.sql b/backend/migrations/postgres/000001_create_extensions_and_first_tables.up.sql new file mode 100644 index 00000000..cb395a74 --- /dev/null +++ b/backend/migrations/postgres/000001_create_extensions_and_first_tables.up.sql @@ -0,0 +1,24 @@ +CREATE EXTENSION IF NOT EXISTS vector; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +CREATE TABLE IF NOT EXISTS assistant ( + assistant_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + user_id VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + config JSON NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC'), + public BOOLEAN NOT NULL +); + +CREATE TABLE IF NOT EXISTS thread ( + thread_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + assistant_id UUID REFERENCES assistant(assistant_id) ON DELETE SET NULL, + user_id VARCHAR(255) NOT NULL, + name VARCHAR(255) NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +); + +CREATE TABLE IF NOT EXISTS checkpoints ( + thread_id TEXT PRIMARY KEY, + checkpoint BYTEA +); \ No newline at end of file diff --git a/backend/migrations/postgres/000002_checkpoints_update_schema.down.sql b/backend/migrations/postgres/000002_checkpoints_update_schema.down.sql new file mode 100644 index 00000000..c8a249eb --- /dev/null +++ b/backend/migrations/postgres/000002_checkpoints_update_schema.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE checkpoints + DROP CONSTRAINT IF EXISTS checkpoints_pkey, + ADD PRIMARY KEY (thread_id), + DROP COLUMN IF EXISTS thread_ts, + DROP COLUMN IF EXISTS parent_ts; diff --git a/backend/migrations/postgres/000002_checkpoints_update_schema.up.sql b/backend/migrations/postgres/000002_checkpoints_update_schema.up.sql new file mode 100644 index 00000000..9ddd077f --- /dev/null +++ b/backend/migrations/postgres/000002_checkpoints_update_schema.up.sql @@ -0,0 +1,11 @@ +ALTER TABLE checkpoints + ADD COLUMN IF NOT EXISTS thread_ts TIMESTAMPTZ, + ADD COLUMN IF NOT EXISTS parent_ts TIMESTAMPTZ; + +UPDATE checkpoints + SET thread_ts = CURRENT_TIMESTAMP AT TIME ZONE 'UTC' +WHERE thread_ts IS NULL; + +ALTER TABLE checkpoints + DROP CONSTRAINT IF EXISTS checkpoints_pkey, + ADD PRIMARY KEY (thread_id, thread_ts) diff --git a/backend/migrations/postgres/000003_create_user.down.sql b/backend/migrations/postgres/000003_create_user.down.sql new file mode 100644 index 00000000..66c5acad --- /dev/null +++ b/backend/migrations/postgres/000003_create_user.down.sql @@ -0,0 +1,9 @@ +ALTER TABLE assistant + DROP CONSTRAINT fk_assistant_user_id, + ALTER COLUMN user_id TYPE VARCHAR USING (user_id::text); + +ALTER TABLE thread + DROP CONSTRAINT fk_thread_user_id, + ALTER COLUMN user_id TYPE VARCHAR USING (user_id::text); + +DROP TABLE IF EXISTS "user"; \ No newline at end of file diff --git a/backend/migrations/postgres/000003_create_user.up.sql b/backend/migrations/postgres/000003_create_user.up.sql new file mode 100644 index 00000000..45612f9c --- /dev/null +++ b/backend/migrations/postgres/000003_create_user.up.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS "user" ( + user_id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + sub VARCHAR(255) UNIQUE NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +); + +INSERT INTO "user" (user_id, sub) +SELECT DISTINCT user_id::uuid, user_id +FROM assistant +WHERE user_id IS NOT NULL +ON CONFLICT (user_id) DO NOTHING; + +INSERT INTO "user" (user_id, sub) +SELECT DISTINCT user_id::uuid, user_id +FROM thread +WHERE user_id IS NOT NULL +ON CONFLICT (user_id) DO NOTHING; + +ALTER TABLE assistant + ALTER COLUMN user_id TYPE UUID USING (user_id::UUID), + ADD CONSTRAINT fk_assistant_user_id FOREIGN KEY (user_id) REFERENCES "user"(user_id); + +ALTER TABLE thread + ALTER COLUMN user_id TYPE UUID USING (user_id::UUID), + ADD CONSTRAINT fk_thread_user_id FOREIGN KEY (user_id) REFERENCES "user"(user_id); \ No newline at end of file diff --git a/backend/migrations/sqlite/000001_create_extensions_and_first_tables.down.sql b/backend/migrations/sqlite/000001_create_extensions_and_first_tables.down.sql new file mode 100644 index 00000000..08c8d5d5 --- /dev/null +++ b/backend/migrations/sqlite/000001_create_extensions_and_first_tables.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS thread; +DROP TABLE IF EXISTS assistant; +DROP TABLE IF EXISTS checkpoints; diff --git a/backend/migrations/000001_create_extensions_and_first_tables.up.sql b/backend/migrations/sqlite/000001_create_extensions_and_first_tables.up.sql similarity index 100% rename from backend/migrations/000001_create_extensions_and_first_tables.up.sql rename to backend/migrations/sqlite/000001_create_extensions_and_first_tables.up.sql diff --git a/backend/migrations/000002_checkpoints_update_schema.down.sql b/backend/migrations/sqlite/000002_checkpoints_update_schema.down.sql similarity index 100% rename from backend/migrations/000002_checkpoints_update_schema.down.sql rename to backend/migrations/sqlite/000002_checkpoints_update_schema.down.sql diff --git a/backend/migrations/000002_checkpoints_update_schema.up.sql b/backend/migrations/sqlite/000002_checkpoints_update_schema.up.sql similarity index 100% rename from backend/migrations/000002_checkpoints_update_schema.up.sql rename to backend/migrations/sqlite/000002_checkpoints_update_schema.up.sql diff --git a/backend/migrations/000003_create_user.down.sql b/backend/migrations/sqlite/000003_create_user.down.sql similarity index 100% rename from backend/migrations/000003_create_user.down.sql rename to backend/migrations/sqlite/000003_create_user.down.sql diff --git a/backend/migrations/000003_create_user.up.sql b/backend/migrations/sqlite/000003_create_user.up.sql similarity index 100% rename from backend/migrations/000003_create_user.up.sql rename to backend/migrations/sqlite/000003_create_user.up.sql diff --git a/docker-compose.pg.yml b/docker-compose.pg.yml index 079c4874..381b9484 100644 --- a/docker-compose.pg.yml +++ b/docker-compose.pg.yml @@ -24,7 +24,7 @@ services: - ./backend/migrations:/migrations env_file: - .env - command: ["-path", "/migrations", "-database", "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", "up"] + command: ["-path", "/migrations/postgres", "-database", "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", "up"] backend: container_name: opengpts-backend build: diff --git a/docker-compose.yml b/docker-compose.yml index 2b5e4f42..9c8a4501 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: context: tools/sqlite_migrate volumes: - ./backend:/backend - command: ["migrate", "-path", "/backend/migrations", "-database", "sqlite3://opengpts.db", "up"] + command: ["migrate", "-path", "/backend/migrations/sqlite", "-database", "sqlite3://opengpts.db", "up"] backend: container_name: opengpts-backend build: From ce0f4620355ad3d975340355cbf061f2bb34b569 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 12:30:28 +0400 Subject: [PATCH 12/28] Fix circular import. --- backend/app/api/assistants.py | 2 +- backend/app/api/runs.py | 2 +- backend/app/api/threads.py | 2 +- backend/app/auth/handlers.py | 2 +- backend/app/server.py | 2 +- backend/app/storage/__init__.py | 12 ------------ backend/app/storage/storage.py | 10 ++++++++++ 7 files changed, 15 insertions(+), 17 deletions(-) create mode 100644 backend/app/storage/storage.py diff --git a/backend/app/api/assistants.py b/backend/app/api/assistants.py index 7c914423..03fb16b4 100644 --- a/backend/app/api/assistants.py +++ b/backend/app/api/assistants.py @@ -6,7 +6,7 @@ from app.auth.handlers import AuthedUser from app.schema import Assistant -from app.storage import storage +from app.storage.storage import storage router = APIRouter() diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 0c8cd3e1..aa6837fb 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -14,7 +14,7 @@ from app.agent import agent from app.auth.handlers import AuthedUser -from app.storage import storage +from app.storage.storage import storage from app.stream import astream_messages, to_sse router = APIRouter() diff --git a/backend/app/api/threads.py b/backend/app/api/threads.py index 0d4ae967..0330f6c3 100644 --- a/backend/app/api/threads.py +++ b/backend/app/api/threads.py @@ -8,7 +8,7 @@ from app.auth.handlers import AuthedUser from app.schema import Thread -from app.storage import storage +from app.storage.storage import storage router = APIRouter() diff --git a/backend/app/auth/handlers.py b/backend/app/auth/handlers.py index 81177e7e..1486f4f9 100644 --- a/backend/app/auth/handlers.py +++ b/backend/app/auth/handlers.py @@ -9,7 +9,7 @@ from app.auth.settings import AuthType, settings from app.schema import User -from app.storage import storage +from app.storage.storage import storage class AuthHandler(ABC): diff --git a/backend/app/server.py b/backend/app/server.py index e96803d4..5ee0d468 100644 --- a/backend/app/server.py +++ b/backend/app/server.py @@ -10,7 +10,7 @@ from app.api import router as api_router from app.auth.handlers import AuthedUser from app.lifespan import lifespan -from app.storage import storage +from app.storage.storage import storage from app.upload import ingest_runnable logger = logging.getLogger(__name__) diff --git a/backend/app/storage/__init__.py b/backend/app/storage/__init__.py index 702c012e..e69de29b 100644 --- a/backend/app/storage/__init__.py +++ b/backend/app/storage/__init__.py @@ -1,12 +0,0 @@ -from app.storage.postgres import PostgresStorage -from app.storage.settings import StorageType, settings -from app.storage.sqlite import SqliteStorage - -if settings.storage_type == StorageType.SQLITE: - storage = SqliteStorage() -elif settings.storage_type == StorageType.POSTGRES: - storage = PostgresStorage() -else: - raise NotImplementedError() - -__all__ = ["storage"] diff --git a/backend/app/storage/storage.py b/backend/app/storage/storage.py new file mode 100644 index 00000000..7c15f826 --- /dev/null +++ b/backend/app/storage/storage.py @@ -0,0 +1,10 @@ +from app.storage.postgres import PostgresStorage +from app.storage.settings import StorageType, settings +from app.storage.sqlite import SqliteStorage + +if settings.storage_type == StorageType.SQLITE: + storage = SqliteStorage() +elif settings.storage_type == StorageType.POSTGRES: + storage = PostgresStorage() +else: + raise NotImplementedError() From 326e46c2fbe68ccf2f1a122373921ac0b8991d6e Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 14:45:07 +0400 Subject: [PATCH 13/28] Prevent concurrent requests from raising UniqueViolationError for pg's get_or_create_user. --- backend/app/storage/postgres.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/backend/app/storage/postgres.py b/backend/app/storage/postgres.py index 8e20f99f..b0f1e5e3 100644 --- a/backend/app/storage/postgres.py +++ b/backend/app/storage/postgres.py @@ -172,12 +172,14 @@ async def put_thread( async def get_or_create_user(self, sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" async with get_pg_pool().acquire() as conn: - if user := await conn.fetchrow('SELECT * FROM "user" WHERE sub = $1', sub): - return user, False user = await conn.fetchrow( - 'INSERT INTO "user" (sub) VALUES ($1) RETURNING *', sub + 'INSERT INTO "user" (sub) VALUES ($1) ON CONFLICT (sub) DO NOTHING RETURNING *', + sub, ) - return user, True + if user: + return user, True + user = await conn.fetchrow('SELECT * FROM "user" WHERE sub = $1', sub) + return user, False async def delete_thread(self, user_id: str, thread_id: str) -> None: """Delete a thread by ID.""" From 755dd1594a556e4d9c4391b364bb84e98a747da8 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 15:06:36 +0400 Subject: [PATCH 14/28] Format. --- backend/app/chatbot.py | 3 ++- backend/app/message_types.py | 2 +- backend/app/storage/postgres.py | 8 ++++++-- backend/app/storage/sqlite.py | 8 ++++++-- backend/app/upload.py | 2 +- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/app/chatbot.py b/backend/app/chatbot.py index fe19725c..eeb5b787 100644 --- a/backend/app/chatbot.py +++ b/backend/app/chatbot.py @@ -1,11 +1,12 @@ from typing import Annotated, List -from app.message_types import add_messages_liberal from langchain_core.language_models.base import LanguageModelLike from langchain_core.messages import BaseMessage, SystemMessage from langgraph.checkpoint import BaseCheckpointSaver from langgraph.graph.state import StateGraph +from app.message_types import add_messages_liberal + def get_chatbot_executor( llm: LanguageModelLike, diff --git a/backend/app/message_types.py b/backend/app/message_types.py index bd1f7695..9ceea94b 100644 --- a/backend/app/message_types.py +++ b/backend/app/message_types.py @@ -6,7 +6,7 @@ MessageLikeRepresentation, ToolMessage, ) -from langgraph.graph.message import add_messages, Messages +from langgraph.graph.message import Messages, add_messages class LiberalFunctionMessage(FunctionMessage): diff --git a/backend/app/storage/postgres.py b/backend/app/storage/postgres.py index bbe7079c..d762d80c 100644 --- a/backend/app/storage/postgres.py +++ b/backend/app/storage/postgres.py @@ -107,7 +107,9 @@ async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: user_id, ) - async def get_thread_state(self, *, user_id: str, thread_id: str, assistant_id: str): + async def get_thread_state( + self, *, user_id: str, thread_id: str, assistant_id: str + ): """Get state for a thread.""" assistant = await self.get_assistant(user_id, assistant_id) state = await agent.aget_state( @@ -143,7 +145,9 @@ async def update_thread_state( values, ) - async def get_thread_history(self, *, user_id: str, thread_id: str, assistant_id: str): + async def get_thread_history( + self, *, user_id: str, thread_id: str, assistant_id: str + ): """Get the history of a thread.""" assistant = await self.get_assistant(user_id, assistant_id) return [ diff --git a/backend/app/storage/sqlite.py b/backend/app/storage/sqlite.py index 02b4dc1d..85fac4dd 100644 --- a/backend/app/storage/sqlite.py +++ b/backend/app/storage/sqlite.py @@ -131,7 +131,9 @@ async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: row = await cur.fetchone() return Thread(**dict(row)) if row else None - async def get_thread_state(self, *, user_id: str, thread_id: str, assistant_id: str): + async def get_thread_state( + self, *, user_id: str, thread_id: str, assistant_id: str + ): """Get state for a thread.""" assistant = await self.get_assistant(user_id, assistant_id) state = await agent.aget_state( @@ -167,7 +169,9 @@ async def update_thread_state( values, ) - async def get_thread_history(self, *, user_id: str, thread_id: str, assistant_id: str): + async def get_thread_history( + self, *, user_id: str, thread_id: str, assistant_id: str + ): """Get the history of a thread.""" assistant = await self.get_assistant(user_id, assistant_id) return [ diff --git a/backend/app/upload.py b/backend/app/upload.py index ff91c504..873768e7 100644 --- a/backend/app/upload.py +++ b/backend/app/upload.py @@ -13,8 +13,8 @@ from typing import BinaryIO, List, Optional, Union from langchain_chroma import Chroma -from langchain_core.document_loaders.blob_loaders import Blob from langchain_community.vectorstores.pgvector import PGVector +from langchain_core.document_loaders.blob_loaders import Blob from langchain_core.runnables import ( ConfigurableField, RunnableConfig, From 5fa04835c26a6ee0cebbf54b35417fe8e7387365 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 15:08:57 +0400 Subject: [PATCH 15/28] poetry lock --no-update. --- backend/poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/poetry.lock b/backend/poetry.lock index 8bf55869..34aa74e2 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -5413,4 +5413,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9.0,<3.12" -content-hash = "d4cc24e004188ffe360f62239e4bad0e5a5f2c808e4910645fa888eaa780d5c9" +content-hash = "432eb1901d8a047e032be6d23a520d4d38a097a3282892232a4e309ceee76dd7" From 74b9c2cad9e2afd94ce3700440d833edc02ee419 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 15:41:48 +0400 Subject: [PATCH 16/28] Fix SqliteCheckpointer's init. --- backend/app/checkpoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/checkpoint.py b/backend/app/checkpoint.py index 058a1ef2..18029520 100644 --- a/backend/app/checkpoint.py +++ b/backend/app/checkpoint.py @@ -160,7 +160,7 @@ def __init__( serde: Optional[SerializerProtocol] = None, at: Optional[CheckpointAt] = None, ) -> None: - super().__init__(serde=serde, at=at) + super().__init__(conn=None, serde=serde, at=at) @property def config_specs(self) -> list[ConfigurableFieldSpec]: From 36dbed8d734bd12be521a015bea6a646e24e2638 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 15:48:11 +0400 Subject: [PATCH 17/28] Include STORAGE_TYPE in the make test command. --- backend/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Makefile b/backend/Makefile index d86a3b54..a4502f35 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -27,7 +27,7 @@ migrate: test: # We need to update handling of env variables for tests - YDC_API_KEY=placeholder OPENAI_API_KEY=placeholder poetry run pytest $(TEST_FILE) + STORAGE_TYPE=postgres YDC_API_KEY=placeholder OPENAI_API_KEY=placeholder poetry run pytest $(TEST_FILE) test_watch: From 5a499414dbe39036879fb38e60fcac966e3d95d8 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 16:48:24 +0400 Subject: [PATCH 18/28] Use storage settings during tests. --- backend/tests/unit_tests/conftest.py | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/backend/tests/unit_tests/conftest.py b/backend/tests/unit_tests/conftest.py index 4d21da0d..8070c0bf 100644 --- a/backend/tests/unit_tests/conftest.py +++ b/backend/tests/unit_tests/conftest.py @@ -9,6 +9,7 @@ from app.auth.settings import settings as auth_settings from app.lifespan import get_pg_pool, lifespan from app.server import app +from app.storage.settings import settings as storage_settings auth_settings.auth_type = AuthType.NOOP @@ -16,16 +17,16 @@ os.environ["OPENAI_API_KEY"] = "test" TEST_DB = "test" -assert os.environ["POSTGRES_DB"] != TEST_DB, "Test and main database conflict." -os.environ["POSTGRES_DB"] = TEST_DB +assert storage_settings.postgres.db != TEST_DB, "Test and main database conflict." +storage_settings.postgres.db = TEST_DB async def _get_conn() -> asyncpg.Connection: return await asyncpg.connect( - user=os.environ["POSTGRES_USER"], - password=os.environ["POSTGRES_PASSWORD"], - host=os.environ["POSTGRES_HOST"], - port=os.environ["POSTGRES_PORT"], + user=storage_settings.postgres.user, + password=storage_settings.postgres.password, + host=storage_settings.postgres.host, + port=storage_settings.postgres.port, database="postgres", ) @@ -49,7 +50,21 @@ async def _drop_test_db() -> None: def _migrate_test_db() -> None: - subprocess.run(["make", "migrate"], check=True) + subprocess.run( + [ + "migrate", + "-database", + ( + f"postgres://{storage_settings.postgres.user}:{storage_settings.postgres.password}" + f"@{storage_settings.postgres.host}:{storage_settings.postgres.port}" + f"/{storage_settings.postgres.db}?sslmode=disable" + ), + "-path", + "./migrations/postgres", + "up", + ], + check=True, + ) @pytest.fixture(scope="session") From 735ce548da8e9e9b5cfed22b17db55a4717d59c5 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Mon, 22 Apr 2024 17:55:33 +0400 Subject: [PATCH 19/28] Update readme. --- README.md | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index c1d149f0..41f4fe1a 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Because this is open source, if you do not like those architectures or want to m ## Quickstart with Docker This project supports a Docker-based setup, streamlining installation and execution. It automatically builds images for -the frontend and backend and sets up Postgres using docker-compose. +the frontend and backend and sets up either SQLite or Postgres using docker-compose. 1. **Prerequisites:** @@ -53,7 +53,7 @@ the frontend and backend and sets up Postgres using docker-compose. 3. **Set Up Environment Variables:** Create a `.env` file in the root directory of the project by copying `.env.example` as a template, and add the - following environment variables: + following environment variables (if you want to use SQLite, you can skip the Postgres-related variables): ```shell # At least one language model API key is required OPENAI_API_KEY=sk-... @@ -71,14 +71,18 @@ the frontend and backend and sets up Postgres using docker-compose. 4. **Run with Docker Compose:** - In the root directory of the project, execute: + In the root directory of the project, execute one of the following commands to start the services: - ``` + ```shell + # For SQLite based setup docker compose up + + # For Postgres based setup + docker compose -f docker-compose.pg.yml up ``` This command builds the Docker images for the frontend and backend from their respective Dockerfiles and starts all - necessary services, including Postgres. + necessary services, including SQLite/Postgres. 5. **Access the Application:** With the services running, access the frontend at [http://localhost:5173](http://localhost:5173), substituting `5173` with the @@ -87,8 +91,12 @@ the frontend and backend and sets up Postgres using docker-compose. 6. **Rebuilding After Changes:** If you make changes to either the frontend or backend, rebuild the Docker images to reflect these changes. Run: - ``` + ```shell + # For SQLite based setup docker compose up --build + + # For Postgres based setup + docker compose -f docker-compose.pg.yml up --build ``` This command rebuilds the images with your latest changes and restarts the services. @@ -119,6 +127,16 @@ pip install langchain-community brew install libmagic ``` +### Persistence Layer + +The backend supports using SQLite and Postgres for saving agent configurations and chat message history. Set the `STORAGE_TYPE` environment variable to `sqlite` or `postgres`: + +```shell +export STORAGE_TYPE=postgres +``` + +SQLite requires no configuration (apart from [running migrations](####migrations)). The database file will be created in the `backend` directory. However, to configure and use Postgres, follow the instructions below: + **Install Postgres and the Postgres Vector Extension** ``` brew install postgresql pgvector @@ -127,8 +145,7 @@ brew services start postgresql **Configure persistence layer** -The backend uses Postgres for saving agent configurations and chat message history. -In order to use this, you need to set the following environment variables: +Set the following environment variables: ```shell export POSTGRES_HOST=localhost @@ -152,9 +169,9 @@ psql -d opengpts CREATE ROLE postgres WITH LOGIN SUPERUSER CREATEDB CREATEROLE; ``` -**Install Golang Migrate** +#### Migrations -Database migrations are managed with [golang-migrate](https://github.com/golang-migrate/migrate). +Database migrations for both SQLite and Postgres are managed with [golang-migrate](https://github.com/golang-migrate/migrate). On MacOS, you can install it with `brew install golang-migrate`. Instructions for other OSs or the Golang toolchain, can be found [here](https://github.com/golang-migrate/migrate/blob/master/cmd/migrate/README.md#installation). @@ -164,7 +181,7 @@ Once `golang-migrate` is installed, you can run all the migrations with: make migrate ``` -This will enable the backend to use Postgres as a vector database and create the initial tables. +This will create the initial tables. **Install backend dependencies** @@ -176,7 +193,7 @@ poetry install **Alternate vector databases** -The instructions above use Postgres as a vector database, +The instructions above use Postgres or Chroma DB (for SQLite based setup) as a vector database, although you can easily switch this out to use any of the 50+ vector databases in LangChain. **Set up language models** From 5b9871953e2b785831d234305f1159ca1846c8eb Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Tue, 30 Apr 2024 12:07:11 +0400 Subject: [PATCH 20/28] Temporarily use PGVector for both sqlite and postgres. --- backend/app/ingest.py | 8 - backend/app/storage/settings.py | 5 +- backend/app/upload.py | 8 +- backend/poetry.lock | 1180 +------------------------------ backend/pyproject.toml | 1 - docker-compose-prod.yml | 1 + docker-compose.yml | 27 + 7 files changed, 31 insertions(+), 1199 deletions(-) diff --git a/backend/app/ingest.py b/backend/app/ingest.py index f4030e2f..e0708045 100644 --- a/backend/app/ingest.py +++ b/backend/app/ingest.py @@ -20,13 +20,6 @@ def _update_document_metadata(document: Document, namespace: str) -> None: document.metadata["namespace"] = namespace -def _sanitize_document_metadata(document: Document) -> Document: - """Chroma doesn't accept None values in metadata, so we replace them.""" - for k, v in document.metadata.items(): - if v is None: - document.metadata[k] = "" - - def _sanitize_document_content(document: Document) -> Document: """Sanitize the document.""" # Without this, PDF ingestion fails with @@ -53,7 +46,6 @@ def ingest_blob( docs = text_splitter.split_documents([document]) for doc in docs: _sanitize_document_content(doc) - _sanitize_document_metadata(doc) _update_document_metadata(doc, namespace) docs_to_index.extend(docs) diff --git a/backend/app/storage/settings.py b/backend/app/storage/settings.py index 71770d27..a5ffee12 100644 --- a/backend/app/storage/settings.py +++ b/backend/app/storage/settings.py @@ -1,5 +1,4 @@ from enum import Enum -from typing import Optional from pydantic import BaseSettings @@ -22,9 +21,7 @@ class Config: class Settings(BaseSettings): storage_type: StorageType = StorageType.SQLITE - postgres: Optional[PostgresSettings] = None + postgres: PostgresSettings = PostgresSettings() settings = Settings() -if settings.storage_type == StorageType.POSTGRES: - settings.postgres = PostgresSettings() diff --git a/backend/app/upload.py b/backend/app/upload.py index 873768e7..bceda9dc 100644 --- a/backend/app/upload.py +++ b/backend/app/upload.py @@ -12,7 +12,6 @@ import os from typing import BinaryIO, List, Optional, Union -from langchain_chroma import Chroma from langchain_community.vectorstores.pgvector import PGVector from langchain_core.document_loaders.blob_loaders import Blob from langchain_core.runnables import ( @@ -70,7 +69,7 @@ def _get_embedding_function() -> Union[OpenAIEmbeddings, AzureOpenAIEmbeddings]: def _get_vstore() -> VectorStore: - if settings.storage_type == StorageType.POSTGRES: + if settings.storage_type in (StorageType.POSTGRES, StorageType.SQLITE): PG_CONNECTION_STRING = PGVector.connection_string_from_db_params( driver="psycopg2", host=settings.postgres.host, @@ -84,11 +83,6 @@ def _get_vstore() -> VectorStore: embedding_function=_get_embedding_function(), use_jsonb=True, ) - elif settings.storage_type == StorageType.SQLITE: - return Chroma( - persist_directory="./chroma_db", - embedding_function=_get_embedding_function(), - ) raise NotImplementedError() diff --git a/backend/poetry.lock b/backend/poetry.lock index 34aa74e2..83214a7c 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -188,23 +188,6 @@ files = [ feedparser = "6.0.10" requests = "2.31.0" -[[package]] -name = "asgiref" -version = "3.8.1" -description = "ASGI specs, helper code, and adapters" -optional = false -python-versions = ">=3.8" -files = [ - {file = "asgiref-3.8.1-py3-none-any.whl", hash = "sha256:3e1e3ecc849832fe52ccf2cb6686b7a55f82bb1d6aee72a58826471390335e47"}, - {file = "asgiref-3.8.1.tar.gz", hash = "sha256:c343bd80a0bec947a9860adb4c432ffa7db769836c64238fc34bdc3fec84d590"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4", markers = "python_version < \"3.11\""} - -[package.extras] -tests = ["mypy (>=0.800)", "pytest", "pytest-asyncio"] - [[package]] name = "async-timeout" version = "4.0.3" @@ -302,46 +285,6 @@ files = [ {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, ] -[[package]] -name = "bcrypt" -version = "4.1.2" -description = "Modern password hashing for your software and your servers" -optional = false -python-versions = ">=3.7" -files = [ - {file = "bcrypt-4.1.2-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e"}, - {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1"}, - {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326"}, - {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c"}, - {file = "bcrypt-4.1.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966"}, - {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2"}, - {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c"}, - {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5"}, - {file = "bcrypt-4.1.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0"}, - {file = "bcrypt-4.1.2-cp37-abi3-win32.whl", hash = "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369"}, - {file = "bcrypt-4.1.2-cp37-abi3-win_amd64.whl", hash = "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551"}, - {file = "bcrypt-4.1.2-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63"}, - {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483"}, - {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc"}, - {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7"}, - {file = "bcrypt-4.1.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb"}, - {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1"}, - {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4"}, - {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c"}, - {file = "bcrypt-4.1.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a"}, - {file = "bcrypt-4.1.2-cp39-abi3-win32.whl", hash = "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f"}, - {file = "bcrypt-4.1.2-cp39-abi3-win_amd64.whl", hash = "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42"}, - {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946"}, - {file = "bcrypt-4.1.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d"}, - {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab"}, - {file = "bcrypt-4.1.2-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb"}, - {file = "bcrypt-4.1.2.tar.gz", hash = "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258"}, -] - -[package.extras] -tests = ["pytest (>=3.2.1,!=3.3.0)"] -typecheck = ["mypy"] - [[package]] name = "beautifulsoup4" version = "4.12.3" @@ -404,31 +347,6 @@ urllib3 = [ [package.extras] crt = ["awscrt (==0.19.19)"] -[[package]] -name = "build" -version = "1.2.1" -description = "A simple, correct Python build frontend" -optional = false -python-versions = ">=3.8" -files = [ - {file = "build-1.2.1-py3-none-any.whl", hash = "sha256:75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4"}, - {file = "build-1.2.1.tar.gz", hash = "sha256:526263f4870c26f26c433545579475377b2b7588b6f1eac76a001e873ae3e19d"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "os_name == \"nt\""} -importlib-metadata = {version = ">=4.6", markers = "python_full_version < \"3.10.2\""} -packaging = ">=19.1" -pyproject_hooks = "*" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[package.extras] -docs = ["furo (>=2023.08.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] -test = ["build[uv,virtualenv]", "filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "setuptools (>=56.0.0)", "setuptools (>=67.8.0)", "wheel (>=0.36.0)"] -typing = ["build[uv]", "importlib-metadata (>=5.1)", "mypy (>=1.9.0,<1.10.0)", "tomli", "typing-extensions (>=3.7.4.3)"] -uv = ["uv (>=0.1.18)"] -virtualenv = ["virtualenv (>=20.0.35)"] - [[package]] name = "cachetools" version = "5.3.2" @@ -625,84 +543,6 @@ files = [ {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] -[[package]] -name = "chroma-hnswlib" -version = "0.7.3" -description = "Chromas fork of hnswlib" -optional = false -python-versions = "*" -files = [ - {file = "chroma-hnswlib-0.7.3.tar.gz", hash = "sha256:b6137bedde49fffda6af93b0297fe00429fc61e5a072b1ed9377f909ed95a932"}, - {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:59d6a7c6f863c67aeb23e79a64001d537060b6995c3eca9a06e349ff7b0998ca"}, - {file = "chroma_hnswlib-0.7.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d71a3f4f232f537b6152947006bd32bc1629a8686df22fd97777b70f416c127a"}, - {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c92dc1ebe062188e53970ba13f6b07e0ae32e64c9770eb7f7ffa83f149d4210"}, - {file = "chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49da700a6656fed8753f68d44b8cc8ae46efc99fc8a22a6d970dc1697f49b403"}, - {file = "chroma_hnswlib-0.7.3-cp310-cp310-win_amd64.whl", hash = "sha256:108bc4c293d819b56476d8f7865803cb03afd6ca128a2a04d678fffc139af029"}, - {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:11e7ca93fb8192214ac2b9c0943641ac0daf8f9d4591bb7b73be808a83835667"}, - {file = "chroma_hnswlib-0.7.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6f552e4d23edc06cdeb553cdc757d2fe190cdeb10d43093d6a3319f8d4bf1c6b"}, - {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f96f4d5699e486eb1fb95849fe35ab79ab0901265805be7e60f4eaa83ce263ec"}, - {file = "chroma_hnswlib-0.7.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:368e57fe9ebae05ee5844840fa588028a023d1182b0cfdb1d13f607c9ea05756"}, - {file = "chroma_hnswlib-0.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:b7dca27b8896b494456db0fd705b689ac6b73af78e186eb6a42fea2de4f71c6f"}, - {file = "chroma_hnswlib-0.7.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:70f897dc6218afa1d99f43a9ad5eb82f392df31f57ff514ccf4eeadecd62f544"}, - {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5aef10b4952708f5a1381c124a29aead0c356f8d7d6e0b520b778aaa62a356f4"}, - {file = "chroma_hnswlib-0.7.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee2d8d1529fca3898d512079144ec3e28a81d9c17e15e0ea4665697a7923253"}, - {file = "chroma_hnswlib-0.7.3-cp37-cp37m-win_amd64.whl", hash = "sha256:a4021a70e898783cd6f26e00008b494c6249a7babe8774e90ce4766dd288c8ba"}, - {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a8f61fa1d417fda848e3ba06c07671f14806a2585272b175ba47501b066fe6b1"}, - {file = "chroma_hnswlib-0.7.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7563be58bc98e8f0866907368e22ae218d6060601b79c42f59af4eccbbd2e0a"}, - {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51b8d411486ee70d7b66ec08cc8b9b6620116b650df9c19076d2d8b6ce2ae914"}, - {file = "chroma_hnswlib-0.7.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d706782b628e4f43f1b8a81e9120ac486837fbd9bcb8ced70fe0d9b95c72d77"}, - {file = "chroma_hnswlib-0.7.3-cp38-cp38-win_amd64.whl", hash = "sha256:54f053dedc0e3ba657f05fec6e73dd541bc5db5b09aa8bc146466ffb734bdc86"}, - {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e607c5a71c610a73167a517062d302c0827ccdd6e259af6e4869a5c1306ffb5d"}, - {file = "chroma_hnswlib-0.7.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2358a795870156af6761890f9eb5ca8cade57eb10c5f046fe94dae1faa04b9e"}, - {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cea425df2e6b8a5e201fff0d922a1cc1d165b3cfe762b1408075723c8892218"}, - {file = "chroma_hnswlib-0.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:454df3dd3e97aa784fba7cf888ad191e0087eef0fd8c70daf28b753b3b591170"}, - {file = "chroma_hnswlib-0.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:df587d15007ca701c6de0ee7d5585dd5e976b7edd2b30ac72bc376b3c3f85882"}, -] - -[package.dependencies] -numpy = "*" - -[[package]] -name = "chromadb" -version = "0.4.24" -description = "Chroma." -optional = false -python-versions = ">=3.8" -files = [ - {file = "chromadb-0.4.24-py3-none-any.whl", hash = "sha256:3a08e237a4ad28b5d176685bd22429a03717fe09d35022fb230d516108da01da"}, - {file = "chromadb-0.4.24.tar.gz", hash = "sha256:a5c80b4e4ad9b236ed2d4899a5b9e8002b489293f2881cb2cadab5b199ee1c72"}, -] - -[package.dependencies] -bcrypt = ">=4.0.1" -build = ">=1.0.3" -chroma-hnswlib = "0.7.3" -fastapi = ">=0.95.2" -grpcio = ">=1.58.0" -importlib-resources = "*" -kubernetes = ">=28.1.0" -mmh3 = ">=4.0.1" -numpy = ">=1.22.5" -onnxruntime = ">=1.14.1" -opentelemetry-api = ">=1.2.0" -opentelemetry-exporter-otlp-proto-grpc = ">=1.2.0" -opentelemetry-instrumentation-fastapi = ">=0.41b0" -opentelemetry-sdk = ">=1.2.0" -orjson = ">=3.9.12" -overrides = ">=7.3.1" -posthog = ">=2.4.0" -pulsar-client = ">=3.1.0" -pydantic = ">=1.9" -pypika = ">=0.48.9" -PyYAML = ">=6.0.0" -requests = ">=2.28" -tenacity = ">=8.2.3" -tokenizers = ">=0.13.2" -tqdm = ">=4.65.0" -typer = ">=0.9.0" -typing-extensions = ">=4.5.0" -uvicorn = {version = ">=0.18.3", extras = ["standard"]} - [[package]] name = "click" version = "8.1.7" @@ -745,23 +585,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "coloredlogs" -version = "15.0.1" -description = "Colored terminal output for Python's logging module" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, - {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, -] - -[package.dependencies] -humanfriendly = ">=9.1" - -[package.extras] -cron = ["capturer (>=2.4)"] - [[package]] name = "coverage" version = "7.3.2" @@ -1115,17 +938,6 @@ httpx-sse = "*" Pillow = "*" pydantic = "*" -[[package]] -name = "flatbuffers" -version = "24.3.25" -description = "The FlatBuffers serialization format for Python" -optional = false -python-versions = "*" -files = [ - {file = "flatbuffers-24.3.25-py2.py3-none-any.whl", hash = "sha256:8dbdec58f935f3765e4f7f3cf635ac3a77f83568138d6a2311f524ec96364812"}, - {file = "flatbuffers-24.3.25.tar.gz", hash = "sha256:de2ec5b203f21441716617f38443e0a8ebf3d25bf0d9c0bb0ce68fa00ad546a4"}, -] - [[package]] name = "frozenlist" version = "1.4.0" @@ -1733,54 +1545,6 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] trio = ["trio (>=0.22.0,<0.23.0)"] -[[package]] -name = "httptools" -version = "0.6.1" -description = "A collection of framework independent HTTP protocol utils." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d2f6c3c4cb1948d912538217838f6e9960bc4a521d7f9b323b3da579cd14532f"}, - {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:00d5d4b68a717765b1fabfd9ca755bd12bf44105eeb806c03d1962acd9b8e563"}, - {file = "httptools-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:639dc4f381a870c9ec860ce5c45921db50205a37cc3334e756269736ff0aac58"}, - {file = "httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e57997ac7fb7ee43140cc03664de5f268813a481dff6245e0075925adc6aa185"}, - {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0ac5a0ae3d9f4fe004318d64b8a854edd85ab76cffbf7ef5e32920faef62f142"}, - {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3f30d3ce413088a98b9db71c60a6ada2001a08945cb42dd65a9a9fe228627658"}, - {file = "httptools-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:1ed99a373e327f0107cb513b61820102ee4f3675656a37a50083eda05dc9541b"}, - {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7a7ea483c1a4485c71cb5f38be9db078f8b0e8b4c4dc0210f531cdd2ddac1ef1"}, - {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85ed077c995e942b6f1b07583e4eb0a8d324d418954fc6af913d36db7c05a5a0"}, - {file = "httptools-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b0bb634338334385351a1600a73e558ce619af390c2b38386206ac6a27fecfc"}, - {file = "httptools-0.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9ceb2c957320def533671fc9c715a80c47025139c8d1f3797477decbc6edd2"}, - {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4f0f8271c0a4db459f9dc807acd0eadd4839934a4b9b892f6f160e94da309837"}, - {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6a4f5ccead6d18ec072ac0b84420e95d27c1cdf5c9f1bc8fbd8daf86bd94f43d"}, - {file = "httptools-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:5cceac09f164bcba55c0500a18fe3c47df29b62353198e4f37bbcc5d591172c3"}, - {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75c8022dca7935cba14741a42744eee13ba05db00b27a4b940f0d646bd4d56d0"}, - {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:48ed8129cd9a0d62cf4d1575fcf90fb37e3ff7d5654d3a5814eb3d55f36478c2"}, - {file = "httptools-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f58e335a1402fb5a650e271e8c2d03cfa7cea46ae124649346d17bd30d59c90"}, - {file = "httptools-0.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ad80d7176aa5788902f207a4e79885f0576134695dfb0fefc15b7a4648d503"}, - {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9bb68d3a085c2174c2477eb3ffe84ae9fb4fde8792edb7bcd09a1d8467e30a84"}, - {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b512aa728bc02354e5ac086ce76c3ce635b62f5fbc32ab7082b5e582d27867bb"}, - {file = "httptools-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:97662ce7fb196c785344d00d638fc9ad69e18ee4bfb4000b35a52efe5adcc949"}, - {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8e216a038d2d52ea13fdd9b9c9c7459fb80d78302b257828285eca1c773b99b3"}, - {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3e802e0b2378ade99cd666b5bffb8b2a7cc8f3d28988685dc300469ea8dd86cb"}, - {file = "httptools-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd3e488b447046e386a30f07af05f9b38d3d368d1f7b4d8f7e10af85393db97"}, - {file = "httptools-0.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe467eb086d80217b7584e61313ebadc8d187a4d95bb62031b7bab4b205c3ba3"}, - {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3c3b214ce057c54675b00108ac42bacf2ab8f85c58e3f324a4e963bbc46424f4"}, - {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8ae5b97f690badd2ca27cbf668494ee1b6d34cf1c464271ef7bfa9ca6b83ffaf"}, - {file = "httptools-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:405784577ba6540fa7d6ff49e37daf104e04f4b4ff2d1ac0469eaa6a20fde084"}, - {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:95fb92dd3649f9cb139e9c56604cc2d7c7bf0fc2e7c8d7fbd58f96e35eddd2a3"}, - {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dcbab042cc3ef272adc11220517278519adf8f53fd3056d0e68f0a6f891ba94e"}, - {file = "httptools-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cf2372e98406efb42e93bfe10f2948e467edfd792b015f1b4ecd897903d3e8d"}, - {file = "httptools-0.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:678fcbae74477a17d103b7cae78b74800d795d702083867ce160fc202104d0da"}, - {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e0b281cf5a125c35f7f6722b65d8542d2e57331be573e9e88bc8b0115c4a7a81"}, - {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:95658c342529bba4e1d3d2b1a874db16c7cca435e8827422154c9da76ac4e13a"}, - {file = "httptools-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ebaec1bf683e4bf5e9fbb49b8cc36da482033596a415b3e4ebab5a4c0d7ec5e"}, - {file = "httptools-0.6.1.tar.gz", hash = "sha256:c6e26c30455600b95d94b1b836085138e82f177351454ee841c148f93a9bad5a"}, -] - -[package.extras] -test = ["Cython (>=0.29.24,<0.30.0)"] - [[package]] name = "httpx" version = "0.25.2" @@ -1849,20 +1613,6 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jed torch = ["torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] -[[package]] -name = "humanfriendly" -version = "10.0" -description = "Human friendly output for text interfaces using Python" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"}, - {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"}, -] - -[package.dependencies] -pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""} - [[package]] name = "idna" version = "3.6" @@ -1874,43 +1624,6 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] -[[package]] -name = "importlib-metadata" -version = "7.0.0" -description = "Read metadata from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_metadata-7.0.0-py3-none-any.whl", hash = "sha256:d97503976bb81f40a193d41ee6570868479c69d5068651eb039c40d850c59d67"}, - {file = "importlib_metadata-7.0.0.tar.gz", hash = "sha256:7fc841f8b8332803464e5dc1c63a2e59121f46ca186c0e2e182e80bf8c1319f7"}, -] - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -perf = ["ipython"] -testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] - -[[package]] -name = "importlib-resources" -version = "6.4.0" -description = "Read resources from Python packages" -optional = false -python-versions = ">=3.8" -files = [ - {file = "importlib_resources-6.4.0-py3-none-any.whl", hash = "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c"}, - {file = "importlib_resources-6.4.0.tar.gz", hash = "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145"}, -] - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["jaraco.test (>=5.4)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"] - [[package]] name = "iniconfig" version = "2.0.0" @@ -1993,32 +1706,6 @@ files = [ [package.dependencies] requests = ">=2" -[[package]] -name = "kubernetes" -version = "29.0.0" -description = "Kubernetes python client" -optional = false -python-versions = ">=3.6" -files = [ - {file = "kubernetes-29.0.0-py2.py3-none-any.whl", hash = "sha256:ab8cb0e0576ccdfb71886366efb102c6a20f268d817be065ce7f9909c631e43e"}, - {file = "kubernetes-29.0.0.tar.gz", hash = "sha256:c4812e227ae74d07d53c88293e564e54b850452715a59a927e7e1bc6b9a60459"}, -] - -[package.dependencies] -certifi = ">=14.05.14" -google-auth = ">=1.0.1" -oauthlib = ">=3.2.2" -python-dateutil = ">=2.5.3" -pyyaml = ">=5.4.1" -requests = "*" -requests-oauthlib = "*" -six = ">=1.9.0" -urllib3 = ">=1.24.2" -websocket-client = ">=0.32.0,<0.40.0 || >0.40.0,<0.41.dev0 || >=0.43.dev0" - -[package.extras] -adal = ["adal (>=1.0.2)"] - [[package]] name = "langchain" version = "0.1.10" @@ -2076,23 +1763,6 @@ anthropic = ">=0.23.0,<1" defusedxml = ">=0.7.1,<0.8.0" langchain-core = ">=0.1.42,<0.2.0" -[[package]] -name = "langchain-chroma" -version = "0.1.0" -description = "An integration package connecting Chroma and LangChain" -optional = false -python-versions = "<3.13,>=3.8.1" -files = [ - {file = "langchain_chroma-0.1.0-py3-none-any.whl", hash = "sha256:a9b5624009f4e436f2a7273333c590594daabec6b5a36c0e31ccce29bb5cebf5"}, - {file = "langchain_chroma-0.1.0.tar.gz", hash = "sha256:4f17b0e633b2c182cc546850275420d939d587435c6ebbd821e5d03829539a7a"}, -] - -[package.dependencies] -chromadb = ">=0.4.0,<0.5.0" -fastapi = ">=0.95.2,<1" -langchain-core = ">=0.1.40,<0.2.0" -numpy = ">=1,<2" - [[package]] name = "langchain-community" version = "0.0.29" @@ -2369,30 +2039,6 @@ html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] source = ["Cython (>=3.0.7)"] -[[package]] -name = "markdown-it-py" -version = "3.0.0" -description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false -python-versions = ">=3.8" -files = [ - {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, - {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, -] - -[package.dependencies] -mdurl = ">=0.1,<1.0" - -[package.extras] -benchmarking = ["psutil", "pytest", "pytest-benchmark"] -code-style = ["pre-commit (>=3.0,<4.0)"] -compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] -linkify = ["linkify-it-py (>=1,<3)"] -plugins = ["mdit-py-plugins"] -profiling = ["gprof2dot"] -rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] -testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] - [[package]] name = "marshmallow" version = "3.20.1" @@ -2413,137 +2059,6 @@ docs = ["alabaster (==0.7.13)", "autodocsumm (==0.2.11)", "sphinx (==7.0.1)", "s lint = ["flake8 (==6.0.0)", "flake8-bugbear (==23.7.10)", "mypy (==1.4.1)", "pre-commit (>=2.4,<4.0)"] tests = ["pytest", "pytz", "simplejson"] -[[package]] -name = "mdurl" -version = "0.1.2" -description = "Markdown URL utilities" -optional = false -python-versions = ">=3.7" -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - -[[package]] -name = "mmh3" -version = "4.1.0" -description = "Python extension for MurmurHash (MurmurHash3), a set of fast and robust hash functions." -optional = false -python-versions = "*" -files = [ - {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be5ac76a8b0cd8095784e51e4c1c9c318c19edcd1709a06eb14979c8d850c31a"}, - {file = "mmh3-4.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:98a49121afdfab67cd80e912b36404139d7deceb6773a83620137aaa0da5714c"}, - {file = "mmh3-4.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5259ac0535874366e7d1a5423ef746e0d36a9e3c14509ce6511614bdc5a7ef5b"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5950827ca0453a2be357696da509ab39646044e3fa15cad364eb65d78797437"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1dd0f652ae99585b9dd26de458e5f08571522f0402155809fd1dc8852a613a39"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99d25548070942fab1e4a6f04d1626d67e66d0b81ed6571ecfca511f3edf07e6"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53db8d9bad3cb66c8f35cbc894f336273f63489ce4ac416634932e3cbe79eb5b"}, - {file = "mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75da0f615eb55295a437264cc0b736753f830b09d102aa4c2a7d719bc445ec05"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b926b07fd678ea84b3a2afc1fa22ce50aeb627839c44382f3d0291e945621e1a"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:c5b053334f9b0af8559d6da9dc72cef0a65b325ebb3e630c680012323c950bb6"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:5bf33dc43cd6de2cb86e0aa73a1cc6530f557854bbbe5d59f41ef6de2e353d7b"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:fa7eacd2b830727ba3dd65a365bed8a5c992ecd0c8348cf39a05cc77d22f4970"}, - {file = "mmh3-4.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:42dfd6742b9e3eec599f85270617debfa0bbb913c545bb980c8a4fa7b2d047da"}, - {file = "mmh3-4.1.0-cp310-cp310-win32.whl", hash = "sha256:2974ad343f0d39dcc88e93ee6afa96cedc35a9883bc067febd7ff736e207fa47"}, - {file = "mmh3-4.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:74699a8984ded645c1a24d6078351a056f5a5f1fe5838870412a68ac5e28d865"}, - {file = "mmh3-4.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:f0dc874cedc23d46fc488a987faa6ad08ffa79e44fb08e3cd4d4cf2877c00a00"}, - {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3280a463855b0eae64b681cd5b9ddd9464b73f81151e87bb7c91a811d25619e6"}, - {file = "mmh3-4.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:97ac57c6c3301769e757d444fa7c973ceb002cb66534b39cbab5e38de61cd896"}, - {file = "mmh3-4.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a7b6502cdb4dbd880244818ab363c8770a48cdccecf6d729ade0241b736b5ec0"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52ba2da04671a9621580ddabf72f06f0e72c1c9c3b7b608849b58b11080d8f14"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a5fef4c4ecc782e6e43fbeab09cff1bac82c998a1773d3a5ee6a3605cde343e"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5135358a7e00991f73b88cdc8eda5203bf9de22120d10a834c5761dbeb07dd13"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cff9ae76a54f7c6fe0167c9c4028c12c1f6de52d68a31d11b6790bb2ae685560"}, - {file = "mmh3-4.1.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6f02576a4d106d7830ca90278868bf0983554dd69183b7bbe09f2fcd51cf54f"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:073d57425a23721730d3ff5485e2da489dd3c90b04e86243dd7211f889898106"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:71e32ddec7f573a1a0feb8d2cf2af474c50ec21e7a8263026e8d3b4b629805db"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7cbb20b29d57e76a58b40fd8b13a9130db495a12d678d651b459bf61c0714cea"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a42ad267e131d7847076bb7e31050f6c4378cd38e8f1bf7a0edd32f30224d5c9"}, - {file = "mmh3-4.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a013979fc9390abadc445ea2527426a0e7a4495c19b74589204f9b71bcaafeb"}, - {file = "mmh3-4.1.0-cp311-cp311-win32.whl", hash = "sha256:1d3b1cdad7c71b7b88966301789a478af142bddcb3a2bee563f7a7d40519a00f"}, - {file = "mmh3-4.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:0dc6dc32eb03727467da8e17deffe004fbb65e8b5ee2b502d36250d7a3f4e2ec"}, - {file = "mmh3-4.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:9ae3a5c1b32dda121c7dc26f9597ef7b01b4c56a98319a7fe86c35b8bc459ae6"}, - {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0033d60c7939168ef65ddc396611077a7268bde024f2c23bdc283a19123f9e9c"}, - {file = "mmh3-4.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d6af3e2287644b2b08b5924ed3a88c97b87b44ad08e79ca9f93d3470a54a41c5"}, - {file = "mmh3-4.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d82eb4defa245e02bb0b0dc4f1e7ee284f8d212633389c91f7fba99ba993f0a2"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba245e94b8d54765e14c2d7b6214e832557e7856d5183bc522e17884cab2f45d"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb04e2feeabaad6231e89cd43b3d01a4403579aa792c9ab6fdeef45cc58d4ec0"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e3b1a27def545ce11e36158ba5d5390cdbc300cfe456a942cc89d649cf7e3b2"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce0ab79ff736d7044e5e9b3bfe73958a55f79a4ae672e6213e92492ad5e734d5"}, - {file = "mmh3-4.1.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b02268be6e0a8eeb8a924d7db85f28e47344f35c438c1e149878bb1c47b1cd3"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:deb887f5fcdaf57cf646b1e062d56b06ef2f23421c80885fce18b37143cba828"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:99dd564e9e2b512eb117bd0cbf0f79a50c45d961c2a02402787d581cec5448d5"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:08373082dfaa38fe97aa78753d1efd21a1969e51079056ff552e687764eafdfe"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:54b9c6a2ea571b714e4fe28d3e4e2db37abfd03c787a58074ea21ee9a8fd1740"}, - {file = "mmh3-4.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a7b1edf24c69e3513f879722b97ca85e52f9032f24a52284746877f6a7304086"}, - {file = "mmh3-4.1.0-cp312-cp312-win32.whl", hash = "sha256:411da64b951f635e1e2284b71d81a5a83580cea24994b328f8910d40bed67276"}, - {file = "mmh3-4.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:bebc3ecb6ba18292e3d40c8712482b4477abd6981c2ebf0e60869bd90f8ac3a9"}, - {file = "mmh3-4.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:168473dd608ade6a8d2ba069600b35199a9af837d96177d3088ca91f2b3798e3"}, - {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:372f4b7e1dcde175507640679a2a8790185bb71f3640fc28a4690f73da986a3b"}, - {file = "mmh3-4.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:438584b97f6fe13e944faf590c90fc127682b57ae969f73334040d9fa1c7ffa5"}, - {file = "mmh3-4.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6e27931b232fc676675fac8641c6ec6b596daa64d82170e8597f5a5b8bdcd3b6"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:571a92bad859d7b0330e47cfd1850b76c39b615a8d8e7aa5853c1f971fd0c4b1"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a69d6afe3190fa08f9e3a58e5145549f71f1f3fff27bd0800313426929c7068"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:afb127be0be946b7630220908dbea0cee0d9d3c583fa9114a07156f98566dc28"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:940d86522f36348ef1a494cbf7248ab3f4a1638b84b59e6c9e90408bd11ad729"}, - {file = "mmh3-4.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3dcccc4935686619a8e3d1f7b6e97e3bd89a4a796247930ee97d35ea1a39341"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:01bb9b90d61854dfc2407c5e5192bfb47222d74f29d140cb2dd2a69f2353f7cc"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bcb1b8b951a2c0b0fb8a5426c62a22557e2ffc52539e0a7cc46eb667b5d606a9"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6477a05d5e5ab3168e82e8b106e316210ac954134f46ec529356607900aea82a"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:da5892287e5bea6977364b15712a2573c16d134bc5fdcdd4cf460006cf849278"}, - {file = "mmh3-4.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:99180d7fd2327a6fffbaff270f760576839dc6ee66d045fa3a450f3490fda7f5"}, - {file = "mmh3-4.1.0-cp38-cp38-win32.whl", hash = "sha256:9b0d4f3949913a9f9a8fb1bb4cc6ecd52879730aab5ff8c5a3d8f5b593594b73"}, - {file = "mmh3-4.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:598c352da1d945108aee0c3c3cfdd0e9b3edef74108f53b49d481d3990402169"}, - {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:475d6d1445dd080f18f0f766277e1237fa2914e5fe3307a3b2a3044f30892103"}, - {file = "mmh3-4.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5ca07c41e6a2880991431ac717c2a049056fff497651a76e26fc22224e8b5732"}, - {file = "mmh3-4.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ebe052fef4bbe30c0548d12ee46d09f1b69035ca5208a7075e55adfe091be44"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eaefd42e85afb70f2b855a011f7b4d8a3c7e19c3f2681fa13118e4d8627378c5"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0ae43caae5a47afe1b63a1ae3f0986dde54b5fb2d6c29786adbfb8edc9edfb"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6218666f74c8c013c221e7f5f8a693ac9cf68e5ac9a03f2373b32d77c48904de"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ac59294a536ba447b5037f62d8367d7d93b696f80671c2c45645fa9f1109413c"}, - {file = "mmh3-4.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:086844830fcd1e5c84fec7017ea1ee8491487cfc877847d96f86f68881569d2e"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e42b38fad664f56f77f6fbca22d08450f2464baa68acdbf24841bf900eb98e87"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d08b790a63a9a1cde3b5d7d733ed97d4eb884bfbc92f075a091652d6bfd7709a"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:73ea4cc55e8aea28c86799ecacebca09e5f86500414870a8abaedfcbaf74d288"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:f90938ff137130e47bcec8dc1f4ceb02f10178c766e2ef58a9f657ff1f62d124"}, - {file = "mmh3-4.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:aa1f13e94b8631c8cd53259250556edcf1de71738936b60febba95750d9632bd"}, - {file = "mmh3-4.1.0-cp39-cp39-win32.whl", hash = "sha256:a3b680b471c181490cf82da2142029edb4298e1bdfcb67c76922dedef789868d"}, - {file = "mmh3-4.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:fefef92e9c544a8dbc08f77a8d1b6d48006a750c4375bbcd5ff8199d761e263b"}, - {file = "mmh3-4.1.0-cp39-cp39-win_arm64.whl", hash = "sha256:8e2c1f6a2b41723a4f82bd5a762a777836d29d664fc0095f17910bea0adfd4a6"}, - {file = "mmh3-4.1.0.tar.gz", hash = "sha256:a1cf25348b9acd229dda464a094d6170f47d2850a1fcb762a3b6172d2ce6ca4a"}, -] - -[package.extras] -test = ["mypy (>=1.0)", "pytest (>=7.0.0)"] - -[[package]] -name = "monotonic" -version = "1.6" -description = "An implementation of time.monotonic() for Python 2 & < 3.3" -optional = false -python-versions = "*" -files = [ - {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"}, - {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -description = "Python library for arbitrary-precision floating-point arithmetic" -optional = false -python-versions = "*" -files = [ - {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, - {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, -] - -[package.extras] -develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] -docs = ["sphinx"] -gmpy = ["gmpy2 (>=2.1.0a4)"] -tests = ["pytest (>=4.6)"] - [[package]] name = "multidict" version = "6.0.4" @@ -2700,64 +2215,6 @@ files = [ {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] -[[package]] -name = "oauthlib" -version = "3.2.2" -description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -optional = false -python-versions = ">=3.6" -files = [ - {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, - {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, -] - -[package.extras] -rsa = ["cryptography (>=3.0.0)"] -signals = ["blinker (>=1.4.0)"] -signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] - -[[package]] -name = "onnxruntime" -version = "1.17.3" -description = "ONNX Runtime is a runtime accelerator for Machine Learning models" -optional = false -python-versions = "*" -files = [ - {file = "onnxruntime-1.17.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:d86dde9c0bb435d709e51bd25991c9fe5b9a5b168df45ce119769edc4d198b15"}, - {file = "onnxruntime-1.17.3-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9d87b68bf931ac527b2d3c094ead66bb4381bac4298b65f46c54fe4d1e255865"}, - {file = "onnxruntime-1.17.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:26e950cf0333cf114a155f9142e71da344d2b08dfe202763a403ae81cc02ebd1"}, - {file = "onnxruntime-1.17.3-cp310-cp310-win32.whl", hash = "sha256:0962a4d0f5acebf62e1f0bf69b6e0adf16649115d8de854c1460e79972324d68"}, - {file = "onnxruntime-1.17.3-cp310-cp310-win_amd64.whl", hash = "sha256:468ccb8a0faa25c681a41787b1594bf4448b0252d3efc8b62fd8b2411754340f"}, - {file = "onnxruntime-1.17.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e8cd90c1c17d13d47b89ab076471e07fb85467c01dcd87a8b8b5cdfbcb40aa51"}, - {file = "onnxruntime-1.17.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a058b39801baefe454eeb8acf3ada298c55a06a4896fafc224c02d79e9037f60"}, - {file = "onnxruntime-1.17.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2f823d5eb4807007f3da7b27ca972263df6a1836e6f327384eb266274c53d05d"}, - {file = "onnxruntime-1.17.3-cp311-cp311-win32.whl", hash = "sha256:b66b23f9109e78ff2791628627a26f65cd335dcc5fbd67ff60162733a2f7aded"}, - {file = "onnxruntime-1.17.3-cp311-cp311-win_amd64.whl", hash = "sha256:570760ca53a74cdd751ee49f13de70d1384dcf73d9888b8deac0917023ccda6d"}, - {file = "onnxruntime-1.17.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:77c318178d9c16e9beadd9a4070d8aaa9f57382c3f509b01709f0f010e583b99"}, - {file = "onnxruntime-1.17.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23da8469049b9759082e22c41a444f44a520a9c874b084711b6343672879f50b"}, - {file = "onnxruntime-1.17.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2949730215af3f9289008b2e31e9bbef952012a77035b911c4977edea06f3f9e"}, - {file = "onnxruntime-1.17.3-cp312-cp312-win32.whl", hash = "sha256:6c7555a49008f403fb3b19204671efb94187c5085976ae526cb625f6ede317bc"}, - {file = "onnxruntime-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:58672cf20293a1b8a277a5c6c55383359fcdf6119b2f14df6ce3b140f5001c39"}, - {file = "onnxruntime-1.17.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:4395ba86e3c1e93c794a00619ef1aec597ab78f5a5039f3c6d2e9d0695c0a734"}, - {file = "onnxruntime-1.17.3-cp38-cp38-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bdf354c04344ec38564fc22394e1fe08aa6d70d790df00159205a0055c4a4d3f"}, - {file = "onnxruntime-1.17.3-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a94b600b7af50e922d44b95a57981e3e35103c6e3693241a03d3ca204740bbda"}, - {file = "onnxruntime-1.17.3-cp38-cp38-win32.whl", hash = "sha256:5a335c76f9c002a8586c7f38bc20fe4b3725ced21f8ead835c3e4e507e42b2ab"}, - {file = "onnxruntime-1.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:8f56a86fbd0ddc8f22696ddeda0677b041381f4168a2ca06f712ef6ec6050d6d"}, - {file = "onnxruntime-1.17.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:e0ae39f5452278cd349520c296e7de3e90d62dc5b0157c6868e2748d7f28b871"}, - {file = "onnxruntime-1.17.3-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ff2dc012bd930578aff5232afd2905bf16620815f36783a941aafabf94b3702"}, - {file = "onnxruntime-1.17.3-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf6c37483782e4785019b56e26224a25e9b9a35b849d0169ce69189867a22bb1"}, - {file = "onnxruntime-1.17.3-cp39-cp39-win32.whl", hash = "sha256:351bf5a1140dcc43bfb8d3d1a230928ee61fcd54b0ea664c8e9a889a8e3aa515"}, - {file = "onnxruntime-1.17.3-cp39-cp39-win_amd64.whl", hash = "sha256:57a3de15778da8d6cc43fbf6cf038e1e746146300b5f0b1fbf01f6f795dc6440"}, -] - -[package.dependencies] -coloredlogs = "*" -flatbuffers = "*" -numpy = ">=1.21.6" -packaging = "*" -protobuf = "*" -sympy = "*" - [[package]] name = "openai" version = "1.10.0" @@ -2781,168 +2238,6 @@ typing-extensions = ">=4.7,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] -[[package]] -name = "opentelemetry-api" -version = "1.24.0" -description = "OpenTelemetry Python API" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_api-1.24.0-py3-none-any.whl", hash = "sha256:0f2c363d98d10d1ce93330015ca7fd3a65f60be64e05e30f557c61de52c80ca2"}, - {file = "opentelemetry_api-1.24.0.tar.gz", hash = "sha256:42719f10ce7b5a9a73b10a4baf620574fb8ad495a9cbe5c18d76b75d8689c67e"}, -] - -[package.dependencies] -deprecated = ">=1.2.6" -importlib-metadata = ">=6.0,<=7.0" - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.24.0" -description = "OpenTelemetry Protobuf encoding" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_exporter_otlp_proto_common-1.24.0-py3-none-any.whl", hash = "sha256:e51f2c9735054d598ad2df5d3eca830fecfb5b0bda0a2fa742c9c7718e12f641"}, - {file = "opentelemetry_exporter_otlp_proto_common-1.24.0.tar.gz", hash = "sha256:5d31fa1ff976cacc38be1ec4e3279a3f88435c75b38b1f7a099a1faffc302461"}, -] - -[package.dependencies] -opentelemetry-proto = "1.24.0" - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.24.0" -description = "OpenTelemetry Collector Protobuf over gRPC Exporter" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_exporter_otlp_proto_grpc-1.24.0-py3-none-any.whl", hash = "sha256:f40d62aa30a0a43cc1657428e59fcf82ad5f7ea8fff75de0f9d9cb6f739e0a3b"}, - {file = "opentelemetry_exporter_otlp_proto_grpc-1.24.0.tar.gz", hash = "sha256:217c6e30634f2c9797999ea9da29f7300479a94a610139b9df17433f915e7baa"}, -] - -[package.dependencies] -deprecated = ">=1.2.6" -googleapis-common-protos = ">=1.52,<2.0" -grpcio = ">=1.0.0,<2.0.0" -opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.24.0" -opentelemetry-proto = "1.24.0" -opentelemetry-sdk = ">=1.24.0,<1.25.0" - -[package.extras] -test = ["pytest-grpc"] - -[[package]] -name = "opentelemetry-instrumentation" -version = "0.45b0" -description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_instrumentation-0.45b0-py3-none-any.whl", hash = "sha256:06c02e2c952c1b076e8eaedf1b82f715e2937ba7eeacab55913dd434fbcec258"}, - {file = "opentelemetry_instrumentation-0.45b0.tar.gz", hash = "sha256:6c47120a7970bbeb458e6a73686ee9ba84b106329a79e4a4a66761f933709c7e"}, -] - -[package.dependencies] -opentelemetry-api = ">=1.4,<2.0" -setuptools = ">=16.0" -wrapt = ">=1.0.0,<2.0.0" - -[[package]] -name = "opentelemetry-instrumentation-asgi" -version = "0.45b0" -description = "ASGI instrumentation for OpenTelemetry" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_instrumentation_asgi-0.45b0-py3-none-any.whl", hash = "sha256:8be1157ed62f0db24e45fdf7933c530c4338bd025c5d4af7830e903c0756021b"}, - {file = "opentelemetry_instrumentation_asgi-0.45b0.tar.gz", hash = "sha256:97f55620f163fd3d20323e9fd8dc3aacc826c03397213ff36b877e0f4b6b08a6"}, -] - -[package.dependencies] -asgiref = ">=3.0,<4.0" -opentelemetry-api = ">=1.12,<2.0" -opentelemetry-instrumentation = "0.45b0" -opentelemetry-semantic-conventions = "0.45b0" -opentelemetry-util-http = "0.45b0" - -[package.extras] -instruments = ["asgiref (>=3.0,<4.0)"] - -[[package]] -name = "opentelemetry-instrumentation-fastapi" -version = "0.45b0" -description = "OpenTelemetry FastAPI Instrumentation" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_instrumentation_fastapi-0.45b0-py3-none-any.whl", hash = "sha256:77d9c123a363129148f5f66d44094f3d67aaaa2b201396d94782b4a7f9ce4314"}, - {file = "opentelemetry_instrumentation_fastapi-0.45b0.tar.gz", hash = "sha256:5a6b91e1c08a01601845fcfcfdefd0a2aecdb3c356d4a436a3210cb58c21487e"}, -] - -[package.dependencies] -opentelemetry-api = ">=1.12,<2.0" -opentelemetry-instrumentation = "0.45b0" -opentelemetry-instrumentation-asgi = "0.45b0" -opentelemetry-semantic-conventions = "0.45b0" -opentelemetry-util-http = "0.45b0" - -[package.extras] -instruments = ["fastapi (>=0.58,<1.0)"] - -[[package]] -name = "opentelemetry-proto" -version = "1.24.0" -description = "OpenTelemetry Python Proto" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_proto-1.24.0-py3-none-any.whl", hash = "sha256:bcb80e1e78a003040db71ccf83f2ad2019273d1e0828089d183b18a1476527ce"}, - {file = "opentelemetry_proto-1.24.0.tar.gz", hash = "sha256:ff551b8ad63c6cabb1845ce217a6709358dfaba0f75ea1fa21a61ceddc78cab8"}, -] - -[package.dependencies] -protobuf = ">=3.19,<5.0" - -[[package]] -name = "opentelemetry-sdk" -version = "1.24.0" -description = "OpenTelemetry Python SDK" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_sdk-1.24.0-py3-none-any.whl", hash = "sha256:fa731e24efe832e98bcd90902085b359dcfef7d9c9c00eb5b9a18587dae3eb59"}, - {file = "opentelemetry_sdk-1.24.0.tar.gz", hash = "sha256:75bc0563affffa827700e0f4f4a68e1e257db0df13372344aebc6f8a64cde2e5"}, -] - -[package.dependencies] -opentelemetry-api = "1.24.0" -opentelemetry-semantic-conventions = "0.45b0" -typing-extensions = ">=3.7.4" - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.45b0" -description = "OpenTelemetry Semantic Conventions" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_semantic_conventions-0.45b0-py3-none-any.whl", hash = "sha256:a4a6fb9a7bacd9167c082aa4681009e9acdbfa28ffb2387af50c2fef3d30c864"}, - {file = "opentelemetry_semantic_conventions-0.45b0.tar.gz", hash = "sha256:7c84215a44ac846bc4b8e32d5e78935c5c43482e491812a0bb8aaf87e4d92118"}, -] - -[[package]] -name = "opentelemetry-util-http" -version = "0.45b0" -description = "Web util for OpenTelemetry" -optional = false -python-versions = ">=3.8" -files = [ - {file = "opentelemetry_util_http-0.45b0-py3-none-any.whl", hash = "sha256:6628868b501b3004e1860f976f410eeb3d3499e009719d818000f24ce17b6e33"}, - {file = "opentelemetry_util_http-0.45b0.tar.gz", hash = "sha256:4ce08b6a7d52dd7c96b7705b5b4f06fdb6aa3eac1233b3b0bfef8a0cab9a92cd"}, -] - [[package]] name = "orjson" version = "3.10.0" @@ -3003,17 +2298,6 @@ files = [ {file = "orjson-3.10.0.tar.gz", hash = "sha256:ba4d8cac5f2e2cff36bea6b6481cdb92b38c202bcec603d6f5ff91960595a1ed"}, ] -[[package]] -name = "overrides" -version = "7.7.0" -description = "A decorator to automatically detect mismatch when overriding a method." -optional = false -python-versions = ">=3.6" -files = [ - {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, - {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, -] - [[package]] name = "packaging" version = "23.2" @@ -3158,29 +2442,6 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] -[[package]] -name = "posthog" -version = "3.5.0" -description = "Integrate PostHog into any python application." -optional = false -python-versions = "*" -files = [ - {file = "posthog-3.5.0-py2.py3-none-any.whl", hash = "sha256:3c672be7ba6f95d555ea207d4486c171d06657eb34b3ce25eb043bfe7b6b5b76"}, - {file = "posthog-3.5.0.tar.gz", hash = "sha256:8f7e3b2c6e8714d0c0c542a2109b83a7549f63b7113a133ab2763a89245ef2ef"}, -] - -[package.dependencies] -backoff = ">=1.10.0" -monotonic = ">=1.5" -python-dateutil = ">2.1" -requests = ">=2.7,<3.0" -six = ">=1.5" - -[package.extras] -dev = ["black", "flake8", "flake8-print", "isort", "pre-commit"] -sentry = ["django", "sentry-sdk"] -test = ["coverage", "flake8", "freezegun (==0.3.15)", "mock (>=2.0.0)", "pylint", "pytest", "pytest-timeout"] - [[package]] name = "proto-plus" version = "1.23.0" @@ -3299,53 +2560,6 @@ files = [ {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"}, ] -[[package]] -name = "pulsar-client" -version = "3.5.0" -description = "Apache Pulsar Python client library" -optional = false -python-versions = "*" -files = [ - {file = "pulsar_client-3.5.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:c18552edb2f785de85280fe624bc507467152bff810fc81d7660fa2dfa861f38"}, - {file = "pulsar_client-3.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18d438e456c146f01be41ef146f649dedc8f7bc714d9eaef94cff2e34099812b"}, - {file = "pulsar_client-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18a26a0719841103c7a89eb1492c4a8fedf89adaa386375baecbb4fa2707e88f"}, - {file = "pulsar_client-3.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ab0e1605dc5f44a126163fd06cd0a768494ad05123f6e0de89a2c71d6e2d2319"}, - {file = "pulsar_client-3.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cdef720891b97656fdce3bf5913ea7729b2156b84ba64314f432c1e72c6117fa"}, - {file = "pulsar_client-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:a42544e38773191fe550644a90e8050579476bb2dcf17ac69a4aed62a6cb70e7"}, - {file = "pulsar_client-3.5.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:fd94432ea5d398ea78f8f2e09a217ec5058d26330c137a22690478c031e116da"}, - {file = "pulsar_client-3.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6252ae462e07ece4071213fdd9c76eab82ca522a749f2dc678037d4cbacd40b"}, - {file = "pulsar_client-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03b4d440b2d74323784328b082872ee2f206c440b5d224d7941eb3c083ec06c6"}, - {file = "pulsar_client-3.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f60af840b8d64a2fac5a0c1ce6ae0ddffec5f42267c6ded2c5e74bad8345f2a1"}, - {file = "pulsar_client-3.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2277a447c3b7f6571cb1eb9fc5c25da3fdd43d0b2fb91cf52054adfadc7d6842"}, - {file = "pulsar_client-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:f20f3e9dd50db2a37059abccad42078b7a4754b8bc1d3ae6502e71c1ad2209f0"}, - {file = "pulsar_client-3.5.0-cp312-cp312-macosx_10_15_universal2.whl", hash = "sha256:d61f663d85308e12f44033ba95af88730f581a7e8da44f7a5c080a3aaea4878d"}, - {file = "pulsar_client-3.5.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a1ba0be25b6f747bcb28102b7d906ec1de48dc9f1a2d9eacdcc6f44ab2c9e17"}, - {file = "pulsar_client-3.5.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a181e3e60ac39df72ccb3c415d7aeac61ad0286497a6e02739a560d5af28393a"}, - {file = "pulsar_client-3.5.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3c72895ff7f51347e4f78b0375b2213fa70dd4790bbb78177b4002846f1fd290"}, - {file = "pulsar_client-3.5.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:547dba1b185a17eba915e51d0a3aca27c80747b6187e5cd7a71a3ca33921decc"}, - {file = "pulsar_client-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:443b786eed96bc86d2297a6a42e79f39d1abf217ec603e0bd303f3488c0234af"}, - {file = "pulsar_client-3.5.0-cp38-cp38-macosx_10_15_universal2.whl", hash = "sha256:15b58f5d759dd6166db8a2d90ed05a38063b05cda76c36d190d86ef5c9249397"}, - {file = "pulsar_client-3.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af34bfe813dddf772a8a298117fa0a036ee963595d8bc8f00d969a0329ae6ed9"}, - {file = "pulsar_client-3.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27a0fec1dd74e1367d3742ce16679c1807994df60f5e666f440cf39323938fad"}, - {file = "pulsar_client-3.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbcd26ef9c03f96fb9cd91baec3bbd3c4b997834eb3556670d31f41cc25b5f64"}, - {file = "pulsar_client-3.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:afea1d0b6e793fd56e56463145751ff3aa79fdcd5b26e90d0da802a1bbabe07e"}, - {file = "pulsar_client-3.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:da1ab2fb1bef64b966e9403a0a186ebc90368d99e054ce2cae5b1128478f4ef4"}, - {file = "pulsar_client-3.5.0-cp39-cp39-macosx_10_15_universal2.whl", hash = "sha256:9ad5dcc0eb8d2a7c0fb8e1fa146a0c6d4bdaf934f1169080b2c64b2f0573e086"}, - {file = "pulsar_client-3.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5870c6805b1a57962ed908d1173e97e13470415998393925c86a43694420389"}, - {file = "pulsar_client-3.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29cb5fedb969895b78301dc00a979133e69940812b8332e4de948bb0ad3db7cb"}, - {file = "pulsar_client-3.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e53c74bfa59b20c66adea95023169060f5048dd8d843e6ef9cd3b8ee2d23e93b"}, - {file = "pulsar_client-3.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:99dbadb13967f1add57010971ed36b5a77d24afcdaea01960d0e55e56cf4ba6f"}, - {file = "pulsar_client-3.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:058887661d438796f42307dcc8054c84dea88a37683dae36498b95d7e1c39b37"}, -] - -[package.dependencies] -certifi = "*" - -[package.extras] -all = ["apache-bookkeeper-client (>=4.16.1)", "fastavro (>=1.9.2)", "grpcio (>=1.60.0)", "prometheus-client", "protobuf (>=3.6.1,<=3.20.3)", "ratelimit"] -avro = ["fastavro (>=1.9.2)"] -functions = ["apache-bookkeeper-client (>=4.16.1)", "grpcio (>=1.60.0)", "prometheus-client", "protobuf (>=3.6.1,<=3.20.3)", "ratelimit"] - [[package]] name = "pyasn1" version = "0.5.1" @@ -3454,21 +2668,6 @@ requests = ">=2.14.0" typing-extensions = ">=4.0.0" urllib3 = ">=1.26.0" -[[package]] -name = "pygments" -version = "2.17.2" -description = "Pygments is a syntax highlighting package written in Python." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, - {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, -] - -[package.extras] -plugins = ["importlib-metadata"] -windows-terminal = ["colorama (>=0.4.6)"] - [[package]] name = "pyjwt" version = "2.8.0" @@ -3515,41 +2714,6 @@ cffi = ">=1.4.1" docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] -[[package]] -name = "pypika" -version = "0.48.9" -description = "A SQL query builder API for Python" -optional = false -python-versions = "*" -files = [ - {file = "PyPika-0.48.9.tar.gz", hash = "sha256:838836a61747e7c8380cd1b7ff638694b7a7335345d0f559b04b2cd832ad5378"}, -] - -[[package]] -name = "pyproject-hooks" -version = "1.0.0" -description = "Wrappers to call pyproject.toml-based build backend hooks." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pyproject_hooks-1.0.0-py3-none-any.whl", hash = "sha256:283c11acd6b928d2f6a7c73fa0d01cb2bdc5f07c57a2eeb6e83d5e56b97976f8"}, - {file = "pyproject_hooks-1.0.0.tar.gz", hash = "sha256:f271b298b97f5955d53fb12b72c1fb1948c22c1a6b70b315c54cedaca0264ef5"}, -] - -[package.dependencies] -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} - -[[package]] -name = "pyreadline3" -version = "3.4.1" -description = "A python implementation of GNU readline." -optional = false -python-versions = "*" -files = [ - {file = "pyreadline3-3.4.1-py3-none-any.whl", hash = "sha256:b0efb6516fd4fb07b45949053826a62fa4cb353db5be2bbb4a7aa1fdd1e345fb"}, - {file = "pyreadline3-3.4.1.tar.gz", hash = "sha256:6f3d1f7b8a31ba32b73917cefc1f28cc660562f39aea8646d30bd6eff21f7bae"}, -] - [[package]] name = "pytest" version = "7.4.3" @@ -3698,20 +2862,6 @@ files = [ lxml = ">=3.1.0" typing-extensions = "*" -[[package]] -name = "python-dotenv" -version = "1.0.1" -description = "Read key-value pairs from a .env file and set them as environment variables" -optional = false -python-versions = ">=3.8" -files = [ - {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, - {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, -] - -[package.extras] -cli = ["click (>=5.0)"] - [[package]] name = "python-iso639" version = "2024.2.7" @@ -4031,42 +3181,6 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "requests-oauthlib" -version = "2.0.0" -description = "OAuthlib authentication support for Requests." -optional = false -python-versions = ">=3.4" -files = [ - {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"}, - {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"}, -] - -[package.dependencies] -oauthlib = ">=3.0.0" -requests = ">=2.0.0" - -[package.extras] -rsa = ["oauthlib[signedtoken] (>=3.0.0)"] - -[[package]] -name = "rich" -version = "13.7.1" -description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false -python-versions = ">=3.7.0" -files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, -] - -[package.dependencies] -markdown-it-py = ">=2.2.0" -pygments = ">=2.13.0,<3.0.0" - -[package.extras] -jupyter = ["ipywidgets (>=7.5.1,<9)"] - [[package]] name = "rsa" version = "4.9" @@ -4207,17 +3321,6 @@ numpy = ">=1.14" docs = ["matplotlib", "numpydoc (==1.1.*)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"] test = ["pytest", "pytest-cov"] -[[package]] -name = "shellingham" -version = "1.5.4" -description = "Tool to Detect Surrounding Shell" -optional = false -python-versions = ">=3.7" -files = [ - {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, - {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, -] - [[package]] name = "six" version = "1.16.0" @@ -4384,20 +3487,6 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] -[[package]] -name = "sympy" -version = "1.12" -description = "Computer algebra system (CAS) in Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "sympy-1.12-py3-none-any.whl", hash = "sha256:c3588cd4295d0c0f603d0f2ae780587e64e2efeedb3521e46b9bb1d08d184fa5"}, - {file = "sympy-1.12.tar.gz", hash = "sha256:ebf595c8dac3e0fdc4152c51878b498396ec7f30e7a914d6071e674d49420fb8"}, -] - -[package.dependencies] -mpmath = ">=0.19" - [[package]] name = "tabulate" version = "0.9.0" @@ -4647,23 +3736,6 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] -[[package]] -name = "typer" -version = "0.12.3" -description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false -python-versions = ">=3.7" -files = [ - {file = "typer-0.12.3-py3-none-any.whl", hash = "sha256:070d7ca53f785acbccba8e7d28b08dcd88f79f1fbda035ade0aecec71ca5c914"}, - {file = "typer-0.12.3.tar.gz", hash = "sha256:49e73131481d804288ef62598d97a1ceef3058905aa536a1134f90891ba35482"}, -] - -[package.dependencies] -click = ">=8.0.0" -rich = ">=10.11.0" -shellingham = ">=1.3.0" -typing-extensions = ">=3.7.4.3" - [[package]] name = "types-protobuf" version = "4.24.0.20240106" @@ -4908,63 +3980,12 @@ files = [ [package.dependencies] click = ">=7.0" -colorama = {version = ">=0.4", optional = true, markers = "sys_platform == \"win32\" and extra == \"standard\""} h11 = ">=0.8" -httptools = {version = ">=0.5.0", optional = true, markers = "extra == \"standard\""} -python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} -pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} -uvloop = {version = ">=0.14.0,<0.15.0 || >0.15.0,<0.15.1 || >0.15.1", optional = true, markers = "(sys_platform != \"win32\" and sys_platform != \"cygwin\") and platform_python_implementation != \"PyPy\" and extra == \"standard\""} -watchfiles = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} -websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} [package.extras] standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"] -[[package]] -name = "uvloop" -version = "0.19.0" -description = "Fast implementation of asyncio event loop on top of libuv" -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de4313d7f575474c8f5a12e163f6d89c0a878bc49219641d49e6f1444369a90e"}, - {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5588bd21cf1fcf06bded085f37e43ce0e00424197e7c10e77afd4bbefffef428"}, - {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b1fd71c3843327f3bbc3237bedcdb6504fd50368ab3e04d0410e52ec293f5b8"}, - {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a05128d315e2912791de6088c34136bfcdd0c7cbc1cf85fd6fd1bb321b7c849"}, - {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cd81bdc2b8219cb4b2556eea39d2e36bfa375a2dd021404f90a62e44efaaf957"}, - {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f17766fb6da94135526273080f3455a112f82570b2ee5daa64d682387fe0dcd"}, - {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ce6b0af8f2729a02a5d1575feacb2a94fc7b2e983868b009d51c9a9d2149bef"}, - {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:31e672bb38b45abc4f26e273be83b72a0d28d074d5b370fc4dcf4c4eb15417d2"}, - {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:570fc0ed613883d8d30ee40397b79207eedd2624891692471808a95069a007c1"}, - {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5138821e40b0c3e6c9478643b4660bd44372ae1e16a322b8fc07478f92684e24"}, - {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:91ab01c6cd00e39cde50173ba4ec68a1e578fee9279ba64f5221810a9e786533"}, - {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:47bf3e9312f63684efe283f7342afb414eea4d3011542155c7e625cd799c3b12"}, - {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:da8435a3bd498419ee8c13c34b89b5005130a476bda1d6ca8cfdde3de35cd650"}, - {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:02506dc23a5d90e04d4f65c7791e65cf44bd91b37f24cfc3ef6cf2aff05dc7ec"}, - {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693049be9d36fef81741fddb3f441673ba12a34a704e7b4361efb75cf30befc"}, - {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7010271303961c6f0fe37731004335401eb9075a12680738731e9c92ddd96ad6"}, - {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5daa304d2161d2918fa9a17d5635099a2f78ae5b5960e742b2fcfbb7aefaa593"}, - {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7207272c9520203fea9b93843bb775d03e1cf88a80a936ce760f60bb5add92f3"}, - {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:78ab247f0b5671cc887c31d33f9b3abfb88d2614b84e4303f1a63b46c046c8bd"}, - {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:472d61143059c84947aa8bb74eabbace30d577a03a1805b77933d6bd13ddebbd"}, - {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45bf4c24c19fb8a50902ae37c5de50da81de4922af65baf760f7c0c42e1088be"}, - {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271718e26b3e17906b28b67314c45d19106112067205119dddbd834c2b7ce797"}, - {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:34175c9fd2a4bc3adc1380e1261f60306344e3407c20a4d684fd5f3be010fa3d"}, - {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e27f100e1ff17f6feeb1f33968bc185bf8ce41ca557deee9d9bbbffeb72030b7"}, - {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13dfdf492af0aa0a0edf66807d2b465607d11c4fa48f4a1fd41cbea5b18e8e8b"}, - {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e3d4e85ac060e2342ff85e90d0c04157acb210b9ce508e784a944f852a40e67"}, - {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca4956c9ab567d87d59d49fa3704cf29e37109ad348f2d5223c9bf761a332e7"}, - {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f467a5fd23b4fc43ed86342641f3936a68ded707f4627622fa3f82a120e18256"}, - {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:492e2c32c2af3f971473bc22f086513cedfc66a130756145a931a90c3958cb17"}, - {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2df95fca285a9f5bfe730e51945ffe2fa71ccbfdde3b0da5772b4ee4f2e770d5"}, - {file = "uvloop-0.19.0.tar.gz", hash = "sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd"}, -] - -[package.extras] -docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] -test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] - [[package]] name = "watchdog" version = "3.0.0" @@ -5004,190 +4025,6 @@ files = [ [package.extras] watchmedo = ["PyYAML (>=3.10)"] -[[package]] -name = "watchfiles" -version = "0.21.0" -description = "Simple, modern and high performance file watching and code reload in python." -optional = false -python-versions = ">=3.8" -files = [ - {file = "watchfiles-0.21.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:27b4035013f1ea49c6c0b42d983133b136637a527e48c132d368eb19bf1ac6aa"}, - {file = "watchfiles-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c81818595eff6e92535ff32825f31c116f867f64ff8cdf6562cd1d6b2e1e8f3e"}, - {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c107ea3cf2bd07199d66f156e3ea756d1b84dfd43b542b2d870b77868c98c03"}, - {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d9ac347653ebd95839a7c607608703b20bc07e577e870d824fa4801bc1cb124"}, - {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5eb86c6acb498208e7663ca22dbe68ca2cf42ab5bf1c776670a50919a56e64ab"}, - {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f564bf68404144ea6b87a78a3f910cc8de216c6b12a4cf0b27718bf4ec38d303"}, - {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d0f32ebfaa9c6011f8454994f86108c2eb9c79b8b7de00b36d558cadcedaa3d"}, - {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d45d9b699ecbac6c7bd8e0a2609767491540403610962968d258fd6405c17c"}, - {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:aff06b2cac3ef4616e26ba17a9c250c1fe9dd8a5d907d0193f84c499b1b6e6a9"}, - {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d9792dff410f266051025ecfaa927078b94cc7478954b06796a9756ccc7e14a9"}, - {file = "watchfiles-0.21.0-cp310-none-win32.whl", hash = "sha256:214cee7f9e09150d4fb42e24919a1e74d8c9b8a9306ed1474ecaddcd5479c293"}, - {file = "watchfiles-0.21.0-cp310-none-win_amd64.whl", hash = "sha256:1ad7247d79f9f55bb25ab1778fd47f32d70cf36053941f07de0b7c4e96b5d235"}, - {file = "watchfiles-0.21.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:668c265d90de8ae914f860d3eeb164534ba2e836811f91fecc7050416ee70aa7"}, - {file = "watchfiles-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a23092a992e61c3a6a70f350a56db7197242f3490da9c87b500f389b2d01eef"}, - {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e7941bbcfdded9c26b0bf720cb7e6fd803d95a55d2c14b4bd1f6a2772230c586"}, - {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11cd0c3100e2233e9c53106265da31d574355c288e15259c0d40a4405cbae317"}, - {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d78f30cbe8b2ce770160d3c08cff01b2ae9306fe66ce899b73f0409dc1846c1b"}, - {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6674b00b9756b0af620aa2a3346b01f8e2a3dc729d25617e1b89cf6af4a54eb1"}, - {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd7ac678b92b29ba630d8c842d8ad6c555abda1b9ef044d6cc092dacbfc9719d"}, - {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c873345680c1b87f1e09e0eaf8cf6c891b9851d8b4d3645e7efe2ec20a20cc7"}, - {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49f56e6ecc2503e7dbe233fa328b2be1a7797d31548e7a193237dcdf1ad0eee0"}, - {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:02d91cbac553a3ad141db016e3350b03184deaafeba09b9d6439826ee594b365"}, - {file = "watchfiles-0.21.0-cp311-none-win32.whl", hash = "sha256:ebe684d7d26239e23d102a2bad2a358dedf18e462e8808778703427d1f584400"}, - {file = "watchfiles-0.21.0-cp311-none-win_amd64.whl", hash = "sha256:4566006aa44cb0d21b8ab53baf4b9c667a0ed23efe4aaad8c227bfba0bf15cbe"}, - {file = "watchfiles-0.21.0-cp311-none-win_arm64.whl", hash = "sha256:c550a56bf209a3d987d5a975cdf2063b3389a5d16caf29db4bdddeae49f22078"}, - {file = "watchfiles-0.21.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:51ddac60b96a42c15d24fbdc7a4bfcd02b5a29c047b7f8bf63d3f6f5a860949a"}, - {file = "watchfiles-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:511f0b034120cd1989932bf1e9081aa9fb00f1f949fbd2d9cab6264916ae89b1"}, - {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cfb92d49dbb95ec7a07511bc9efb0faff8fe24ef3805662b8d6808ba8409a71a"}, - {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f92944efc564867bbf841c823c8b71bb0be75e06b8ce45c084b46411475a915"}, - {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:642d66b75eda909fd1112d35c53816d59789a4b38c141a96d62f50a3ef9b3360"}, - {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d23bcd6c8eaa6324fe109d8cac01b41fe9a54b8c498af9ce464c1aeeb99903d6"}, - {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18d5b4da8cf3e41895b34e8c37d13c9ed294954907929aacd95153508d5d89d7"}, - {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b8d1eae0f65441963d805f766c7e9cd092f91e0c600c820c764a4ff71a0764c"}, - {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1fd9a5205139f3c6bb60d11f6072e0552f0a20b712c85f43d42342d162be1235"}, - {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a1e3014a625bcf107fbf38eece0e47fa0190e52e45dc6eee5a8265ddc6dc5ea7"}, - {file = "watchfiles-0.21.0-cp312-none-win32.whl", hash = "sha256:9d09869f2c5a6f2d9df50ce3064b3391d3ecb6dced708ad64467b9e4f2c9bef3"}, - {file = "watchfiles-0.21.0-cp312-none-win_amd64.whl", hash = "sha256:18722b50783b5e30a18a8a5db3006bab146d2b705c92eb9a94f78c72beb94094"}, - {file = "watchfiles-0.21.0-cp312-none-win_arm64.whl", hash = "sha256:a3b9bec9579a15fb3ca2d9878deae789df72f2b0fdaf90ad49ee389cad5edab6"}, - {file = "watchfiles-0.21.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:4ea10a29aa5de67de02256a28d1bf53d21322295cb00bd2d57fcd19b850ebd99"}, - {file = "watchfiles-0.21.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:40bca549fdc929b470dd1dbfcb47b3295cb46a6d2c90e50588b0a1b3bd98f429"}, - {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9b37a7ba223b2f26122c148bb8d09a9ff312afca998c48c725ff5a0a632145f7"}, - {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec8c8900dc5c83650a63dd48c4d1d245343f904c4b64b48798c67a3767d7e165"}, - {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ad3fe0a3567c2f0f629d800409cd528cb6251da12e81a1f765e5c5345fd0137"}, - {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d353c4cfda586db2a176ce42c88f2fc31ec25e50212650c89fdd0f560ee507b"}, - {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:83a696da8922314ff2aec02987eefb03784f473281d740bf9170181829133765"}, - {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a03651352fc20975ee2a707cd2d74a386cd303cc688f407296064ad1e6d1562"}, - {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3ad692bc7792be8c32918c699638b660c0de078a6cbe464c46e1340dadb94c19"}, - {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06247538e8253975bdb328e7683f8515ff5ff041f43be6c40bff62d989b7d0b0"}, - {file = "watchfiles-0.21.0-cp38-none-win32.whl", hash = "sha256:9a0aa47f94ea9a0b39dd30850b0adf2e1cd32a8b4f9c7aa443d852aacf9ca214"}, - {file = "watchfiles-0.21.0-cp38-none-win_amd64.whl", hash = "sha256:8d5f400326840934e3507701f9f7269247f7c026d1b6cfd49477d2be0933cfca"}, - {file = "watchfiles-0.21.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7f762a1a85a12cc3484f77eee7be87b10f8c50b0b787bb02f4e357403cad0c0e"}, - {file = "watchfiles-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6e9be3ef84e2bb9710f3f777accce25556f4a71e15d2b73223788d528fcc2052"}, - {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4c48a10d17571d1275701e14a601e36959ffada3add8cdbc9e5061a6e3579a5d"}, - {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c889025f59884423428c261f212e04d438de865beda0b1e1babab85ef4c0f01"}, - {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66fac0c238ab9a2e72d026b5fb91cb902c146202bbd29a9a1a44e8db7b710b6f"}, - {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4a21f71885aa2744719459951819e7bf5a906a6448a6b2bbce8e9cc9f2c8128"}, - {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c9198c989f47898b2c22201756f73249de3748e0fc9de44adaf54a8b259cc0c"}, - {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f57c4461cd24fda22493109c45b3980863c58a25b8bec885ca8bea6b8d4b28"}, - {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:853853cbf7bf9408b404754b92512ebe3e3a83587503d766d23e6bf83d092ee6"}, - {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d5b1dc0e708fad9f92c296ab2f948af403bf201db8fb2eb4c8179db143732e49"}, - {file = "watchfiles-0.21.0-cp39-none-win32.whl", hash = "sha256:59137c0c6826bd56c710d1d2bda81553b5e6b7c84d5a676747d80caf0409ad94"}, - {file = "watchfiles-0.21.0-cp39-none-win_amd64.whl", hash = "sha256:6cb8fdc044909e2078c248986f2fc76f911f72b51ea4a4fbbf472e01d14faa58"}, - {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:ab03a90b305d2588e8352168e8c5a1520b721d2d367f31e9332c4235b30b8994"}, - {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:927c589500f9f41e370b0125c12ac9e7d3a2fd166b89e9ee2828b3dda20bfe6f"}, - {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd467213195e76f838caf2c28cd65e58302d0254e636e7c0fca81efa4a2e62c"}, - {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02b73130687bc3f6bb79d8a170959042eb56eb3a42df3671c79b428cd73f17cc"}, - {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:08dca260e85ffae975448e344834d765983237ad6dc308231aa16e7933db763e"}, - {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3ccceb50c611c433145502735e0370877cced72a6c70fd2410238bcbc7fe51d8"}, - {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57d430f5fb63fea141ab71ca9c064e80de3a20b427ca2febcbfcef70ff0ce895"}, - {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dd5fad9b9c0dd89904bbdea978ce89a2b692a7ee8a0ce19b940e538c88a809c"}, - {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:be6dd5d52b73018b21adc1c5d28ac0c68184a64769052dfeb0c5d9998e7f56a2"}, - {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b3cab0e06143768499384a8a5efb9c4dc53e19382952859e4802f294214f36ec"}, - {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6ed10c2497e5fedadf61e465b3ca12a19f96004c15dcffe4bd442ebadc2d85"}, - {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43babacef21c519bc6631c5fce2a61eccdfc011b4bcb9047255e9620732c8097"}, - {file = "watchfiles-0.21.0.tar.gz", hash = "sha256:c76c635fabf542bb78524905718c39f736a98e5ab25b23ec6d4abede1a85a6a3"}, -] - -[package.dependencies] -anyio = ">=3.0.0" - -[[package]] -name = "websocket-client" -version = "1.7.0" -description = "WebSocket client for Python with low level API options" -optional = false -python-versions = ">=3.8" -files = [ - {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, - {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, -] - -[package.extras] -docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - -[[package]] -name = "websockets" -version = "12.0" -description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -optional = false -python-versions = ">=3.8" -files = [ - {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, - {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, - {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, - {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, - {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, - {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, - {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, - {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, - {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, - {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, - {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, - {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, - {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, - {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, - {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, - {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, - {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, - {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, - {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, - {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, - {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, - {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, - {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, - {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, - {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, - {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, - {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, - {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, - {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, - {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, - {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, - {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, - {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, - {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, - {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, - {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, - {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, - {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, - {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, - {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, - {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, - {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, - {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, - {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, - {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, - {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, - {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, - {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, - {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, - {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, - {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, - {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, - {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, - {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, - {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, - {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, - {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, - {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, - {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, - {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, - {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, - {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, - {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, -] - [[package]] name = "wikipedia" version = "1.4.0" @@ -5395,22 +4232,7 @@ files = [ idna = ">=2.0" multidict = ">=4.0" -[[package]] -name = "zipp" -version = "3.18.1" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.8" -files = [ - {file = "zipp-3.18.1-py3-none-any.whl", hash = "sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b"}, - {file = "zipp-3.18.1.tar.gz", hash = "sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] - [metadata] lock-version = "2.0" python-versions = "^3.9.0,<3.12" -content-hash = "432eb1901d8a047e032be6d23a520d4d38a097a3282892232a4e309ceee76dd7" +content-hash = "ef25af4c14d08efd97d33d63ca0c51965fa0eaddcc9d107e49a05354dbe86057" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 631dcda3..61f6c7c8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -44,7 +44,6 @@ asyncpg = "^0.29.0" langchain-core = "^0.1.44" pyjwt = {extras = ["crypto"], version = "^2.8.0"} langchain-anthropic = "^0.1.8" -langchain-chroma = "^0.1.0" aiosqlite = "^0.20.0" [tool.poetry.group.dev.dependencies] diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index e94ac4ac..157ebcc9 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -36,4 +36,5 @@ services: env_file: - .env environment: + STORAGE_TYPE: "postgres" POSTGRES_HOST: "postgres" diff --git a/docker-compose.yml b/docker-compose.yml index 9c8a4501..36aada47 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,30 @@ version: "3" services: + postgres: + image: pgvector/pgvector:pg16 + healthcheck: + test: pg_isready -U $POSTGRES_USER + start_interval: 1s + start_period: 5s + interval: 5s + retries: 5 + ports: + - "5433:5432" + env_file: + - .env + volumes: + - ./postgres-volume:/var/lib/postgresql/data + postgres-setup: + image: migrate/migrate + depends_on: + postgres: + condition: service_healthy + volumes: + - ./backend/migrations:/migrations + env_file: + - .env + command: ["-path", "/migrations/postgres", "-database", "postgres://$POSTGRES_USER:$POSTGRES_PASSWORD@postgres:$POSTGRES_PORT/$POSTGRES_DB?sslmode=disable", "up"] sqlite-setup: build: context: tools/sqlite_migrate @@ -16,12 +40,15 @@ services: depends_on: sqlite-setup: condition: service_completed_successfully + postgres-setup: + condition: service_completed_successfully env_file: - .env volumes: - ./backend:/backend environment: STORAGE_TYPE: "sqlite" + POSTGRES_HOST: "postgres" command: - --reload frontend: From c5cc23e7dd4ce7d2d2611301d4aa83625089fdeb Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Tue, 30 Apr 2024 12:11:34 +0400 Subject: [PATCH 21/28] Cleanup. --- .gitignore | 1 - README.md | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 373a8669..a4d6dc2d 100644 --- a/.gitignore +++ b/.gitignore @@ -62,5 +62,4 @@ pnpm-debug.log* # Local db files -chroma_db/ opengpts.db \ No newline at end of file diff --git a/README.md b/README.md index 41f4fe1a..4ea9b1fc 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ the frontend and backend and sets up either SQLite or Postgres using docker-comp 3. **Set Up Environment Variables:** Create a `.env` file in the root directory of the project by copying `.env.example` as a template, and add the - following environment variables (if you want to use SQLite, you can skip the Postgres-related variables): + following environment variables: ```shell # At least one language model API key is required OPENAI_API_KEY=sk-... @@ -193,7 +193,7 @@ poetry install **Alternate vector databases** -The instructions above use Postgres or Chroma DB (for SQLite based setup) as a vector database, +The instructions above use Postgres as a vector database, although you can easily switch this out to use any of the 50+ vector databases in LangChain. **Set up language models** From 3fd7d6a4a11daaf958c73f470927f7d85c2fe549 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Tue, 30 Apr 2024 12:20:28 +0400 Subject: [PATCH 22/28] Update public assistants storage method. --- backend/app/storage/postgres.py | 15 +++------------ backend/app/storage/sqlite.py | 13 ++----------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/backend/app/storage/postgres.py b/backend/app/storage/postgres.py index d762d80c..7120f6a1 100644 --- a/backend/app/storage/postgres.py +++ b/backend/app/storage/postgres.py @@ -24,24 +24,15 @@ async def get_assistant( """Get an assistant by ID.""" async with get_pg_pool().acquire() as conn: return await conn.fetchrow( - "SELECT * FROM assistant WHERE assistant_id = $1 AND (user_id = $2 OR public = true)", + "SELECT * FROM assistant WHERE assistant_id = $1 AND (user_id = $2 OR public IS true)", assistant_id, user_id, ) - async def list_public_assistants( - self, assistant_ids: Sequence[str] - ) -> List[Assistant]: + async def list_public_assistants(self) -> List[Assistant]: """List all the public assistants.""" async with get_pg_pool().acquire() as conn: - return await conn.fetch( - ( - "SELECT * FROM assistant " - "WHERE assistant_id = ANY($1::uuid[]) " - "AND public = true;" - ), - assistant_ids, - ) + return await conn.fetch(("SELECT * FROM assistant WHERE public IS true;")) async def put_assistant( self, diff --git a/backend/app/storage/sqlite.py b/backend/app/storage/sqlite.py index 85fac4dd..d9553828 100644 --- a/backend/app/storage/sqlite.py +++ b/backend/app/storage/sqlite.py @@ -53,19 +53,10 @@ async def get_assistant( ) return Assistant(**assistant_data) - async def list_public_assistants( - self, assistant_ids: Sequence[str] - ) -> list[Assistant]: + async def list_public_assistants(self) -> list[Assistant]: """List all the public assistants.""" - assistant_ids_tuple = tuple( - assistant_ids - ) # SQL requires a tuple for the IN operator. - placeholders = ", ".join("?" for _ in assistant_ids) async with sqlite_conn() as conn, conn.cursor() as cur: - await cur.execute( - f"SELECT * FROM assistant WHERE assistant_id IN ({placeholders}) AND public = 1", - assistant_ids_tuple, - ) + await cur.execute("SELECT * FROM assistant WHERE public = 1") rows = await cur.fetchall() return [Assistant(**dict(row)) for row in rows] From edf3d0b924ab5154423c3f6332eee9196ff20605 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Tue, 30 Apr 2024 12:26:07 +0400 Subject: [PATCH 23/28] Add a TODO comment for the future. --- backend/app/upload.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/app/upload.py b/backend/app/upload.py index bceda9dc..1d9ba958 100644 --- a/backend/app/upload.py +++ b/backend/app/upload.py @@ -69,6 +69,8 @@ def _get_embedding_function() -> Union[OpenAIEmbeddings, AzureOpenAIEmbeddings]: def _get_vstore() -> VectorStore: + # TODO Need to add a sqlite-based vectorstore for StorageType.SQLITE. + # Using PGVector is temporary. if settings.storage_type in (StorageType.POSTGRES, StorageType.SQLITE): PG_CONNECTION_STRING = PGVector.connection_string_from_db_params( driver="psycopg2", From d5c3327d00c70f5eb9e4bc3e4b9631212616f510 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Tue, 30 Apr 2024 14:14:19 +0400 Subject: [PATCH 24/28] Minor fixes and cleanup. --- backend/app/storage/postgres.py | 3 + backend/app/storage/sqlite.py | 124 ++++++++++++++++---------------- 2 files changed, 63 insertions(+), 64 deletions(-) diff --git a/backend/app/storage/postgres.py b/backend/app/storage/postgres.py index 7120f6a1..3247d425 100644 --- a/backend/app/storage/postgres.py +++ b/backend/app/storage/postgres.py @@ -108,6 +108,7 @@ async def get_thread_state( "configurable": { **assistant["config"]["configurable"], "thread_id": thread_id, + "assistant_id": assistant_id, } } ) @@ -131,6 +132,7 @@ async def update_thread_state( "configurable": { **assistant["config"]["configurable"], **config["configurable"], + "assistant_id": assistant_id, } }, values, @@ -153,6 +155,7 @@ async def get_thread_history( "configurable": { **assistant["config"]["configurable"], "thread_id": thread_id, + "assistant_id": assistant_id, } } ) diff --git a/backend/app/storage/sqlite.py b/backend/app/storage/sqlite.py index d9553828..62e485f7 100644 --- a/backend/app/storage/sqlite.py +++ b/backend/app/storage/sqlite.py @@ -3,6 +3,7 @@ from typing import Any, Optional, Sequence, Union from uuid import uuid4 +import aiosqlite from langchain_core.messages import AnyMessage from langchain_core.runnables import RunnableConfig @@ -12,26 +13,45 @@ from app.storage.base import BaseStorage +def _deserialize_assistant(row: aiosqlite.Row) -> Assistant: + """Deserialize an assistant from a SQLite row.""" + return { + "assistant_id": row["assistant_id"], + "user_id": row["user_id"], + "name": row["name"], + "config": json.loads(row["config"]), + "updated_at": datetime.fromisoformat(row["updated_at"]), + "public": bool(row["public"]), + } + + +def _deserialize_thread(row: aiosqlite.Row) -> Thread: + """Deserialize a thread from a SQLite row.""" + return { + "thread_id": row["thread_id"], + "user_id": row["user_id"], + "assistant_id": row["assistant_id"], + "name": row["name"], + "updated_at": datetime.fromisoformat(row["updated_at"]), + } + + +def _deserialize_user(row: aiosqlite.Row) -> User: + """Deserialize a user from a SQLite row.""" + return { + "user_id": row["user_id"], + "sub": row["sub"], + "created_at": datetime.fromisoformat(row["created_at"]), + } + + class SqliteStorage(BaseStorage): async def list_assistants(self, user_id: str) -> list[Assistant]: """List all assistants for the current user.""" async with sqlite_conn() as conn, conn.cursor() as cur: await cur.execute("SELECT * FROM assistant WHERE user_id = ?", (user_id,)) rows = await cur.fetchall() - - # Deserialize the 'config' field from a JSON string to a dict for each row - assistants = [] - for row in rows: - assistant_data = dict(row) # Convert sqlite3.Row to dict - assistant_data["config"] = ( - json.loads(assistant_data["config"]) - if "config" in assistant_data and assistant_data["config"] - else {} - ) - assistant = Assistant(**assistant_data) - assistants.append(assistant) - - return assistants + return [_deserialize_assistant(row) for row in rows] async def get_assistant( self, user_id: str, assistant_id: str @@ -43,22 +63,14 @@ async def get_assistant( (assistant_id, user_id), ) row = await cur.fetchone() - if not row: - return None - assistant_data = dict(row) # Convert sqlite3.Row to dict - assistant_data["config"] = ( - json.loads(assistant_data["config"]) - if "config" in assistant_data and assistant_data["config"] - else {} - ) - return Assistant(**assistant_data) + return _deserialize_assistant(row) if row else None async def list_public_assistants(self) -> list[Assistant]: """List all the public assistants.""" async with sqlite_conn() as conn, conn.cursor() as cur: await cur.execute("SELECT * FROM assistant WHERE public = 1") rows = await cur.fetchall() - return [Assistant(**dict(row)) for row in rows] + return [_deserialize_assistant(row) for row in rows] async def put_assistant( self, @@ -72,8 +84,6 @@ async def put_assistant( """Modify an assistant.""" updated_at = datetime.now(timezone.utc) async with sqlite_conn() as conn, conn.cursor() as cur: - # Convert the config dict to a JSON string for storage. - config_str = json.dumps(config) await cur.execute( """ INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) @@ -90,27 +100,27 @@ async def put_assistant( assistant_id, user_id, name, - config_str, + json.dumps(config), updated_at.isoformat(), public, ), ) await conn.commit() - return Assistant( - assistant_id=assistant_id, - user_id=user_id, - name=name, - config=config, - updated_at=updated_at, - public=public, - ) + return { + "assistant_id": assistant_id, + "user_id": user_id, + "name": name, + "config": config, + "updated_at": updated_at, + "public": public, + } async def list_threads(self, user_id: str) -> list[Thread]: """List all threads for the current user.""" async with sqlite_conn() as conn, conn.cursor() as cur: await cur.execute("SELECT * FROM thread WHERE user_id = ?", (user_id,)) rows = await cur.fetchall() - return [Thread(**dict(row)) for row in rows] + return [_deserialize_thread(row) for row in rows] async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" @@ -120,7 +130,7 @@ async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: (thread_id, user_id), ) row = await cur.fetchone() - return Thread(**dict(row)) if row else None + return _deserialize_thread(row) if row else None async def get_thread_state( self, *, user_id: str, thread_id: str, assistant_id: str @@ -132,13 +142,11 @@ async def get_thread_state( "configurable": { **assistant["config"]["configurable"], "thread_id": thread_id, + "assistant_id": assistant_id, } } ) - return { - "values": state.values, - "next": state.next, - } + return {"values": state.values, "next": state.next} async def update_thread_state( self, @@ -155,6 +163,7 @@ async def update_thread_state( "configurable": { **assistant["config"]["configurable"], **config["configurable"], + "assistant_id": assistant_id, } }, values, @@ -177,6 +186,7 @@ async def get_thread_history( "configurable": { **assistant["config"]["configurable"], "thread_id": thread_id, + "assistant_id": assistant_id, } } ) @@ -199,7 +209,7 @@ async def put_thread( name = EXCLUDED.name, updated_at = EXCLUDED.updated_at """, - (thread_id, user_id, assistant_id, name, updated_at), + (thread_id, user_id, assistant_id, name, updated_at.isoformat()), ) await conn.commit() return { @@ -213,38 +223,24 @@ async def put_thread( async def get_or_create_user(self, sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" async with sqlite_conn() as conn, conn.cursor() as cur: - # start a write transaction to avoid the unique contraint error due to - # concurrent inserts + # Start a write transaction to avoid the unique contraint error due to + # concurrent inserts. await cur.execute("BEGIN EXCLUSIVE") await cur.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) - user_row = await cur.fetchone() - - if user_row: - # Convert sqlite3.Row to a User object - user = User( - user_id=user_row["user_id"], - sub=user_row["sub"], - created_at=user_row["created_at"], - ) - return user, False + row = await cur.fetchone() + if row: + return _deserialize_user(row), False # SQLite doesn't support RETURNING *, so we need to manually fetch the created user. await cur.execute( 'INSERT INTO "user" (user_id, sub, created_at) VALUES (?, ?, ?)', - (str(uuid4()), sub, datetime.now()), + (str(uuid4()), sub, datetime.now(timezone.utc).isoformat()), ) await conn.commit() - # Fetch the newly created user await cur.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) - new_user_row = await cur.fetchone() - - new_user = User( - user_id=new_user_row["user_id"], - sub=new_user_row["sub"], - created_at=new_user_row["created_at"], - ) - return new_user, True + row = await cur.fetchone() + return _deserialize_user(row), True async def delete_thread(self, user_id: str, thread_id: str) -> None: """Delete a thread by ID.""" From b0ff0ca96df95078572da8746b5cca8f7926b97c Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Fri, 3 May 2024 15:52:06 +0400 Subject: [PATCH 25/28] Create storage setup and teardown methods. --- backend/app/checkpoint.py | 10 ++-- backend/app/lifespan.py | 68 ++----------------------- backend/app/storage/base.py | 8 +++ backend/app/storage/postgres.py | 64 +++++++++++++++++++---- backend/app/storage/sqlite.py | 59 ++++++++++++++++----- backend/tests/unit_tests/conftest.py | 5 +- tools/redis_to_postgres/migrate_data.py | 5 +- 7 files changed, 122 insertions(+), 97 deletions(-) diff --git a/backend/app/checkpoint.py b/backend/app/checkpoint.py index 18029520..5bf17f27 100644 --- a/backend/app/checkpoint.py +++ b/backend/app/checkpoint.py @@ -15,9 +15,9 @@ SerializerProtocol, ) -from app.lifespan import create_sqlite_conn, get_pg_pool from app.storage.settings import StorageType from app.storage.settings import settings as storage_settings +from app.storage.storage import storage def loads(value: bytes) -> Checkpoint: @@ -58,7 +58,7 @@ def put(self, config: RunnableConfig, checkpoint: Checkpoint) -> RunnableConfig: raise NotImplementedError async def alist(self, config: RunnableConfig) -> AsyncIterator[CheckpointTuple]: - async with get_pg_pool().acquire() as db, db.transaction(): + async with storage.get_pg_pool().acquire() as db, db.transaction(): thread_id = config["configurable"]["thread_id"] async for value in db.cursor( "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = $1 ORDER BY thread_ts DESC", @@ -85,7 +85,7 @@ async def alist(self, config: RunnableConfig) -> AsyncIterator[CheckpointTuple]: async def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: thread_id = config["configurable"]["thread_id"] thread_ts = config["configurable"].get("thread_ts") - async with get_pg_pool().acquire() as conn: + async with storage.get_pg_pool().acquire() as conn: if thread_ts: if value := await conn.fetchrow( "SELECT checkpoint, parent_ts FROM checkpoints WHERE thread_id = $1 AND thread_ts = $2", @@ -129,7 +129,7 @@ async def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: async def aput(self, config: RunnableConfig, checkpoint: Checkpoint) -> None: thread_id = config["configurable"]["thread_id"] - async with get_pg_pool().acquire() as conn: + async with storage.get_pg_pool().acquire() as conn: await conn.execute( """ INSERT INTO checkpoints (thread_id, thread_ts, parent_ts, checkpoint) @@ -179,7 +179,7 @@ def config_specs(self) -> list[ConfigurableFieldSpec]: async def setup(self) -> None: if self.is_setup: return - self.conn = await create_sqlite_conn(global_=True) + self.conn = await storage.create_sqlite_conn(global_=True) self.is_setup = True diff --git a/backend/app/lifespan.py b/backend/app/lifespan.py index 45f33afa..a6ef3f80 100644 --- a/backend/app/lifespan.py +++ b/backend/app/lifespan.py @@ -1,74 +1,12 @@ from contextlib import asynccontextmanager -from typing import AsyncGenerator -import aiosqlite -import asyncpg -import orjson from fastapi import FastAPI -from app.storage.settings import StorageType -from app.storage.settings import settings as storage_settings - -_pg_pool = None -_global_sqlite_connections = [] - - -async def create_sqlite_conn(global_: bool = False, **kwargs) -> aiosqlite.Connection: - conn = await aiosqlite.connect("opengpts.db", **kwargs) - conn.row_factory = aiosqlite.Row - if global_: - _global_sqlite_connections.append(conn) - return conn - - -@asynccontextmanager -async def sqlite_conn(**kwargs) -> AsyncGenerator[aiosqlite.Connection, None]: - conn = await create_sqlite_conn(**kwargs) - try: - yield conn - finally: - await conn.close() - - -async def _close_global_sqlite_connections() -> None: - for conn in _global_sqlite_connections: - await conn.close() - _global_sqlite_connections.clear() - - -def get_pg_pool() -> asyncpg.pool.Pool: - return _pg_pool - - -async def _init_connection(conn) -> None: - await conn.set_type_codec( - "json", - encoder=lambda v: orjson.dumps(v).decode(), - decoder=orjson.loads, - schema="pg_catalog", - ) - await conn.set_type_codec( - "uuid", encoder=lambda v: str(v), decoder=lambda v: v, schema="pg_catalog" - ) +from app.storage.storage import storage @asynccontextmanager async def lifespan(app: FastAPI): - if storage_settings.storage_type == StorageType.POSTGRES: - global _pg_pool - _pg_pool = await asyncpg.create_pool( - database=storage_settings.postgres.db, - user=storage_settings.postgres.user, - password=storage_settings.postgres.password, - host=storage_settings.postgres.host, - port=storage_settings.postgres.port, - init=_init_connection, - ) - + await storage.setup() yield - - if storage_settings.storage_type == StorageType.POSTGRES: - await _pg_pool.close() - _pg_pool = None - elif storage_settings.storage_type == StorageType.SQLITE: - await _close_global_sqlite_connections() + await storage.teardown() diff --git a/backend/app/storage/base.py b/backend/app/storage/base.py index 22b69659..9b97b815 100644 --- a/backend/app/storage/base.py +++ b/backend/app/storage/base.py @@ -7,6 +7,14 @@ class BaseStorage(ABC): + @abstractmethod + async def setup(self) -> None: + """Setup the storage.""" + + @abstractmethod + async def teardown(self) -> None: + """Teardown the storage.""" + @abstractmethod async def list_assistants(self, user_id: str) -> list[Assistant]: """List all assistants for the current user.""" diff --git a/backend/app/storage/postgres.py b/backend/app/storage/postgres.py index 3247d425..5ea29a52 100644 --- a/backend/app/storage/postgres.py +++ b/backend/app/storage/postgres.py @@ -1,19 +1,55 @@ from datetime import datetime, timezone from typing import Any, List, Optional, Sequence, Union +import asyncpg +import orjson from langchain_core.messages import AnyMessage from langchain_core.runnables import RunnableConfig -from app.agent import agent -from app.lifespan import get_pg_pool from app.schema import Assistant, Thread, User from app.storage.base import BaseStorage +from app.storage.settings import settings as storage_settings class PostgresStorage(BaseStorage): + _pg_pool: asyncpg.pool.Pool = None + _is_setup: bool = False + + async def setup(self) -> None: + if self._is_setup: + return + self._pg_pool = await asyncpg.create_pool( + database=storage_settings.postgres.db, + user=storage_settings.postgres.user, + password=storage_settings.postgres.password, + host=storage_settings.postgres.host, + port=storage_settings.postgres.port, + init=self._init_connection, + ) + self._is_setup = True + + async def teardown(self) -> None: + await self._pg_pool.close() + self._pg_pool = None + self._is_setup = False + + async def _init_connection(self, conn) -> None: + await conn.set_type_codec( + "json", + encoder=lambda v: orjson.dumps(v).decode(), + decoder=orjson.loads, + schema="pg_catalog", + ) + await conn.set_type_codec( + "uuid", encoder=lambda v: str(v), decoder=lambda v: v, schema="pg_catalog" + ) + + def get_pg_pool(self) -> asyncpg.pool.Pool: + return self._pg_pool + async def list_assistants(self, user_id: str) -> List[Assistant]: """List all assistants for the current user.""" - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: return await conn.fetch( "SELECT * FROM assistant WHERE user_id = $1", user_id ) @@ -22,7 +58,7 @@ async def get_assistant( self, user_id: str, assistant_id: str ) -> Optional[Assistant]: """Get an assistant by ID.""" - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: return await conn.fetchrow( "SELECT * FROM assistant WHERE assistant_id = $1 AND (user_id = $2 OR public IS true)", assistant_id, @@ -31,7 +67,7 @@ async def get_assistant( async def list_public_assistants(self) -> List[Assistant]: """List all the public assistants.""" - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: return await conn.fetch(("SELECT * FROM assistant WHERE public IS true;")) async def put_assistant( @@ -56,7 +92,7 @@ async def put_assistant( return the assistant model if no exception is raised. """ updated_at = datetime.now(timezone.utc) - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: async with conn.transaction(): await conn.execute( ( @@ -86,12 +122,12 @@ async def put_assistant( async def list_threads(self, user_id: str) -> List[Thread]: """List all threads for the current user.""" - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: return await conn.fetch("SELECT * FROM thread WHERE user_id = $1", user_id) async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: return await conn.fetchrow( "SELECT * FROM thread WHERE thread_id = $1 AND user_id = $2", thread_id, @@ -102,6 +138,8 @@ async def get_thread_state( self, *, user_id: str, thread_id: str, assistant_id: str ): """Get state for a thread.""" + from app.agent import agent + assistant = await self.get_assistant(user_id, assistant_id) state = await agent.aget_state( { @@ -126,6 +164,8 @@ async def update_thread_state( assistant_id: str, ): """Add state to a thread.""" + from app.agent import agent + assistant = await self.get_assistant(user_id, assistant_id) await agent.aupdate_state( { @@ -142,6 +182,8 @@ async def get_thread_history( self, *, user_id: str, thread_id: str, assistant_id: str ): """Get the history of a thread.""" + from app.agent import agent + assistant = await self.get_assistant(user_id, assistant_id) return [ { @@ -166,7 +208,7 @@ async def put_thread( ) -> Thread: """Modify a thread.""" updated_at = datetime.now(timezone.utc) - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: await conn.execute( ( "INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) VALUES ($1, $2, $3, $4, $5) " @@ -192,7 +234,7 @@ async def put_thread( async def get_or_create_user(self, sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: user = await conn.fetchrow( 'INSERT INTO "user" (sub) VALUES ($1) ON CONFLICT (sub) DO NOTHING RETURNING *', sub, @@ -204,7 +246,7 @@ async def get_or_create_user(self, sub: str) -> tuple[User, bool]: async def delete_thread(self, user_id: str, thread_id: str) -> None: """Delete a thread by ID.""" - async with get_pg_pool().acquire() as conn: + async with self.get_pg_pool().acquire() as conn: await conn.execute( "DELETE FROM thread WHERE thread_id = $1 AND user_id = $2", thread_id, diff --git a/backend/app/storage/sqlite.py b/backend/app/storage/sqlite.py index 62e485f7..f20aebdf 100644 --- a/backend/app/storage/sqlite.py +++ b/backend/app/storage/sqlite.py @@ -1,14 +1,13 @@ import json +from contextlib import asynccontextmanager from datetime import datetime, timezone -from typing import Any, Optional, Sequence, Union +from typing import Any, AsyncGenerator, Optional, Sequence, Union from uuid import uuid4 import aiosqlite from langchain_core.messages import AnyMessage from langchain_core.runnables import RunnableConfig -from app.agent import agent -from app.lifespan import sqlite_conn from app.schema import Assistant, Thread, User from app.storage.base import BaseStorage @@ -46,9 +45,39 @@ def _deserialize_user(row: aiosqlite.Row) -> User: class SqliteStorage(BaseStorage): + _global_sqlite_connections = [] + + async def setup(self) -> None: + pass + + async def teardown(self) -> None: + await self._close_global_sqlite_connections() + + async def create_sqlite_conn( + self, global_: bool = False, **kwargs + ) -> aiosqlite.Connection: + conn = await aiosqlite.connect("opengpts.db", **kwargs) + conn.row_factory = aiosqlite.Row + if global_: + self._global_sqlite_connections.append(conn) + return conn + + @asynccontextmanager + async def sqlite_conn(self, **kwargs) -> AsyncGenerator[aiosqlite.Connection, None]: + conn = await self.create_sqlite_conn(**kwargs) + try: + yield conn + finally: + await conn.close() + + async def _close_global_sqlite_connections(self) -> None: + for conn in self._global_sqlite_connections: + await conn.close() + self._global_sqlite_connections.clear() + async def list_assistants(self, user_id: str) -> list[Assistant]: """List all assistants for the current user.""" - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute("SELECT * FROM assistant WHERE user_id = ?", (user_id,)) rows = await cur.fetchall() return [_deserialize_assistant(row) for row in rows] @@ -57,7 +86,7 @@ async def get_assistant( self, user_id: str, assistant_id: str ) -> Optional[Assistant]: """Get an assistant by ID.""" - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute( "SELECT * FROM assistant WHERE assistant_id = ? AND (user_id = ? OR public = 1)", (assistant_id, user_id), @@ -67,7 +96,7 @@ async def get_assistant( async def list_public_assistants(self) -> list[Assistant]: """List all the public assistants.""" - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute("SELECT * FROM assistant WHERE public = 1") rows = await cur.fetchall() return [_deserialize_assistant(row) for row in rows] @@ -83,7 +112,7 @@ async def put_assistant( ) -> Assistant: """Modify an assistant.""" updated_at = datetime.now(timezone.utc) - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute( """ INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) @@ -117,14 +146,14 @@ async def put_assistant( async def list_threads(self, user_id: str) -> list[Thread]: """List all threads for the current user.""" - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute("SELECT * FROM thread WHERE user_id = ?", (user_id,)) rows = await cur.fetchall() return [_deserialize_thread(row) for row in rows] async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute( "SELECT * FROM thread WHERE thread_id = ? AND user_id = ?", (thread_id, user_id), @@ -136,6 +165,8 @@ async def get_thread_state( self, *, user_id: str, thread_id: str, assistant_id: str ): """Get state for a thread.""" + from app.agent import agent + assistant = await self.get_assistant(user_id, assistant_id) state = await agent.aget_state( { @@ -157,6 +188,8 @@ async def update_thread_state( assistant_id: str, ): """Add state to a thread.""" + from app.agent import agent + assistant = await self.get_assistant(user_id, assistant_id) await agent.aupdate_state( { @@ -173,6 +206,8 @@ async def get_thread_history( self, *, user_id: str, thread_id: str, assistant_id: str ): """Get the history of a thread.""" + from app.agent import agent + assistant = await self.get_assistant(user_id, assistant_id) return [ { @@ -197,7 +232,7 @@ async def put_thread( ) -> Thread: """Modify a thread.""" updated_at = datetime.now(timezone.utc) - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute( """ INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) @@ -222,7 +257,7 @@ async def put_thread( async def get_or_create_user(self, sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: # Start a write transaction to avoid the unique contraint error due to # concurrent inserts. await cur.execute("BEGIN EXCLUSIVE") @@ -244,7 +279,7 @@ async def get_or_create_user(self, sub: str) -> tuple[User, bool]: async def delete_thread(self, user_id: str, thread_id: str) -> None: """Delete a thread by ID.""" - async with sqlite_conn() as conn, conn.cursor() as cur: + async with self.sqlite_conn() as conn, conn.cursor() as cur: await cur.execute( "DELETE FROM thread WHERE thread_id = ? AND user_id = ?", (thread_id, user_id), diff --git a/backend/tests/unit_tests/conftest.py b/backend/tests/unit_tests/conftest.py index 8070c0bf..2b64e19a 100644 --- a/backend/tests/unit_tests/conftest.py +++ b/backend/tests/unit_tests/conftest.py @@ -7,9 +7,10 @@ from app.auth.settings import AuthType from app.auth.settings import settings as auth_settings -from app.lifespan import get_pg_pool, lifespan +from app.lifespan import lifespan from app.server import app from app.storage.settings import settings as storage_settings +from app.storage.storage import storage auth_settings.auth_type = AuthType.NOOP @@ -73,7 +74,7 @@ async def pool(): await _create_test_db() _migrate_test_db() async with lifespan(app): - yield get_pg_pool() + yield storage.get_pg_pool() await _drop_test_db() diff --git a/tools/redis_to_postgres/migrate_data.py b/tools/redis_to_postgres/migrate_data.py index 84cfd9a0..644b79ec 100644 --- a/tools/redis_to_postgres/migrate_data.py +++ b/tools/redis_to_postgres/migrate_data.py @@ -21,7 +21,8 @@ from redis.client import Redis as RedisType from app.checkpoint import PostgresCheckpoint -from app.lifespan import get_pg_pool, lifespan +from app.lifespan import lifespan +from app.storage.storage import storage from app.server import app logging.basicConfig( @@ -265,7 +266,7 @@ def _get_embedding(doc: dict) -> str: async def migrate_data(): logger.info("Starting to migrate data from Redis to Postgres.") - async with get_pg_pool().acquire() as conn, conn.transaction(): + async with storage.get_pg_pool().acquire() as conn, conn.transaction(): await migrate_assistants(conn) await migrate_threads(conn) await migrate_checkpoints() From 37b241356018115b88773c980dcc6a95ac1c067b Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Fri, 3 May 2024 16:11:54 +0400 Subject: [PATCH 26/28] poetry lock --no-update --- backend/poetry.lock | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/poetry.lock b/backend/poetry.lock index 156563e5..22082937 100644 --- a/backend/poetry.lock +++ b/backend/poetry.lock @@ -1953,6 +1953,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = false python-versions = ">=3.6" files = [ + {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:704f5572ff473a5f897745abebc6df40f22d4133c1e0a1f124e4f2bd3330ff7e"}, {file = "lxml-5.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9d3c0f8567ffe7502d969c2c1b809892dc793b5d0665f602aad19895f8d508da"}, {file = "lxml-5.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5fcfbebdb0c5d8d18b84118842f31965d59ee3e66996ac842e21f957eb76138c"}, {file = "lxml-5.1.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f37c6d7106a9d6f0708d4e164b707037b7380fcd0b04c5bd9cae1fb46a856fb"}, @@ -1962,6 +1963,7 @@ files = [ {file = "lxml-5.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82bddf0e72cb2af3cbba7cec1d2fd11fda0de6be8f4492223d4a268713ef2147"}, {file = "lxml-5.1.0-cp310-cp310-win32.whl", hash = "sha256:b66aa6357b265670bb574f050ffceefb98549c721cf28351b748be1ef9577d93"}, {file = "lxml-5.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:4946e7f59b7b6a9e27bef34422f645e9a368cb2be11bf1ef3cafc39a1f6ba68d"}, + {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:14deca1460b4b0f6b01f1ddc9557704e8b365f55c63070463f6c18619ebf964f"}, {file = "lxml-5.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed8c3d2cd329bf779b7ed38db176738f3f8be637bb395ce9629fc76f78afe3d4"}, {file = "lxml-5.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:436a943c2900bb98123b06437cdd30580a61340fbdb7b28aaf345a459c19046a"}, {file = "lxml-5.1.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:acb6b2f96f60f70e7f34efe0c3ea34ca63f19ca63ce90019c6cbca6b676e81fa"}, @@ -1971,6 +1973,7 @@ files = [ {file = "lxml-5.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f4c9bda132ad108b387c33fabfea47866af87f4ea6ffb79418004f0521e63204"}, {file = "lxml-5.1.0-cp311-cp311-win32.whl", hash = "sha256:bc64d1b1dab08f679fb89c368f4c05693f58a9faf744c4d390d7ed1d8223869b"}, {file = "lxml-5.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:a5ab722ae5a873d8dcee1f5f45ddd93c34210aed44ff2dc643b5025981908cda"}, + {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:9aa543980ab1fbf1720969af1d99095a548ea42e00361e727c58a40832439114"}, {file = "lxml-5.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6f11b77ec0979f7e4dc5ae081325a2946f1fe424148d3945f943ceaede98adb8"}, {file = "lxml-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a36c506e5f8aeb40680491d39ed94670487ce6614b9d27cabe45d94cd5d63e1e"}, {file = "lxml-5.1.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f643ffd2669ffd4b5a3e9b41c909b72b2a1d5e4915da90a77e119b8d48ce867a"}, @@ -1996,8 +1999,8 @@ files = [ {file = "lxml-5.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8f52fe6859b9db71ee609b0c0a70fea5f1e71c3462ecf144ca800d3f434f0764"}, {file = "lxml-5.1.0-cp37-cp37m-win32.whl", hash = "sha256:d42e3a3fc18acc88b838efded0e6ec3edf3e328a58c68fbd36a7263a874906c8"}, {file = "lxml-5.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:eac68f96539b32fce2c9b47eb7c25bb2582bdaf1bbb360d25f564ee9e04c542b"}, + {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ae15347a88cf8af0949a9872b57a320d2605ae069bcdf047677318bc0bba45b1"}, {file = "lxml-5.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c26aab6ea9c54d3bed716b8851c8bfc40cb249b8e9880e250d1eddde9f709bf5"}, - {file = "lxml-5.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cfbac9f6149174f76df7e08c2e28b19d74aed90cad60383ad8671d3af7d0502f"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:342e95bddec3a698ac24378d61996b3ee5ba9acfeb253986002ac53c9a5f6f84"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:725e171e0b99a66ec8605ac77fa12239dbe061482ac854d25720e2294652eeaa"}, {file = "lxml-5.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d184e0d5c918cff04cdde9dbdf9600e960161d773666958c9d7b565ccc60c45"}, @@ -2005,6 +2008,7 @@ files = [ {file = "lxml-5.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6d48fc57e7c1e3df57be5ae8614bab6d4e7b60f65c5457915c26892c41afc59e"}, {file = "lxml-5.1.0-cp38-cp38-win32.whl", hash = "sha256:7ec465e6549ed97e9f1e5ed51c657c9ede767bc1c11552f7f4d022c4df4a977a"}, {file = "lxml-5.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:b21b4031b53d25b0858d4e124f2f9131ffc1530431c6d1321805c90da78388d1"}, + {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:52427a7eadc98f9e62cb1368a5079ae826f94f05755d2d567d93ee1bc3ceb354"}, {file = "lxml-5.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6a2a2c724d97c1eb8cf966b16ca2915566a4904b9aad2ed9a09c748ffe14f969"}, {file = "lxml-5.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:843b9c835580d52828d8f69ea4302537337a21e6b4f1ec711a52241ba4a824f3"}, {file = "lxml-5.1.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9b99f564659cfa704a2dd82d0684207b1aadf7d02d33e54845f9fc78e06b7581"}, @@ -2922,6 +2926,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -4230,4 +4235,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.9.0,<3.12" -content-hash = "fc96cf95416874baa59fc1b85463f4e1e9e5ade9e1c76febb74cadf341da11bb" +content-hash = "16cd528b4f7ee3c971f435c204690a07cdab1c508c72cfd926600cf8ed376a64" From bf3bd8226fa838a2b57288d0aa2ce9fd6d9252b9 Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Fri, 3 May 2024 18:58:35 +0400 Subject: [PATCH 27/28] Clean up PostgresStorage. --- backend/app/checkpoint.py | 6 ++-- backend/app/storage/postgres.py | 32 ++++++++++--------- .../unit_tests/agent_executor/test_upload.py | 3 +- backend/tests/unit_tests/conftest.py | 2 +- tools/redis_to_postgres/migrate_data.py | 2 +- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/backend/app/checkpoint.py b/backend/app/checkpoint.py index 5bf17f27..307cf424 100644 --- a/backend/app/checkpoint.py +++ b/backend/app/checkpoint.py @@ -58,7 +58,7 @@ def put(self, config: RunnableConfig, checkpoint: Checkpoint) -> RunnableConfig: raise NotImplementedError async def alist(self, config: RunnableConfig) -> AsyncIterator[CheckpointTuple]: - async with storage.get_pg_pool().acquire() as db, db.transaction(): + async with storage.get_pool().acquire() as db, db.transaction(): thread_id = config["configurable"]["thread_id"] async for value in db.cursor( "SELECT checkpoint, thread_ts, parent_ts FROM checkpoints WHERE thread_id = $1 ORDER BY thread_ts DESC", @@ -85,7 +85,7 @@ async def alist(self, config: RunnableConfig) -> AsyncIterator[CheckpointTuple]: async def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: thread_id = config["configurable"]["thread_id"] thread_ts = config["configurable"].get("thread_ts") - async with storage.get_pg_pool().acquire() as conn: + async with storage.get_pool().acquire() as conn: if thread_ts: if value := await conn.fetchrow( "SELECT checkpoint, parent_ts FROM checkpoints WHERE thread_id = $1 AND thread_ts = $2", @@ -129,7 +129,7 @@ async def aget_tuple(self, config: RunnableConfig) -> Optional[CheckpointTuple]: async def aput(self, config: RunnableConfig, checkpoint: Checkpoint) -> None: thread_id = config["configurable"]["thread_id"] - async with storage.get_pg_pool().acquire() as conn: + async with storage.get_pool().acquire() as conn: await conn.execute( """ INSERT INTO checkpoints (thread_id, thread_ts, parent_ts, checkpoint) diff --git a/backend/app/storage/postgres.py b/backend/app/storage/postgres.py index 5ea29a52..75193fa3 100644 --- a/backend/app/storage/postgres.py +++ b/backend/app/storage/postgres.py @@ -12,13 +12,13 @@ class PostgresStorage(BaseStorage): - _pg_pool: asyncpg.pool.Pool = None + _pool: asyncpg.pool.Pool = None _is_setup: bool = False async def setup(self) -> None: if self._is_setup: return - self._pg_pool = await asyncpg.create_pool( + self._pool = await asyncpg.create_pool( database=storage_settings.postgres.db, user=storage_settings.postgres.user, password=storage_settings.postgres.password, @@ -29,8 +29,8 @@ async def setup(self) -> None: self._is_setup = True async def teardown(self) -> None: - await self._pg_pool.close() - self._pg_pool = None + await self._pool.close() + self._pool = None self._is_setup = False async def _init_connection(self, conn) -> None: @@ -44,12 +44,14 @@ async def _init_connection(self, conn) -> None: "uuid", encoder=lambda v: str(v), decoder=lambda v: v, schema="pg_catalog" ) - def get_pg_pool(self) -> asyncpg.pool.Pool: - return self._pg_pool + def get_pool(self) -> asyncpg.pool.Pool: + if not self._is_setup: + raise RuntimeError("Storage is not set up.") + return self._pool async def list_assistants(self, user_id: str) -> List[Assistant]: """List all assistants for the current user.""" - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: return await conn.fetch( "SELECT * FROM assistant WHERE user_id = $1", user_id ) @@ -58,7 +60,7 @@ async def get_assistant( self, user_id: str, assistant_id: str ) -> Optional[Assistant]: """Get an assistant by ID.""" - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: return await conn.fetchrow( "SELECT * FROM assistant WHERE assistant_id = $1 AND (user_id = $2 OR public IS true)", assistant_id, @@ -67,7 +69,7 @@ async def get_assistant( async def list_public_assistants(self) -> List[Assistant]: """List all the public assistants.""" - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: return await conn.fetch(("SELECT * FROM assistant WHERE public IS true;")) async def put_assistant( @@ -92,7 +94,7 @@ async def put_assistant( return the assistant model if no exception is raised. """ updated_at = datetime.now(timezone.utc) - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: async with conn.transaction(): await conn.execute( ( @@ -122,12 +124,12 @@ async def put_assistant( async def list_threads(self, user_id: str) -> List[Thread]: """List all threads for the current user.""" - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: return await conn.fetch("SELECT * FROM thread WHERE user_id = $1", user_id) async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: return await conn.fetchrow( "SELECT * FROM thread WHERE thread_id = $1 AND user_id = $2", thread_id, @@ -208,7 +210,7 @@ async def put_thread( ) -> Thread: """Modify a thread.""" updated_at = datetime.now(timezone.utc) - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: await conn.execute( ( "INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) VALUES ($1, $2, $3, $4, $5) " @@ -234,7 +236,7 @@ async def put_thread( async def get_or_create_user(self, sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: user = await conn.fetchrow( 'INSERT INTO "user" (sub) VALUES ($1) ON CONFLICT (sub) DO NOTHING RETURNING *', sub, @@ -246,7 +248,7 @@ async def get_or_create_user(self, sub: str) -> tuple[User, bool]: async def delete_thread(self, user_id: str, thread_id: str) -> None: """Delete a thread by ID.""" - async with self.get_pg_pool().acquire() as conn: + async with self.get_pool().acquire() as conn: await conn.execute( "DELETE FROM thread WHERE thread_id = $1 AND user_id = $2", thread_id, diff --git a/backend/tests/unit_tests/agent_executor/test_upload.py b/backend/tests/unit_tests/agent_executor/test_upload.py index e239ef02..19736027 100644 --- a/backend/tests/unit_tests/agent_executor/test_upload.py +++ b/backend/tests/unit_tests/agent_executor/test_upload.py @@ -1,7 +1,8 @@ from io import BytesIO -from langchain.text_splitter import RecursiveCharacterTextSplitter from fastapi import UploadFile +from langchain.text_splitter import RecursiveCharacterTextSplitter + from app.upload import IngestRunnable, _guess_mimetype, convert_ingestion_input_to_blob from tests.unit_tests.fixtures import get_sample_paths from tests.unit_tests.utils import InMemoryVectorStore diff --git a/backend/tests/unit_tests/conftest.py b/backend/tests/unit_tests/conftest.py index 2b64e19a..29635759 100644 --- a/backend/tests/unit_tests/conftest.py +++ b/backend/tests/unit_tests/conftest.py @@ -74,7 +74,7 @@ async def pool(): await _create_test_db() _migrate_test_db() async with lifespan(app): - yield storage.get_pg_pool() + yield storage.get_pool() await _drop_test_db() diff --git a/tools/redis_to_postgres/migrate_data.py b/tools/redis_to_postgres/migrate_data.py index 644b79ec..bd20e27e 100644 --- a/tools/redis_to_postgres/migrate_data.py +++ b/tools/redis_to_postgres/migrate_data.py @@ -266,7 +266,7 @@ def _get_embedding(doc: dict) -> str: async def migrate_data(): logger.info("Starting to migrate data from Redis to Postgres.") - async with storage.get_pg_pool().acquire() as conn, conn.transaction(): + async with storage.get_pool().acquire() as conn, conn.transaction(): await migrate_assistants(conn) await migrate_threads(conn) await migrate_checkpoints() From 5e2a9dd077a080bb30f57503e477bc3bb0f6fe2d Mon Sep 17 00:00:00 2001 From: Bakar Tavadze Date: Fri, 3 May 2024 19:08:55 +0400 Subject: [PATCH 28/28] WIP/POC sharing a single sqlite connection across the application. --- backend/app/checkpoint.py | 2 +- backend/app/storage/sqlite.py | 73 ++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/backend/app/checkpoint.py b/backend/app/checkpoint.py index 307cf424..5404a1cd 100644 --- a/backend/app/checkpoint.py +++ b/backend/app/checkpoint.py @@ -179,7 +179,7 @@ def config_specs(self) -> list[ConfigurableFieldSpec]: async def setup(self) -> None: if self.is_setup: return - self.conn = await storage.create_sqlite_conn(global_=True) + self.conn = storage.get_conn() self.is_setup = True diff --git a/backend/app/storage/sqlite.py b/backend/app/storage/sqlite.py index f20aebdf..48d2d44e 100644 --- a/backend/app/storage/sqlite.py +++ b/backend/app/storage/sqlite.py @@ -1,7 +1,6 @@ import json -from contextlib import asynccontextmanager from datetime import datetime, timezone -from typing import Any, AsyncGenerator, Optional, Sequence, Union +from typing import Any, Optional, Sequence, Union from uuid import uuid4 import aiosqlite @@ -45,39 +44,33 @@ def _deserialize_user(row: aiosqlite.Row) -> User: class SqliteStorage(BaseStorage): - _global_sqlite_connections = [] + _conn: aiosqlite.Connection = None + _is_setup: bool = False async def setup(self) -> None: - pass + if self._is_setup: + return + self._conn = await aiosqlite.connect("opengpts.db") + self._conn.row_factory = aiosqlite.Row + await self._conn.execute("pragma journal_mode=wal") + self._is_setup = True + + # TODO remove + await self._conn.set_trace_callback(print) async def teardown(self) -> None: - await self._close_global_sqlite_connections() - - async def create_sqlite_conn( - self, global_: bool = False, **kwargs - ) -> aiosqlite.Connection: - conn = await aiosqlite.connect("opengpts.db", **kwargs) - conn.row_factory = aiosqlite.Row - if global_: - self._global_sqlite_connections.append(conn) - return conn - - @asynccontextmanager - async def sqlite_conn(self, **kwargs) -> AsyncGenerator[aiosqlite.Connection, None]: - conn = await self.create_sqlite_conn(**kwargs) - try: - yield conn - finally: - await conn.close() - - async def _close_global_sqlite_connections(self) -> None: - for conn in self._global_sqlite_connections: - await conn.close() - self._global_sqlite_connections.clear() + await self._conn.close() + self._conn = None + self._is_setup = False + + def get_conn(self) -> aiosqlite.Connection: + if not self._is_setup: + raise RuntimeError("Storage is not set up.") + return self._conn async def list_assistants(self, user_id: str) -> list[Assistant]: """List all assistants for the current user.""" - async with self.sqlite_conn() as conn, conn.cursor() as cur: + async with self.get_conn().cursor() as cur: await cur.execute("SELECT * FROM assistant WHERE user_id = ?", (user_id,)) rows = await cur.fetchall() return [_deserialize_assistant(row) for row in rows] @@ -86,7 +79,7 @@ async def get_assistant( self, user_id: str, assistant_id: str ) -> Optional[Assistant]: """Get an assistant by ID.""" - async with self.sqlite_conn() as conn, conn.cursor() as cur: + async with self.get_conn().cursor() as cur: await cur.execute( "SELECT * FROM assistant WHERE assistant_id = ? AND (user_id = ? OR public = 1)", (assistant_id, user_id), @@ -96,7 +89,7 @@ async def get_assistant( async def list_public_assistants(self) -> list[Assistant]: """List all the public assistants.""" - async with self.sqlite_conn() as conn, conn.cursor() as cur: + async with self.get_conn().cursor() as cur: await cur.execute("SELECT * FROM assistant WHERE public = 1") rows = await cur.fetchall() return [_deserialize_assistant(row) for row in rows] @@ -112,7 +105,8 @@ async def put_assistant( ) -> Assistant: """Modify an assistant.""" updated_at = datetime.now(timezone.utc) - async with self.sqlite_conn() as conn, conn.cursor() as cur: + conn = self.get_conn() + async with conn.cursor() as cur: await cur.execute( """ INSERT INTO assistant (assistant_id, user_id, name, config, updated_at, public) @@ -146,14 +140,14 @@ async def put_assistant( async def list_threads(self, user_id: str) -> list[Thread]: """List all threads for the current user.""" - async with self.sqlite_conn() as conn, conn.cursor() as cur: + async with self.get_conn().cursor() as cur: await cur.execute("SELECT * FROM thread WHERE user_id = ?", (user_id,)) rows = await cur.fetchall() return [_deserialize_thread(row) for row in rows] async def get_thread(self, user_id: str, thread_id: str) -> Optional[Thread]: """Get a thread by ID.""" - async with self.sqlite_conn() as conn, conn.cursor() as cur: + async with self.get_conn().cursor() as cur: await cur.execute( "SELECT * FROM thread WHERE thread_id = ? AND user_id = ?", (thread_id, user_id), @@ -232,7 +226,8 @@ async def put_thread( ) -> Thread: """Modify a thread.""" updated_at = datetime.now(timezone.utc) - async with self.sqlite_conn() as conn, conn.cursor() as cur: + conn = self.get_conn() + async with conn.cursor() as cur: await cur.execute( """ INSERT INTO thread (thread_id, user_id, assistant_id, name, updated_at) @@ -257,13 +252,18 @@ async def put_thread( async def get_or_create_user(self, sub: str) -> tuple[User, bool]: """Returns a tuple of the user and a boolean indicating whether the user was created.""" - async with self.sqlite_conn() as conn, conn.cursor() as cur: + conn = self.get_conn() + async with conn.cursor() as cur: # Start a write transaction to avoid the unique contraint error due to # concurrent inserts. + # TODO worked when connection wasn't shared across app await cur.execute("BEGIN EXCLUSIVE") await cur.execute('SELECT * FROM "user" WHERE sub = ?', (sub,)) row = await cur.fetchone() if row: + # Since we are using a single connection in the whole application, + # we can't leave the transaction open, so we need to commit it here. + await conn.commit() return _deserialize_user(row), False # SQLite doesn't support RETURNING *, so we need to manually fetch the created user. @@ -279,7 +279,8 @@ async def get_or_create_user(self, sub: str) -> tuple[User, bool]: async def delete_thread(self, user_id: str, thread_id: str) -> None: """Delete a thread by ID.""" - async with self.sqlite_conn() as conn, conn.cursor() as cur: + conn = self.get_conn() + async with conn.cursor() as cur: await cur.execute( "DELETE FROM thread WHERE thread_id = ? AND user_id = ?", (thread_id, user_id),