forked from cleveragents/cleveragents-core
aditya
ff52ef4f69
First simulation testing subdirectory output support. RAG app with separate modules: document_loader, chunker, embeddings, vector_store, retriever, generator — plus config.py and main.py CLI entry point. All Python files pass syntax check. 9/10 DoD checks pass. Also adds sim_test_commands.md with 9 new simulation definitions (SIM9-SIM17) for manual terminal testing.
73 lines
2.0 KiB
Python
73 lines
2.0 KiB
Python
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
|
|
@dataclass
|
|
class RAGConfig:
|
|
"""Configuration for the RAG system."""
|
|
|
|
# OpenAI API settings
|
|
openai_api_key: str = ""
|
|
embedding_model: str = "text-embedding-3-small"
|
|
chat_model: str = "gpt-3.5-turbo"
|
|
temperature: float = 0.1
|
|
|
|
# Chunking parameters
|
|
chunk_size: int = 1000
|
|
chunk_overlap: int = 200
|
|
|
|
# Embedding settings
|
|
embedding_batch_size: int = 100
|
|
embedding_dimension: int = 1536
|
|
|
|
# Retrieval settings
|
|
top_k: int = 5
|
|
similarity_threshold: float = 0.7
|
|
|
|
# File paths
|
|
documents_dir: str = "documents"
|
|
index_dir: str = "index"
|
|
|
|
def get_index_path(self) -> str:
|
|
"""Get path to FAISS index file."""
|
|
return str(Path(self.index_dir) / "faiss.index")
|
|
|
|
def get_metadata_path(self) -> str:
|
|
"""Get path to metadata file."""
|
|
return str(Path(self.index_dir) / "metadata.pkl")
|
|
|
|
def ensure_directories(self):
|
|
"""Create necessary directories if they don't exist."""
|
|
Path(self.documents_dir).mkdir(exist_ok=True)
|
|
Path(self.index_dir).mkdir(exist_ok=True)
|
|
|
|
|
|
@dataclass
|
|
class ModelConfig:
|
|
"""Configuration for different model options."""
|
|
|
|
# Available embedding models
|
|
EMBEDDING_MODELS = {
|
|
"text-embedding-3-small": 1536,
|
|
"text-embedding-3-large": 3072,
|
|
"text-embedding-ada-002": 1536
|
|
}
|
|
|
|
# Available chat models
|
|
CHAT_MODELS = [
|
|
"gpt-3.5-turbo",
|
|
"gpt-4",
|
|
"gpt-4-turbo-preview"
|
|
]
|
|
|
|
@classmethod
|
|
def get_embedding_dimension(cls, model: str) -> int:
|
|
"""Get embedding dimension for a model."""
|
|
return cls.EMBEDDING_MODELS.get(model, 1536)
|
|
|
|
@classmethod
|
|
def validate_models(cls, embedding_model: str, chat_model: str) -> bool:
|
|
"""Validate model selections."""
|
|
embedding_valid = embedding_model in cls.EMBEDDING_MODELS
|
|
chat_valid = chat_model in cls.CHAT_MODELS
|
|
return embedding_valid and chat_valid |