aditya
7d646d3439
First simulation testing subdirectory output support. RAG app with separate modules: document_loader, chunker, embeddings, vector_store, retriever, generator — plus config.py and main.py CLI entry point. All Python files pass syntax check. 9/10 DoD checks pass. Also adds sim_test_commands.md with 9 new simulation definitions (SIM9-SIM17) for manual terminal testing.
41 lines
1.4 KiB
Python
41 lines
1.4 KiB
Python
import os
|
|
from typing import List, Tuple
|
|
from pathlib import Path
|
|
|
|
|
|
class DocumentLoader:
|
|
"""Loads text documents from a directory."""
|
|
|
|
def __init__(self, documents_dir: str):
|
|
self.documents_dir = Path(documents_dir)
|
|
|
|
def load_documents(self) -> List[Tuple[str, str]]:
|
|
"""
|
|
Load all .txt files from the documents directory.
|
|
|
|
Returns:
|
|
List of tuples containing (filename, content)
|
|
"""
|
|
documents = []
|
|
|
|
if not self.documents_dir.exists():
|
|
raise FileNotFoundError(f"Documents directory not found: {self.documents_dir}")
|
|
|
|
for file_path in self.documents_dir.glob("*.txt"):
|
|
try:
|
|
with open(file_path, 'r', encoding='utf-8') as file:
|
|
content = file.read()
|
|
documents.append((file_path.name, content))
|
|
except UnicodeDecodeError:
|
|
try:
|
|
with open(file_path, 'r', encoding='latin-1') as file:
|
|
content = file.read()
|
|
documents.append((file_path.name, content))
|
|
except Exception as e:
|
|
print(f"Error reading file {file_path}: {e}")
|
|
continue
|
|
except Exception as e:
|
|
print(f"Error reading file {file_path}: {e}")
|
|
continue
|
|
|
|
return documents |