Document RAG Pipeline

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.llms.openai import OpenAI

# Load documents from a folder
documents = SimpleDirectoryReader('data/').load_data()

# Build index (chunks, embeds, stores)
index = VectorStoreIndex.from_documents(documents)

# Create query engine
query_engine = index.as_query_engine(
    llm=OpenAI(model='gpt-4o'),
    similarity_top_k=3
)

# Ask a question
response = query_engine.query(
    'What are the main findings in these documents?'
)
print(response)

from llama_index.core import VectorStoreIndex, SimpleDirectoryReader
from llama_index.postprocessor.cohere_rerank import CohereRerank

# Load and index documents
documents = SimpleDirectoryReader('data/').load_data()
index = VectorStoreIndex.from_documents(documents)

# Add Cohere reranker
reranker = CohereRerank(
    api_key='YOUR_COHERE_KEY',
    top_n=3
)

# Query with reranking
query_engine = index.as_query_engine(
    similarity_top_k=10,  # Retrieve more, rerank to 3
    node_postprocessors=[reranker]
)

response = query_engine.query('What is the refund policy?')
print(response)

from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain_chroma import Chroma
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import DirectoryLoader
from langchain.chains import RetrievalQA

# Load documents
loader = DirectoryLoader('docs/', glob='**/*.pdf')
documents = loader.load()

# Semantic chunking
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=['\n\n', '\n', '. ', ' ', '']
)
chunks = text_splitter.split_documents(documents)

# Create persistent vector store
vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=OpenAIEmbeddings(model='text-embedding-3-large'),
    persist_directory='./chroma_db'
)

# Build RAG chain
qa_chain = RetrievalQA.from_chain_type(
    llm=ChatOpenAI(model='gpt-4o'),
    retriever=vectorstore.as_retriever(search_kwargs={'k': 5}),
    return_source_documents=True
)

# Query with sources
result = qa_chain.invoke({'query': 'Summarize the key points'})
print(result['result'])
print('\nSources:')
for doc in result['source_documents']:
    print(f'  - {doc.metadata.get("source", "unknown")}')