RAG Systems —
From 0 to Production

# install: pip install langchain-community pypdf
from langchain_community.document_loaders import (
    PyPDFLoader,
    DirectoryLoader,
    WebBaseLoader,
    CSVLoader,
    JSONLoader
)

# a single PDF
loader = PyPDFLoader("company_policy.pdf")
docs = loader.load()
print(f"Loaded {len(docs)} pages")

# a whole folder of PDFs
loader = DirectoryLoader(
    "./docs/",
    glob="**/*.pdf",
    loader_cls=PyPDFLoader
)
docs = loader.load()
print(f"Loaded {len(docs)} pages")

# loading from a URL
loader = WebBaseLoader("https://docs.example.com/api")
docs = loader.load()

# Google Drive (requires OAuth setup)
from langchain_google_community import GoogleDriveLoader
loader = GoogleDriveLoader(
    folder_id="your_folder_id",
    file_types=["document", "pdf"]
)
docs = loader.load()

from langchain.text_splitter import (
    RecursiveCharacterTextSplitter,
    MarkdownHeaderTextSplitter,
    TokenTextSplitter
)

# Recursive — most recommended for most cases
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,        # characters per chunk
    chunk_overlap=200,      # overlap between chunks — preserves context
    separators=["\n\n", "\n", ".", " "]  # splitting priority order
)
chunks = splitter.split_documents(docs)
print(f"Created {len(chunks)} chunks")

# by Headers (Markdown) — perfect for technical documentation
headers = [("#", "H1"), ("##", "H2"), ("###", "H3")]
md_splitter = MarkdownHeaderTextSplitter(
    headers_to_split_on=headers
)
md_docs = md_splitter.split_text(markdown_text)

# by Tokens — more accurate for calculating API costs
token_splitter = TokenTextSplitter(
    chunk_size=250,
    chunk_overlap=30
)
token_chunks = token_splitter.split_documents(docs)

from langchain_openai import OpenAIEmbeddings
from langchain_community.embeddings import HuggingFaceEmbeddings
import numpy as np

# OpenAI (recommended for quality and accuracy)
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
# 1536 dimensions, ~$0.02 per 1M tokens

# text-embedding-3-large — higher quality, 2× more expensive
embeddings_large = OpenAIEmbeddings(model="text-embedding-3-large")
# 3072 dimensions

# HuggingFace (free, runs locally, supports Hebrew!)
embeddings_hf = HuggingFaceEmbeddings(
    model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
)
# 768 dimensions, supports 50+ languages including Hebrew

# checking Cosine Similarity
v1 = embeddings.embed_query("AI Agent")
v2 = embeddings.embed_query("Autonomous AI System")
similarity = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
print(f"Cosine similarity: {similarity:.3f}")  # ~0.85

# a batch Embedding
texts = ["what are the business days?", "how to reset password", "price list"]
vectors = embeddings.embed_documents(texts)
print(f"Shape: {len(vectors)} x {len(vectors[0])}")

from langchain_pinecone import PineconeVectorStore
from langchain_community.vectorstores import Chroma
import os

# === Pinecone (Production) ===
os.environ["PINECONE_API_KEY"] = "your-api-key"

vectorstore = PineconeVectorStore.from_documents(
    documents=chunks,
    embedding=embeddings,
    index_name="my-rag-index",
    namespace="v1"
)

# connecting to an existing index
vectorstore = PineconeVectorStore(
    index_name="my-rag-index",
    embedding=embeddings,
    namespace="v1"
)

# === Chroma (Development / Local) ===
vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory="./chroma_db"
)

# loading from disk
vectorstore = Chroma(
    persist_directory="./chroma_db",
    embedding_function=embeddings
)

# === Retriever ===
retriever = vectorstore.as_retriever(
    search_type="similarity",  # similarity / mmr / similarity_score_threshold
    search_kwargs={"k": 4}     # number of chunks to return
)

# MMR — diversifying results (prevents duplicates)
retriever_mmr = vectorstore.as_retriever(
    search_type="mmr",
    search_kwargs={"k": 6, "fetch_k": 20, "lambda_mult": 0.5}
)

# manual search
results = vectorstore.similarity_search("what is the return policy?", k=3)
for doc in results:
    print(doc.page_content[:200])
    print(f"Source: {doc.metadata.get('source', 'N/A')}")

from langchain_openai import ChatOpenAI
from langchain.chains import RetrievalQA
from langchain_core.prompts import PromptTemplate

# Prompt Template — the Prompt that defines how the model behaves
TEMPLATE = """Use the following pieces of Context to answer the question at the end.
If you don't know the answer, say "I don't know" — don't make up answers.
Always answer clearly and in an organized way.

Context:
{context}

Question: {question}

Answer:"""

prompt = PromptTemplate(
    template=TEMPLATE,
    input_variables=["context", "question"]
)

# LLM
llm = ChatOpenAI(
    model="gpt-4o",
    temperature=0  # 0 = consistent and deterministic
)

# RAG Chain
qa_chain = RetrievalQA.from_chain_type(
    llm=llm,
    chain_type="stuff",             # stuff = puts everything into the prompt
    retriever=retriever,
    return_source_documents=True,   # returns sources
    chain_type_kwargs={"prompt": prompt}
)

# question!
result = qa_chain.invoke({"query": "what is our return policy?"})
print(result["result"])
print("\nSources:")
for doc in result["source_documents"]:
    print(f"  - {doc.metadata.get('source', 'unknown')}")