Generative AI — Complete Guide

# Pre-training: predict next token on internet-scale text
# "The cat sat on the ___" -> model predicts "mat"

# Training data: CommonCrawl, books, GitHub, Wikipedia
# Scale: GPT-4 ~1 trillion tokens, Llama 3 ~15 trillion

# After pre-training: instruction tuning + RLHF
# Makes model helpful, harmless, and honest

import tiktoken
enc = tiktoken.encoding_for_model("gpt-4")

text = "Generative AI is transforming software."
tokens = enc.encode(text)
print(tokens)        # [86230, 15592, 15592, ...]
print(len(tokens))   # 8 tokens  (~6 words)

# Cost implication: GPT-4 charges per token
# 1M tokens ~ $30 input / $60 output (approx)

from openai import OpenAI
client = OpenAI()

response = client.embeddings.create(
    model="text-embedding-3-small",
    input="How do neural networks learn?"
)
embedding = response.data[0].embedding
# Returns a list of 1536 floats
# Cosine similarity between two embeddings = semantic similarity

# Each token is projected into Q (query), K (key), V (value)
Q = X @ W_q   # "What am I looking for?"
K = X @ W_k   # "What do I contain?"
V = X @ W_v   # "What information do I contribute?"

# Attention scores: how much each token should attend to each other
scores  = Q @ K.T / sqrt(d_k)   # scaled dot-product
weights = softmax(scores)        # probabilities

# Attended representation = weighted sum of Values
output = weights @ V

# Multi-head: run H parallel heads, concatenate
# Each head learns different relationship types

from openai import OpenAI
client = OpenAI()

response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user",   "content": "Explain RAG in 2 sentences."}
    ],
    temperature=0.7,   # 0 = deterministic, 2 = very random
    max_tokens=200
)
print(response.choices[0].message.content)

from openai import OpenAI

client   = OpenAI()
history  = [{"role": "system", "content": "You are a helpful AI assistant."}]

def chat(user_input: str) -> str:
    history.append({"role": "user", "content": user_input})
    
    response = client.chat.completions.create(
        model="gpt-4o-mini",
        messages=history
    )
    reply = response.choices[0].message.content
    history.append({"role": "assistant", "content": reply})
    return reply

# REPL
while True:
    user = input("You: ")
    if user.lower() in ("exit", "quit"): break
    print(f"AI: {chat(user)}")

stream = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[{"role": "user", "content": "Explain neural networks."}],
    stream=True
)
for chunk in stream:
    delta = chunk.choices[0].delta.content or ""
    print(delta, end="", flush=True)

# INDEXING (run once)
docs   = load_documents("./docs/")          # PDF, web, DB
chunks = chunk_text(docs, size=500)         # split into segments
embeds = embed(chunks)                      # vectors
store  = VectorDB.upsert(chunks, embeds)    # persist

# QUERYING (every request)
query       = "What is our refund policy?"
query_embed = embed(query)
results     = store.similarity_search(query_embed, top_k=5)
context     = "

".join(r.text for r in results)

prompt = f"""Answer based only on the context below.
Context: {context}
Question: {query}"""

answer = llm.complete(prompt)

from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model, TaskType
from trl import SFTTrainer

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3-8B",
    load_in_4bit=True   # QLoRA: 4-bit quantisation
)

lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,               # rank — smaller = fewer params to train
    lora_alpha=32,      # scaling factor
    target_modules=["q_proj", "v_proj"],
    lora_dropout=0.1
)
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# trainable params: 8,388,608 / 8,030,261,248 = 0.10%

from langchain.agents import create_react_agent, AgentExecutor
from langchain_openai import ChatOpenAI
from langchain.tools import DuckDuckGoSearchRun, WikipediaQueryRun

llm   = ChatOpenAI(model="gpt-4o", temperature=0)
tools = [DuckDuckGoSearchRun(), WikipediaQueryRun()]

agent   = create_react_agent(llm, tools, prompt)
executor = AgentExecutor(agent=agent, tools=tools, verbose=True)

result = executor.invoke({"input": "What AI breakthroughs happened in 2025?"})
print(result["output"])

from fastapi import FastAPI
from fastapi.responses import StreamingResponse
from openai import OpenAI

app = FastAPI()
client = OpenAI()

@app.post("/chat")
async def chat(message: str):
    async def generate():
        stream = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": message}],
            stream=True
        )
        for chunk in stream:
            delta = chunk.choices[0].delta.content or ""
            yield f"data: {delta}

"
    
    return StreamingResponse(generate(), media_type="text/event-stream")