Build an LLM Playground — Part 4: Build "Deep Research" with Web Search and Reasoning Models

import anthropic
 
client = anthropic.Anthropic()
 
response = client.messages.create(
    model="claude-sonnet-4-6",
    max_tokens=16000,
    temperature=1,  # required for extended thinking
    thinking={
        "type": "enabled",
        "budget_tokens": 10000  # max tokens for thinking
    },
    messages=[{
        "role": "user",
        "content": "How many prime numbers are between 1 and 100?"
    }]
)
 
# The response has two parts: thinking and text
for block in response.content:
    if block.type == "thinking":
        print("=== THINKING ===")
        print(block.thinking)
    elif block.type == "text":
        print("=== ANSWER ===")
        print(block.text)

# Without CoT
response = llm("What is 17 * 24?")
# Model might jump to an answer and get it wrong
 
# With CoT
response = llm("""What is 17 * 24?
 
Let's think step by step.""")
# Model breaks it down:
# "17 * 24 = 17 * 20 + 17 * 4 = 340 + 68 = 408"

prompt = """
Q: Roger has 5 tennis balls. He buys 2 more cans of tennis balls.
Each can has 3 tennis balls. How many tennis balls does he have now?
A: Roger started with 5 balls. He bought 2 cans of 3 balls each.
That's 2 * 3 = 6 new balls. 5 + 6 = 11. The answer is 11.
 
Q: The cafeteria had 23 apples. If they used 20 to make lunch and
bought 6 more, how many apples do they have?
A: The cafeteria started with 23 apples. They used 20, so they had
23 - 20 = 3. Then they bought 6 more, so 3 + 6 = 9. The answer is 9.
 
Q: {user_question}
A: Let's think step by step."""

import anthropic
 
client = anthropic.Anthropic()
 
def solve_with_cot(question: str) -> dict:
    """Solve a problem using Chain-of-Thought prompting."""
    response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=4096,
        system="""You are a careful problem solver. For every question:
1. Break the problem into clear steps
2. Work through each step explicitly
3. Show all calculations
4. State your final answer clearly
 
Format your response as:
## Reasoning
[step-by-step work]
 
## Answer
[final answer]""",
        messages=[{
            "role": "user",
            "content": question
        }]
    )
 
    text = response.content[0].text
 
    # Parse reasoning and answer
    parts = text.split("## Answer")
    reasoning = parts[0].replace("## Reasoning", "").strip() if len(parts) > 1 else text
    answer = parts[1].strip() if len(parts) > 1 else text
 
    return {
        "reasoning": reasoning,
        "answer": answer,
        "tokens_used": response.usage.input_tokens + response.usage.output_tokens
    }
 
# Example
result = solve_with_cot(
    "A store sells apples for $1.50 each and oranges for $2.00 each. "
    "If Sarah buys 3 apples and some oranges, and spends exactly $13.50, "
    "how many oranges did she buy?"
)
print(result["reasoning"])
print(f"\nAnswer: {result['answer']}")

import anthropic
from collections import Counter
 
client = anthropic.Anthropic()
 
def self_consistency(question: str, n_samples: int = 5, temperature: float = 0.7) -> dict:
    """
    Generate multiple reasoning chains and take majority vote.
    Higher temperature = more diverse reasoning paths.
    """
    answers = []
    chains = []
 
    for i in range(n_samples):
        response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=2048,
            temperature=temperature,
            messages=[{
                "role": "user",
                "content": f"""{question}
 
Think step by step, then give your final answer on the last line
in the format: ANSWER: [your answer]"""
            }]
        )
 
        text = response.content[0].text
        chains.append(text)
 
        # Extract answer from last line
        for line in reversed(text.split("\n")):
            if "ANSWER:" in line:
                answer = line.split("ANSWER:")[-1].strip()
                answers.append(answer)
                break
 
    # Majority vote
    vote_counts = Counter(answers)
    best_answer = vote_counts.most_common(1)[0] if vote_counts else ("No consensus", 0)
 
    return {
        "answer": best_answer[0],
        "confidence": best_answer[1] / len(answers) if answers else 0,
        "vote_distribution": dict(vote_counts),
        "n_chains": n_samples,
        "chains": chains
    }
 
# Example
result = self_consistency(
    "A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. "
    "How much does the ball cost?",
    n_samples=7
)
print(f"Answer: {result['answer']}")
print(f"Confidence: {result['confidence']:.0%}")
print(f"Vote distribution: {result['vote_distribution']}")

import anthropic
 
client = anthropic.Anthropic()
 
def sequential_revision(question: str, max_rounds: int = 3) -> dict:
    """
    Iteratively improve an answer through self-critique and revision.
    """
    # Round 1: Initial answer
    response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=2048,
        messages=[{
            "role": "user",
            "content": f"Answer this question thoroughly:\n\n{question}"
        }]
    )
    current_answer = response.content[0].text
    history = [{"round": 0, "type": "initial", "content": current_answer}]
 
    for round_num in range(1, max_rounds + 1):
        # Critique
        critique_response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=1024,
            messages=[{
                "role": "user",
                "content": f"""Here is a question and an answer. Critically evaluate the answer.
Identify specific errors, gaps, unsupported claims, or areas for improvement.
Be harsh but constructive. If the answer is already excellent, say "NO ISSUES FOUND".
 
Question: {question}
 
Answer: {current_answer}
 
Critique:"""
            }]
        )
        critique = critique_response.content[0].text
        history.append({"round": round_num, "type": "critique", "content": critique})
 
        # Check if no issues found
        if "NO ISSUES FOUND" in critique.upper():
            break
 
        # Revise
        revision_response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=2048,
            messages=[{
                "role": "user",
                "content": f"""Revise this answer based on the critique below.
Fix all identified issues while preserving what was already good.
 
Question: {question}
 
Original answer: {current_answer}
 
Critique: {critique}
 
Revised answer:"""
            }]
        )
        current_answer = revision_response.content[0].text
        history.append({"round": round_num, "type": "revision", "content": current_answer})
 
    return {
        "final_answer": current_answer,
        "rounds": len([h for h in history if h["type"] == "revision"]),
        "history": history
    }
 
# Example
result = sequential_revision(
    "Explain the CAP theorem in distributed systems and give a real-world example "
    "of a system that prioritizes each of the three pairs (CP, AP, CA)."
)
print(f"Final answer (after {result['rounds']} revisions):")
print(result["final_answer"])

import anthropic
import json
 
client = anthropic.Anthropic()
 
def tree_of_thoughts(
    problem: str,
    n_branches: int = 3,
    max_depth: int = 3,
    beam_width: int = 2
) -> dict:
    """
    Explore multiple reasoning paths using Tree of Thoughts.
 
    Args:
        problem: The problem to solve
        n_branches: Number of branches to generate at each step
        max_depth: Maximum depth of the reasoning tree
        beam_width: Number of top branches to keep at each level (beam search)
    """
 
    def generate_thoughts(problem: str, context: str, n: int) -> list:
        """Generate n possible next reasoning steps."""
        response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=2048,
            messages=[{
                "role": "user",
                "content": f"""Problem: {problem}
 
Reasoning so far: {context if context else "None — this is the first step."}
 
Generate exactly {n} different possible next steps in the reasoning.
Each should take a DIFFERENT approach or consider a DIFFERENT angle.
 
Return as a JSON array of strings, each being one reasoning step.
Example: ["Step: First approach...", "Step: Alternative approach...", "Step: Third angle..."]"""
            }]
        )
        try:
            return json.loads(response.content[0].text)
        except json.JSONDecodeError:
            return [response.content[0].text]
 
    def evaluate_thought(problem: str, reasoning_path: str) -> float:
        """Evaluate how promising a reasoning path is (0-1)."""
        response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=256,
            messages=[{
                "role": "user",
                "content": f"""Problem: {problem}
 
Reasoning path so far:
{reasoning_path}
 
Rate how promising this reasoning path is for solving the problem.
Consider: Is the logic sound? Is it making progress? Is it heading toward a correct answer?
 
Respond with ONLY a number between 0.0 and 1.0."""
            }]
        )
        try:
            return float(response.content[0].text.strip())
        except ValueError:
            return 0.5
 
    # Initialize with root branches
    current_paths = [{"path": "", "score": 1.0}]
 
    for depth in range(max_depth):
        all_candidates = []
 
        for node in current_paths:
            # Generate possible next steps
            thoughts = generate_thoughts(problem, node["path"], n_branches)
 
            for thought in thoughts:
                new_path = f"{node['path']}\n{thought}" if node["path"] else thought
                score = evaluate_thought(problem, new_path)
                all_candidates.append({"path": new_path, "score": score})
 
        # Keep top beam_width candidates (beam search)
        all_candidates.sort(key=lambda x: x["score"], reverse=True)
        current_paths = all_candidates[:beam_width]
 
        print(f"Depth {depth + 1}: {len(all_candidates)} candidates → kept top {beam_width}")
        for p in current_paths:
            print(f"  Score: {p['score']:.2f}")
 
    # Generate final answer from the best path
    best_path = current_paths[0]
    response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=2048,
        messages=[{
            "role": "user",
            "content": f"""Problem: {problem}
 
Best reasoning path:
{best_path['path']}
 
Based on this reasoning, provide the final, complete answer to the problem."""
        }]
    )
 
    return {
        "answer": response.content[0].text,
        "best_path": best_path["path"],
        "path_score": best_path["score"],
        "explored_paths": sum(len(generate_thoughts(problem, "", n_branches)) for _ in range(max_depth))
    }
 
# Example
result = tree_of_thoughts(
    "Design a system to detect fraudulent transactions in real-time. "
    "Consider latency, accuracy, and false positive rate.",
    n_branches=3,
    max_depth=3,
    beam_width=2
)
print(f"Answer (path score: {result['path_score']:.2f}):")
print(result["answer"])

import anthropic
import subprocess
import tempfile
 
client = anthropic.Anthropic()
 
def search_against_verifier(
    problem: str,
    test_cases: list[dict],
    n_candidates: int = 10,
    temperature: float = 0.8
) -> dict:
    """
    Generate multiple code solutions and verify each against test cases.
    Return the first solution that passes all tests.
    """
    results = []
 
    for i in range(n_candidates):
        # Generate a candidate solution
        response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=2048,
            temperature=temperature,
            messages=[{
                "role": "user",
                "content": f"""Solve this problem in Python. Return ONLY the function, no explanation.
 
{problem}"""
            }]
        )
 
        code = response.content[0].text
        # Strip markdown code fences if present
        if "```python" in code:
            code = code.split("```python")[1].split("```")[0]
 
        # Run against test cases
        passed = 0
        total = len(test_cases)
 
        for test in test_cases:
            test_code = f"""{code}
 
# Test
result = {test['call']}
expected = {test['expected']}
assert result == expected, f"Got {{result}}, expected {{expected}}"
print("PASS")
"""
            try:
                with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
                    f.write(test_code)
                    f.flush()
                    proc = subprocess.run(
                        ['python', f.name],
                        capture_output=True, text=True, timeout=5
                    )
                    if proc.returncode == 0 and "PASS" in proc.stdout:
                        passed += 1
            except (subprocess.TimeoutExpired, Exception):
                pass
 
        results.append({
            "candidate": i + 1,
            "code": code,
            "passed": passed,
            "total": total,
            "all_passed": passed == total
        })
 
        # Early exit if we find a perfect solution
        if passed == total:
            print(f"  Candidate {i + 1}: PASSED all {total} tests")
            return {
                "solution": code,
                "candidates_tried": i + 1,
                "results": results
            }
        else:
            print(f"  Candidate {i + 1}: {passed}/{total} tests passed")
 
    # Return best solution if none passed all tests
    best = max(results, key=lambda r: r["passed"])
    return {
        "solution": best["code"],
        "candidates_tried": n_candidates,
        "best_score": f"{best['passed']}/{best['total']}",
        "results": results
    }
 
# Example
result = search_against_verifier(
    problem="""Write a function `longest_palindrome(s: str) -> str` that returns
the longest palindromic substring in s. If there are multiple with the same length,
return the first one found.""",
    test_cases=[
        {"call": "longest_palindrome('babad')", "expected": "'bab'"},
        {"call": "longest_palindrome('cbbd')", "expected": "'bb'"},
        {"call": "longest_palindrome('a')", "expected": "'a'"},
        {"call": "longest_palindrome('racecar')", "expected": "'racecar'"},
        {"call": "longest_palindrome('')", "expected": "''"},
    ],
    n_candidates=10
)
print(f"\nFound solution after {result['candidates_tried']} candidates")
print(result["solution"])

# Pseudocode for STaR training loop
def star_training(model, questions, correct_answers, num_iterations=5):
    for iteration in range(num_iterations):
        training_data = []
 
        for question, correct_answer in zip(questions, correct_answers):
            # Generate reasoning + answer
            reasoning, predicted_answer = model.generate_with_reasoning(question)
 
            if predicted_answer == correct_answer:
                # Direct: model got it right naturally
                training_data.append({
                    "question": question,
                    "reasoning": reasoning,
                    "answer": correct_answer,
                    "type": "direct"
                })
            else:
                # Rationalization: hint the correct answer and get reasoning
                hint_reasoning, _ = model.generate_with_reasoning(
                    question,
                    hint=f"The correct answer is {correct_answer}. Show your reasoning."
                )
                training_data.append({
                    "question": question,
                    "reasoning": hint_reasoning,
                    "answer": correct_answer,
                    "type": "rationalized"
                })
 
        # Fine-tune model on collected reasoning traces
        model = fine_tune(model, training_data)
        accuracy = evaluate(model, held_out_questions)
        print(f"Iteration {iteration + 1}: accuracy = {accuracy:.2%}")
 
    return model

# Pseudocode for RL-based reasoning training
def train_reasoning_with_rl(policy_model, problems, verifier):
    """
    policy_model: The LLM we're training
    problems: Math/code problems with known correct answers
    verifier: Can check if an answer is correct (test runner, math checker)
    """
    for batch in sample_batches(problems):
        for problem in batch:
            # Generate multiple reasoning traces (exploration)
            traces = []
            for _ in range(K):
                reasoning, answer = policy_model.generate(problem, temperature=0.8)
                is_correct = verifier.check(problem, answer)
                reward = 1.0 if is_correct else 0.0
                traces.append((reasoning, answer, reward))
 
            # Compute advantage: how much better was each trace than average?
            avg_reward = mean([t[2] for t in traces])
            advantages = [(t[0], t[1], t[2] - avg_reward) for t in traces]
 
            # Update policy: increase probability of high-reward traces,
            # decrease probability of low-reward traces
            policy_model.update(advantages)  # e.g., GRPO, PPO

# Pseudocode: Use PRM to guide step-by-step generation
def prm_guided_generation(problem, prm, n_candidates=5):
    """
    At each step, generate multiple continuations, score them with PRM,
    and keep only the best ones (beam search with PRM scoring).
    """
    beams = [{"steps": [], "score": 1.0}]
 
    for step_num in range(max_steps):
        all_candidates = []
 
        for beam in beams:
            # Generate N possible next steps
            next_steps = model.generate_next_steps(
                problem, beam["steps"], n=n_candidates
            )
 
            for step in next_steps:
                # Score this step with PRM
                step_score = prm.score_step(problem, beam["steps"] + [step])
                all_candidates.append({
                    "steps": beam["steps"] + [step],
                    "score": beam["score"] * step_score
                })
 
        # Keep top beams
        all_candidates.sort(key=lambda x: x["score"], reverse=True)
        beams = all_candidates[:beam_width]
 
        # Check if any beam has reached a final answer
        for beam in beams:
            if is_final_answer(beam["steps"][-1]):
                return beam
 
    return beams[0]  # Return best beam

# Pseudocode: Self-refinement training data generation
def generate_refinement_training_data(model, problems, verifier):
    training_examples = []
 
    for problem in problems:
        # Initial attempt
        initial_response = model.generate(problem)
        initial_correct = verifier.check(problem, initial_response)
 
        # Self-critique
        critique = model.generate(
            f"Critique this solution:\n{problem}\n{initial_response}"
        )
 
        # Revision
        revised_response = model.generate(
            f"Revise based on critique:\n{problem}\n{initial_response}\nCritique: {critique}"
        )
        revised_correct = verifier.check(problem, revised_response)
 
        # Only keep examples where refinement actually improved the answer
        if not initial_correct and revised_correct:
            training_examples.append({
                "problem": problem,
                "initial": initial_response,
                "critique": critique,
                "revision": revised_response,
                "label": "improvement"
            })
 
    return training_examples

import anthropic
import json
from datetime import datetime
 
client = anthropic.Anthropic()
 
# --- Tool definitions ---
tools = [
    {
        "name": "search_web",
        "description": "Search the web for current information. Use specific, targeted queries.",
        "input_schema": {
            "type": "object",
            "properties": {
                "query": {
                    "type": "string",
                    "description": "The search query"
                }
            },
            "required": ["query"]
        }
    },
    {
        "name": "fetch_page",
        "description": "Fetch and read the full content of a web page.",
        "input_schema": {
            "type": "object",
            "properties": {
                "url": {
                    "type": "string",
                    "description": "The URL to fetch"
                }
            },
            "required": ["url"]
        }
    }
]
 
 
def execute_tool(name: str, args: dict) -> str:
    """Execute a tool call. Replace with real implementations."""
    if name == "search_web":
        return search_web(args["query"])  # Your search implementation
    elif name == "fetch_page":
        return fetch_page(args["url"])    # Your fetch implementation
    return f"Unknown tool: {name}"
 
 
def run_agent_loop(system_prompt: str, user_message: str, max_steps: int = 15) -> str:
    """Run a ReACT-style agent loop with tool calling."""
    messages = [{"role": "user", "content": user_message}]
 
    for step in range(max_steps):
        response = client.messages.create(
            model="claude-sonnet-4-6",
            max_tokens=4096,
            system=system_prompt,
            tools=tools,
            messages=messages,
        )
 
        messages.append({"role": "assistant", "content": response.content})
 
        if response.stop_reason == "tool_use":
            tool_results = []
            for block in response.content:
                if block.type == "tool_use":
                    result = execute_tool(block.name, block.input)
                    tool_results.append({
                        "type": "tool_result",
                        "tool_use_id": block.id,
                        "content": result
                    })
            messages.append({"role": "user", "content": tool_results})
        else:
            # Extract text from response
            return "".join(
                block.text for block in response.content if hasattr(block, "text")
            )
 
    return "Agent reached maximum steps."
 
 
# --- Phase 1: Question Analysis ---
def analyze_question(question: str) -> dict:
    """Decompose the question into sub-questions and create a research plan."""
    response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=2048,
        temperature=1,
        thinking={"type": "enabled", "budget_tokens": 4000},
        messages=[{
            "role": "user",
            "content": f"""Analyze this research question and create a research plan.
 
Question: {question}
 
Return a JSON object with:
1. "sub_questions": Array of 3-6 specific sub-questions to investigate
2. "source_types": What types of sources would be most valuable
   (academic papers, news articles, documentation, expert blogs, etc.)
3. "search_queries": Array of 5-8 specific search queries to run
4. "known_context": What you already know about this topic (brief)
5. "key_uncertainties": What you're most uncertain about
 
Return ONLY the JSON object."""
        }]
    )
 
    # Extract text content (skip thinking blocks)
    text = ""
    for block in response.content:
        if hasattr(block, "text"):
            text = block.text
            break
 
    try:
        return json.loads(text)
    except json.JSONDecodeError:
        return {
            "sub_questions": [question],
            "search_queries": [question],
            "source_types": ["general"],
            "known_context": "",
            "key_uncertainties": ["Unable to parse analysis"]
        }
 
 
# --- Phase 2: Research Loop ---
def research_sub_question(sub_question: str, search_queries: list) -> str:
    """Research a specific sub-question using web search."""
    system_prompt = f"""You are a research assistant investigating this specific question:
"{sub_question}"
 
Your goal:
1. Search the web using the provided queries (and create new ones if needed)
2. Read the most relevant pages
3. Extract key findings, noting the source for each fact
4. Note any contradictions between sources
5. When you have enough information, provide a structured summary
 
Format your final output as:
## Key Findings
[Numbered list of findings with source attribution]
 
## Source Quality Assessment
[Brief assessment of how reliable your sources are]
 
## Contradictions or Uncertainties
[Any conflicting information or gaps]
 
## Sources
[Numbered list of URLs used]"""
 
    query_text = "\n".join(f"- {q}" for q in search_queries[:3])
    user_message = f"""Research this question: {sub_question}
 
Start with these search queries:
{query_text}
 
Search, read relevant pages, and provide a comprehensive summary."""
 
    return run_agent_loop(system_prompt, user_message, max_steps=12)
 
 
# --- Phase 3: Synthesis with Reasoning ---
def synthesize_research(question: str, research_findings: list) -> str:
    """Use extended thinking to synthesize all research into a coherent analysis."""
    findings_text = "\n\n---\n\n".join(
        f"### Sub-question {i+1}\n{finding}"
        for i, finding in enumerate(research_findings)
    )
 
    response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=8000,
        temperature=1,
        thinking={"type": "enabled", "budget_tokens": 8000},
        messages=[{
            "role": "user",
            "content": f"""You are writing a comprehensive research report.
 
Original question: {question}
 
Here are the research findings from investigating different aspects of this question:
 
{findings_text}
 
Write a comprehensive, well-structured research report that:
1. Synthesizes findings from all sub-questions into a coherent narrative
2. Resolves contradictions (explain which sources are more credible and why)
3. Clearly states confidence levels (high/medium/low) for each major claim
4. Includes inline citations [1], [2], etc.
5. Has a "Limitations and Gaps" section noting what you couldn't find or verify
6. Ends with a consolidated sources list
 
The report should be thorough but readable — like a research briefing for a smart
non-expert who needs to make decisions based on this information."""
        }]
    )
 
    # Extract text (skip thinking)
    for block in response.content:
        if hasattr(block, "text"):
            return block.text
    return "Synthesis failed."
 
 
# --- Phase 4: Self-Critique and Revision ---
def critique_and_revise(question: str, report: str) -> str:
    """Review the report for gaps and errors, then revise."""
    # Step 1: Critique
    critique_response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=2048,
        temperature=1,
        thinking={"type": "enabled", "budget_tokens": 4000},
        messages=[{
            "role": "user",
            "content": f"""Critically review this research report. Be thorough and harsh.
 
Original question: {question}
 
Report:
{report}
 
Evaluate:
1. Are there logical errors or unsupported claims?
2. Are important perspectives or counterarguments missing?
3. Are the confidence levels appropriate?
4. Is any information likely outdated or incorrect?
5. Are there follow-up questions that should have been investigated?
 
If the report is excellent, respond with "REPORT APPROVED".
Otherwise, list specific issues that need to be fixed."""
        }]
    )
 
    critique_text = ""
    for block in critique_response.content:
        if hasattr(block, "text"):
            critique_text = block.text
            break
 
    if "REPORT APPROVED" in critique_text.upper():
        return report
 
    # Step 2: Revise based on critique
    revision_response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=8000,
        temperature=1,
        thinking={"type": "enabled", "budget_tokens": 4000},
        messages=[{
            "role": "user",
            "content": f"""Revise this research report based on the critique below.
Fix all identified issues. If the critique mentions missing information that requires
additional research, note it in the Limitations section rather than making things up.
 
Original question: {question}
 
Report:
{report}
 
Critique:
{critique_text}
 
Provide the complete revised report:"""
        }]
    )
 
    for block in revision_response.content:
        if hasattr(block, "text"):
            return block.text
    return report
 
 
# --- Main: Deep Research Pipeline ---
def deep_research(question: str) -> str:
    """
    Run the full deep research pipeline:
    1. Analyze the question and plan research
    2. Research each sub-question with web search
    3. Synthesize findings with extended reasoning
    4. Self-critique and revise
    """
    print(f"Deep Research: {question}\n")
 
    # Phase 1: Analyze
    print("Phase 1: Analyzing question...")
    plan = analyze_question(question)
    print(f"  Sub-questions: {len(plan['sub_questions'])}")
    print(f"  Search queries: {len(plan['search_queries'])}")
 
    # Phase 2: Research each sub-question
    print("\nPhase 2: Researching...")
    findings = []
    for i, sub_q in enumerate(plan["sub_questions"]):
        print(f"  Researching sub-question {i+1}: {sub_q[:80]}...")
        # Pick relevant search queries for this sub-question
        relevant_queries = plan["search_queries"][
            i * 2 : (i + 1) * 2
        ] or [sub_q]
        finding = research_sub_question(sub_q, relevant_queries)
        findings.append(finding)
 
    # Phase 3: Synthesize
    print("\nPhase 3: Synthesizing research...")
    report = synthesize_research(question, findings)
 
    # Phase 4: Critique and revise
    print("\nPhase 4: Self-critique and revision...")
    final_report = critique_and_revise(question, report)
 
    print("\nDone!")
    return final_report
 
 
# --- Run it ---
if __name__ == "__main__":
    report = deep_research(
        "What are the current best practices for building reasoning-capable AI agents, "
        "and how do inference-time scaling techniques compare to training-time approaches "
        "in terms of cost, quality, and practical applicability?"
    )
    print("\n" + "=" * 80)
    print(report)

def deep_research_with_consistency(question: str, n_runs: int = 3) -> str:
    """
    Run deep research multiple times and synthesize the most consistent findings.
    """
    reports = []
    for i in range(n_runs):
        print(f"\n{'='*40} Run {i+1}/{n_runs} {'='*40}")
        report = deep_research(question)
        reports.append(report)
 
    # Synthesize across runs
    reports_text = "\n\n---\n\n".join(
        f"### Report {i+1}\n{report}" for i, report in enumerate(reports)
    )
 
    response = client.messages.create(
        model="claude-sonnet-4-6",
        max_tokens=8000,
        temperature=1,
        thinking={"type": "enabled", "budget_tokens": 8000},
        messages=[{
            "role": "user",
            "content": f"""You ran a deep research pipeline {n_runs} times on the same question.
Each run independently searched the web, analyzed sources, and produced a report.
 
Question: {question}
 
Here are all {n_runs} reports:
 
{reports_text}
 
Synthesize these into a single, definitive report:
1. Claims that appear in all/most reports are HIGH CONFIDENCE
2. Claims that appear in only one report need verification — mark as LOW CONFIDENCE
3. Contradictions between reports should be explicitly noted
4. Combine the best citations from all reports
 
Produce the final consolidated research report:"""
        }]
    )
 
    for block in response.content:
        if hasattr(block, "text"):
            return block.text
    return reports[0]