Weaviate Cookbooks

uv init multimodal-rag
cd multimodal-rag
uv venv

# macOS/Linux
curl -LsSf https://astral.sh/uv/install.sh | sh

# Or with pip
pip install uv

# Or with Homebrew
brew install uv

uv add "weaviate-client>=4.23.1,<5"

# For loading Hugging Face datasets
uv add datasets

# For PDF processing (pdf2image requires poppler to be installed!)
uv add pdf2image pillow

# For VLM generation via Ollama
uv add ollama

{
    "document_id": str,
    "page_number": int,
    "image": PIL.Image,
    "metadata": dict  # title, author, date, etc.
}

import os
import weaviate
from weaviate.classes.init import Auth

WEAVIATE_URL = os.getenv("WEAVIATE_URL")
WEAVIATE_API_KEY = os.getenv("WEAVIATE_API_KEY")

client = weaviate.connect_to_weaviate_cloud(
    cluster_url=WEAVIATE_URL,
    auth_credentials=Auth.api_key(WEAVIATE_API_KEY),
)

from weaviate.classes.config import Configure, Property, DataType

collection_name = "PDFDocuments"  # Use a descriptive name for your use case

collection = client.collections.create(
    name=collection_name,
    properties=[
        Property(name="doc_page", data_type=DataType.BLOB),
        Property(name="page_id", data_type=DataType.INT),
        Property(name="document_id", data_type=DataType.TEXT),
        Property(name="page_number", data_type=DataType.INT),
        Property(name="title", data_type=DataType.TEXT),
        # Add other metadata properties as needed
    ],
    vector_config=[
        Configure.MultiVectors.multi2vec_weaviate(
            name="doc_vector",
            image_field="doc_page",
            model="ModernVBERT/colmodernvbert",
            encoding=Configure.VectorIndex.MultiVector.Encoding.muvera(
                ksim=4,
                dprojections=16,
                repetitions=20,
            ),
        )
    ],
)

vector_config=[
    Configure.MultiVectors.multi2vec_weaviate(
        name="doc_vector",
        image_field="doc_page",
        model="ModernVBERT/colmodernvbert",
    )
],

import base64
from io import BytesIO

def image_to_base64(image):
    """Convert a PIL Image to a base64-encoded string.

    Args:
        image: PIL.Image object

    Returns:
        Base64-encoded string of the JPEG image
    """
    buffer = BytesIO()
    image.save(buffer, format="JPEG")
    return base64.b64encode(buffer.getvalue()).decode("utf-8")

collection = client.collections.get(collection_name)

with collection.batch.dynamic() as batch:
    for idx, document in enumerate(your_document_dataset):
        # Convert image to base64
        img_base64 = image_to_base64(document["image"])

        # Add object to batch — Weaviate generates embeddings automatically
        batch.add_object(
            properties={
                "doc_page": img_base64,
                "page_id": document["page_id"],
                "document_id": document["document_id"],
                "page_number": document["page_number"],
                "title": document.get("title", ""),
                # Add other properties from your dataset
            },
        )

        # Progress tracking
        if idx % 25 == 0:
            print(f"Indexed {idx+1}/{len(your_document_dataset)} documents")

# Clean up dataset if memory is limited
del your_document_dataset

print(f"Total documents indexed: {len(collection)}")

from weaviate.classes.query import MetadataQuery

def search_documents(query_text, limit=3):
    """Search for documents using Weaviate Embeddings multimodal model.

    Args:
        query_text: Natural language query string
        limit: Number of results to return (default: 3)

    Returns:
        List of dicts with document properties, similarity scores, and base64 images
    """
    collection = client.collections.get(collection_name)

    # Search — Weaviate embeds the query server-side
    # Include doc_page in return_properties to get the base64-encoded image blob
    response = collection.query.near_text(
        query=query_text,
        limit=limit,
        return_properties=["page_id", "document_id", "page_number", "title", "doc_page"],
        return_metadata=MetadataQuery(distance=True),
    )

    # Process and format results
    results = []
    for i, obj in enumerate(response.objects):
        props = obj.properties
        results.append({
            "rank": i + 1,
            "page_id": props["page_id"],
            "document_id": props["document_id"],
            "page_number": props["page_number"],
            "title": props["title"],
            "distance": obj.metadata.distance,
            "image_base64": props["doc_page"],  # Already base64-encoded
        })

    return results

# Example usage
query = "How does DeepSeek-V2 compare against the LLaMA family of LLMs?"
results = search_documents(query, limit=3)

for result in results:
    print(f"{result['rank']}) Distance: {result['distance']:.4f}, "
          f"Title: \"{result['title']}\", Page: {result['page_number']}")

from weaviate.classes.query import Filter

# Example: Filter by document ID
response = collection.query.near_text(
    query="query text",
    limit=5,
    filters=Filter.by_property("document_id").equal("paper_123"),
)

# Example: Filter by page range
response = collection.query.near_text(
    query="query text",
    limit=5,
    filters=Filter.by_property("page_number").less_than(10),
)

# Example: Combine multiple filters
response = collection.query.near_text(
    query="query text",
    limit=5,
    filters=(
        Filter.by_property("document_id").equal("paper_123") &
        Filter.by_property("page_number").less_than(10)
    ),
)

# Hybrid search: vector + keyword (Weaviate handles embedding)
response = collection.query.hybrid(
    query="query text",
    alpha=0.7,  # 0.0=keyword only, 0.5=balanced, 1.0=vector only
    limit=5,
)

# Install Ollama (macOS/Linux)
curl -fsSL https://ollama.com/install.sh | sh

# Or on macOS with Homebrew
brew install ollama

# Pull a vision language model
ollama pull qwen3-vl:4b

ollama list

import base64
import ollama

class OllamaVLM:
    def __init__(self, model_name="qwen3-vl:4b"):
        """Initialize with an Ollama vision model name.

        Args:
            model_name: Ollama model tag (must support vision)
        """
        self.model_name = model_name

    def generate_answer(self, query, images_base64, max_tokens=128):
        """Generate text response based on query and retrieved document images.

        Args:
            query: String text query
            images_base64: List of base64-encoded image strings (as returned by Weaviate)
            max_tokens: Maximum tokens to generate (default: 128)

        Returns:
            Generated text answer as string
        """
        # The Ollama SDK "images" key accepts bytes or path-like strings,
        # so decode the base64 strings from Weaviate into raw bytes
        images_bytes = [base64.b64decode(img) for img in images_base64]

        response = ollama.chat(
            model=self.model_name,
            messages=[{
                "role": "user",
                "content": query,
                "images": images_bytes,
            }],
            options={"num_predict": max_tokens},
        )

        return response["message"]["content"]

# Instantiate the VLM
vlm = OllamaVLM(model_name="qwen3-vl:4b")

def multimodal_rag(query, num_documents=3, max_tokens=128):
    """Complete multimodal RAG pipeline using Weaviate Embeddings + Ollama VLM.

    Args:
        query: Natural language question
        num_documents: Number of documents to retrieve (1-3 recommended)
        max_tokens: Maximum tokens for VLM response

    Returns:
        Dict with query, answer, sources, and metadata
    """
    # Step 1: Retrieve relevant documents (Weaviate handles embedding)
    print(f"Searching for: {query}")
    retrieved_docs = search_documents(query, limit=num_documents)

    # Display retrieved sources
    print(f"\nRetrieved {len(retrieved_docs)} documents:")
    for doc in retrieved_docs:
        print(f"  - {doc['title']}, Page {doc['page_number']} "
              f"(Distance: {doc['distance']:.4f})")

    # Step 2: Extract base64 images from results
    context_images = [doc["image_base64"] for doc in retrieved_docs]

    # Step 3: Generate answer using Ollama VLM
    print(f"\nGenerating answer...")
    answer = vlm.generate_answer(query, context_images, max_tokens=max_tokens)

    # Step 4: Return structured response
    return {
        "query": query,
        "answer": answer,
        "sources": retrieved_docs,
        "num_sources": len(retrieved_docs)
    }

# Example usage
query = "How does DeepSeek-V2 compare against the LLaMA family of LLMs?"
result = multimodal_rag(query, num_documents=1, max_tokens=128)

print(f"\nQuery: {result['query']}")
print(f"Answer: {result['answer']}")
print(f"\nBased on {result['num_sources']} source(s)")

def generate_with_citations(query, retrieved_docs, max_tokens=256):
    """Generate answer with source citations.

    Args:
        query: User question
        retrieved_docs: List of documents from search_documents()
        max_tokens: Maximum response length

    Returns:
        Answer string with embedded citations
    """
    # Build source references
    sources_text = "\n".join([
        f"Source {i+1}: \"{doc['title']}\", Page {doc['page_number']}"
        for i, doc in enumerate(retrieved_docs)
    ])

    # Enhanced prompt with citation instructions
    enhanced_query = f"""{query}

Available sources:
{sources_text}

Instructions: Answer the question based on the provided document images.
Cite sources in your answer using [Source N] notation."""

    # Generate answer with citations
    answer = vlm.generate_answer(
        enhanced_query,
        [doc["image_base64"] for doc in retrieved_docs],
        max_tokens=max_tokens
    )

    return answer, retrieved_docs

# Example usage
query = "What is the architecture of GPT-4?"
answer, sources = generate_with_citations(query, search_documents(query, limit=3))
print(f"Answer: {answer}\n")
print("Sources:")
for src in sources:
    print(f"  - {src['title']}, Page {src['page_number']}")

Error: WEAVIATE_URL environment variable is not set

WeaviateConnectionError: Failed to connect to Weaviate

ConnectionError: Failed to connect to Ollama

ollama serve

ollama._types.ResponseError: model 'qwen3-vl:4b' not found

ollama pull qwen3-vl:4b

response = collection.query.near_text(
    query=query_text,
    limit=limit,
    return_properties=["page_id", "document_id", "page_number", "title", "doc_page"],
)

Exception: Unable to get page count. Is poppler installed and in PATH?

# macOS
brew install poppler

# Ubuntu/Debian
sudo apt-get install poppler-utils

TypeError: _MultiVectors.multi2vec_weaviate() got an unexpected keyword argument 'image_fields'

Configure.MultiVectors.multi2vec_weaviate(
    name="doc_vector",
    image_field="doc_page",
    ...
)