Using LlamaIndex-Backed Agent

This example demonstrates how to create an AI agent using LlamaIndex.

First install the dependencies:

# pip install "llama-index-readers-web" "llama-index-readers-wikipedia" "llama-index-tools-wikipedia" "llama-index-embeddings-azure-openai" "llama-index-llms-azure-openai" "llama-index" "azure-identity"

Let’s import the modules.

import os
from dataclasses import dataclass
from typing import List, Optional

from autogen_core import AgentId, MessageContext, RoutedAgent, SingleThreadedAgentRuntime, message_handler
from azure.identity import DefaultAzureCredential, get_bearer_token_provider
from llama_index.core import Settings
from llama_index.core.agent import ReActAgent
from llama_index.core.agent.runner.base import AgentRunner
from llama_index.core.base.llms.types import (
    ChatMessage,
    MessageRole,
)
from llama_index.core.chat_engine.types import AgentChatResponse
from llama_index.core.memory import ChatSummaryMemoryBuffer
from llama_index.core.memory.types import BaseMemory
from llama_index.embeddings.azure_openai import AzureOpenAIEmbedding
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.llms.azure_openai import AzureOpenAI
from llama_index.llms.openai import OpenAI
from llama_index.tools.wikipedia import WikipediaToolSpec

Define our message type that will be used to communicate with the agent.

@dataclass
class Resource:
    content: str
    node_id: str
    score: Optional[float] = None


@dataclass
class Message:
    content: str
    sources: Optional[List[Resource]] = None

Define the agent using LLamaIndex’s API.

class LlamaIndexAgent(RoutedAgent):
    def __init__(self, description: str, llama_index_agent: AgentRunner, memory: BaseMemory | None = None) -> None:
        super().__init__(description)

        self._llama_index_agent = llama_index_agent
        self._memory = memory

    @message_handler
    async def handle_user_message(self, message: Message, ctx: MessageContext) -> Message:
        # retriever history messages from memory!
        history_messages: List[ChatMessage] = []

        response: AgentChatResponse  # pyright: ignore
        if self._memory is not None:
            history_messages = self._memory.get(input=message.content)

            response = await self._llama_index_agent.achat(message=message.content, history_messages=history_messages)  # pyright: ignore
        else:
            response = await self._llama_index_agent.achat(message=message.content)  # pyright: ignore

        if isinstance(response, AgentChatResponse):
            if self._memory is not None:
                self._memory.put(ChatMessage(role=MessageRole.USER, content=message.content))
                self._memory.put(ChatMessage(role=MessageRole.ASSISTANT, content=response.response))

            assert isinstance(response.response, str)

            resources: List[Resource] = [
                Resource(content=source_node.get_text(), score=source_node.score, node_id=source_node.id_)
                for source_node in response.source_nodes
            ]

            tools: List[Resource] = [
                Resource(content=source.content, node_id=source.tool_name) for source in response.sources
            ]

            resources.extend(tools)
            return Message(content=response.response, sources=resources)
        else:
            return Message(content="I'm sorry, I don't have an answer for you.")

Setting up LlamaIndex.

# llm = AzureOpenAI(
#     deployment_name=os.getenv("AZURE_OPENAI_DEPLOYMENT"),
#     temperature=0.0,
#     azure_ad_token_provider = get_bearer_token_provider(DefaultAzureCredential()),
#     # api_key=os.getenv("AZURE_OPENAI_API_KEY"),
#     azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
#     api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
# )
llm = OpenAI(
    model="gpt-4o",
    temperature=0.0,
    api_key=os.getenv("OPENAI_API_KEY"),
)

# embed_model = AzureOpenAIEmbedding(
#     deployment_name=os.getenv("AZURE_OPENAI_EMBEDDING_MODEL"),
#     temperature=0.0,
#     azure_ad_token_provider = get_bearer_token_provider(DefaultAzureCredential()),
#     api_key=os.getenv("AZURE_OPENAI_API_KEY"),
#     azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"),
#     api_version=os.getenv("AZURE_OPENAI_API_VERSION"),
# )
embed_model = OpenAIEmbedding(
    model="text-embedding-ada-002",
    api_key=os.getenv("OPENAI_API_KEY"),
)

Settings.llm = llm
Settings.embed_model = embed_model

Create the tools.

wiki_spec = WikipediaToolSpec()
wikipedia_tool = wiki_spec.to_tool_list()[1]

Now let’s test the agent. First we need to create an agent runtime and register the agent, by providing the agent’s name and a factory function that will create the agent.

runtime = SingleThreadedAgentRuntime()
await LlamaIndexAgent.register(
    runtime,
    "chat_agent",
    lambda: LlamaIndexAgent(
        description="Llama Index Agent",
        llama_index_agent=ReActAgent.from_tools(
            tools=[wikipedia_tool],
            llm=llm,
            max_iterations=8,
            memory=ChatSummaryMemoryBuffer(llm=llm, token_limit=16000),
            verbose=True,
        ),
    ),
)
agent = AgentId("chat_agent", "default")

Start the agent runtime.

runtime.start()

Send a direct message to the agent, and print the response.

message = Message(content="What are the best movies from studio Ghibli?")
response = await runtime.send_message(message, agent)
assert isinstance(response, Message)
print(response.content)

> Running step 3cbf60cd-9827-4dfe-a3a9-eaff2bed9b75. Step input: What are the best movies from studio Ghibli?
Thought: The current language of the user is: English. I need to use a tool to help me answer the question.
Action: search_data
Action Input: {'query': 'best movies from Studio Ghibli'}
Observation: This is a list of works (films, television, shorts etc.) by the Japanese animation studio Studio Ghibli.


== Works ==


=== Feature films ===


=== Television ===


=== Short films ===

These are short films, including those created for television, theatrical release, and the Ghibli Museum. Original video animation releases and music videos (theatrical and television) are also listed in this section.


=== Commercials ===


=== Video games ===


=== Stage productions ===
Princess Mononoke (2013)
Nausicaä of the Valley of the Wind (2019)
Spirited Away (2022)
My Neighbour Totoro (2022)


=== Other works ===
The works listed here consist of works that do not fall into the above categories. All of these films have been released on DVD or Blu-ray in Japan as part of the Ghibli Gakujutsu Library.


=== Exhibitions ===
A selection of layout designs for animated productions was exhibited in the Studio Ghibli Layout Designs: Understanding the Secrets of Takahata and Miyazaki Animation exhibition tour, which started in the Museum of Contemporary Art Tokyo (July 28, 2008 to September 28, 2008) and subsequently travelled to different museums throughout Japan and Asia, concluding its tour of Japan in the Fukuoka Asian Art Museum (October 12, 2013 to January 26, 2014) and its tour of Asia in the Hong Kong Heritage Museum (May 14, 2014 to August 31, 2014). Between October 4, 2014 and March 1, 2015 the layout designs were exhibited at Art Ludique in Paris. The exhibition catalogues contain annotated reproductions of the displayed artwork.


== Related works ==
These works were not created by Studio Ghibli, but were produced by a variety of studios and people who went on to form or join Studio Ghibli. This includes members of Topcraft that went on to create Studio Ghibli in 1985; works produced by Toei Animation, TMS Entertainment, Nippon Animation or other studios and featuring involvement by Hayao Miyazaki, Isao Takahata or other Ghibli staffers. The list also includes works created in cooperation with Studio Ghibli.


=== Pre-Ghibli ===


=== Cooperative works ===


=== Distributive works ===
These Western animated films (plus one Japanese film) have been distributed by Studio Ghibli, and now through their label, Ghibli Museum Library.


=== Contributive works ===
Studio Ghibli has made contributions to the following anime series and movies:


== Significant achievements ==
The highest-grossing film of 1989 in Japan: Kiki's Delivery Service
The highest-grossing film of 1991 in Japan: Only Yesterday
The highest-grossing film of 1992 in Japan: Porco Rosso
The highest-grossing film of 1994 in Japan: Pom Poko
The highest-grossing film of 1995 in Japan; the first Japanese film in Dolby Digital: Whisper of the Heart
The highest-grossing film of 2002 in Japan: Spirited Away
The highest-grossing film of 2008 in Japan: Ponyo
The highest-grossing Japanese film of 2010 in Japan: The Secret World of Arrietty
The highest-grossing film of 2013 in Japan: The Wind Rises
The first Studio Ghibli film to use computer graphics: Pom Poko
The first Miyazaki feature to use computer graphics, and the first Studio Ghibli film to use digital coloring; the first animated feature in Japan's history to gross more than 10 billion yen at the box office and the first animated film ever to win a National Academy Award for Best Picture of the Year: Princess Mononoke
The first Studio Ghibli film to be shot using a 100% digital process: My Neighbors the Yamadas
The first Miyazaki feature to be shot using a 100% digital process; the first film to gross $200 million worldwide before opening in North America; the film to finally overtake Titanic at the Japanese box office, becoming the top-grossing film in the history of Japanese cinema: Spirited Away
The first anime and traditionally animated winner of the Academy Award for Best Animated Feature: Spirited Away at the 75th Academy Awards. They would later win this award for a second time with The Boy and the Heron at the 96th Academy Awards, marking the second time a traditionally animated film won the award.


== Notes ==


== References ==
> Running step 561e3dd3-d98b-4d37-b612-c99387182ee0. Step input: None
Thought: I can answer without using any more tools. I'll use the user's language to answer.
Answer: Studio Ghibli has produced many acclaimed films over the years. Some of the best and most popular movies from Studio Ghibli include:

1. **Spirited Away (2001)** - Directed by Hayao Miyazaki, this film won the Academy Award for Best Animated Feature and is one of the highest-grossing films in Japanese history.
2. **My Neighbor Totoro (1988)** - Another classic by Hayao Miyazaki, this film is beloved for its heartwarming story and iconic characters.
3. **Princess Mononoke (1997)** - This epic fantasy film, also directed by Miyazaki, is known for its complex themes and stunning animation.
4. **Howl's Moving Castle (2004)** - Based on the novel by Diana Wynne Jones, this film features a magical story and beautiful animation.
5. **Kiki's Delivery Service (1989)** - A charming coming-of-age story about a young witch starting her own delivery service.
6. **Grave of the Fireflies (1988)** - Directed by Isao Takahata, this poignant film is a heartbreaking tale of two siblings struggling to survive during World War II.
7. **Ponyo (2008)** - A delightful and visually stunning film about a young fish-girl who wants to become human.
8. **The Wind Rises (2013)** - A more mature film by Miyazaki, focusing on the life of an aircraft designer during wartime Japan.
9. **The Secret World of Arrietty (2010)** - Based on Mary Norton's novel "The Borrowers," this film tells the story of tiny people living secretly in a human house.
10. **Whisper of the Heart (1995)** - A touching story about a young girl discovering her passion for writing.

These films are celebrated for their storytelling, animation quality, and emotional depth.

Studio Ghibli has produced many acclaimed films over the years. Some of the best and most popular movies from Studio Ghibli include:

1. **Spirited Away (2001)** - Directed by Hayao Miyazaki, this film won the Academy Award for Best Animated Feature and is one of the highest-grossing films in Japanese history.
2. **My Neighbor Totoro (1988)** - Another classic by Hayao Miyazaki, this film is beloved for its heartwarming story and iconic characters.
3. **Princess Mononoke (1997)** - This epic fantasy film, also directed by Miyazaki, is known for its complex themes and stunning animation.
4. **Howl's Moving Castle (2004)** - Based on the novel by Diana Wynne Jones, this film features a magical story and beautiful animation.
5. **Kiki's Delivery Service (1989)** - A charming coming-of-age story about a young witch starting her own delivery service.
6. **Grave of the Fireflies (1988)** - Directed by Isao Takahata, this poignant film is a heartbreaking tale of two siblings struggling to survive during World War II.
7. **Ponyo (2008)** - A delightful and visually stunning film about a young fish-girl who wants to become human.
8. **The Wind Rises (2013)** - A more mature film by Miyazaki, focusing on the life of an aircraft designer during wartime Japan.
9. **The Secret World of Arrietty (2010)** - Based on Mary Norton's novel "The Borrowers," this film tells the story of tiny people living secretly in a human house.
10. **Whisper of the Heart (1995)** - A touching story about a young girl discovering her passion for writing.

These films are celebrated for their storytelling, animation quality, and emotional depth.

if response.sources is not None:
    for source in response.sources:
        print(source.content)

This is a list of works (films, television, shorts etc.) by the Japanese animation studio Studio Ghibli.


== Works ==


=== Feature films ===


=== Television ===


=== Short films ===

These are short films, including those created for television, theatrical release, and the Ghibli Museum. Original video animation releases and music videos (theatrical and television) are also listed in this section.


=== Commercials ===


=== Video games ===


=== Stage productions ===
Princess Mononoke (2013)
Nausicaä of the Valley of the Wind (2019)
Spirited Away (2022)
My Neighbour Totoro (2022)


=== Other works ===
The works listed here consist of works that do not fall into the above categories. All of these films have been released on DVD or Blu-ray in Japan as part of the Ghibli Gakujutsu Library.


=== Exhibitions ===
A selection of layout designs for animated productions was exhibited in the Studio Ghibli Layout Designs: Understanding the Secrets of Takahata and Miyazaki Animation exhibition tour, which started in the Museum of Contemporary Art Tokyo (July 28, 2008 to September 28, 2008) and subsequently travelled to different museums throughout Japan and Asia, concluding its tour of Japan in the Fukuoka Asian Art Museum (October 12, 2013 to January 26, 2014) and its tour of Asia in the Hong Kong Heritage Museum (May 14, 2014 to August 31, 2014). Between October 4, 2014 and March 1, 2015 the layout designs were exhibited at Art Ludique in Paris. The exhibition catalogues contain annotated reproductions of the displayed artwork.


== Related works ==
These works were not created by Studio Ghibli, but were produced by a variety of studios and people who went on to form or join Studio Ghibli. This includes members of Topcraft that went on to create Studio Ghibli in 1985; works produced by Toei Animation, TMS Entertainment, Nippon Animation or other studios and featuring involvement by Hayao Miyazaki, Isao Takahata or other Ghibli staffers. The list also includes works created in cooperation with Studio Ghibli.


=== Pre-Ghibli ===


=== Cooperative works ===


=== Distributive works ===
These Western animated films (plus one Japanese film) have been distributed by Studio Ghibli, and now through their label, Ghibli Museum Library.


=== Contributive works ===
Studio Ghibli has made contributions to the following anime series and movies:


== Significant achievements ==
The highest-grossing film of 1989 in Japan: Kiki's Delivery Service
The highest-grossing film of 1991 in Japan: Only Yesterday
The highest-grossing film of 1992 in Japan: Porco Rosso
The highest-grossing film of 1994 in Japan: Pom Poko
The highest-grossing film of 1995 in Japan; the first Japanese film in Dolby Digital: Whisper of the Heart
The highest-grossing film of 2002 in Japan: Spirited Away
The highest-grossing film of 2008 in Japan: Ponyo
The highest-grossing Japanese film of 2010 in Japan: The Secret World of Arrietty
The highest-grossing film of 2013 in Japan: The Wind Rises
The first Studio Ghibli film to use computer graphics: Pom Poko
The first Miyazaki feature to use computer graphics, and the first Studio Ghibli film to use digital coloring; the first animated feature in Japan's history to gross more than 10 billion yen at the box office and the first animated film ever to win a National Academy Award for Best Picture of the Year: Princess Mononoke
The first Studio Ghibli film to be shot using a 100% digital process: My Neighbors the Yamadas
The first Miyazaki feature to be shot using a 100% digital process; the first film to gross $200 million worldwide before opening in North America; the film to finally overtake Titanic at the Japanese box office, becoming the top-grossing film in the history of Japanese cinema: Spirited Away
The first anime and traditionally animated winner of the Academy Award for Best Animated Feature: Spirited Away at the 75th Academy Awards. They would later win this award for a second time with The Boy and the Heron at the 96th Academy Awards, marking the second time a traditionally animated film won the award.


== Notes ==


== References ==

Stop the agent runtime.

await runtime.stop()