Local Search
In [1]:
Copied!
# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License.
# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License.
In [2]:
Copied!
import os
import pandas as pd
import tiktoken
from graphrag.query.context_builder.entity_extraction import EntityVectorStoreKey
from graphrag.query.indexer_adapters import (
read_indexer_covariates,
read_indexer_entities,
read_indexer_relationships,
read_indexer_reports,
read_indexer_text_units,
)
from graphrag.query.question_gen.local_gen import LocalQuestionGen
from graphrag.query.structured_search.local_search.mixed_context import (
LocalSearchMixedContext,
)
from graphrag.query.structured_search.local_search.search import LocalSearch
from graphrag.vector_stores.lancedb import LanceDBVectorStore
import os
import pandas as pd
import tiktoken
from graphrag.query.context_builder.entity_extraction import EntityVectorStoreKey
from graphrag.query.indexer_adapters import (
read_indexer_covariates,
read_indexer_entities,
read_indexer_relationships,
read_indexer_reports,
read_indexer_text_units,
)
from graphrag.query.question_gen.local_gen import LocalQuestionGen
from graphrag.query.structured_search.local_search.mixed_context import (
LocalSearchMixedContext,
)
from graphrag.query.structured_search.local_search.search import LocalSearch
from graphrag.vector_stores.lancedb import LanceDBVectorStore
Local Search Example¶
Local search method generates answers by combining relevant data from the AI-extracted knowledge-graph with text chunks of the raw documents. This method is suitable for questions that require an understanding of specific entities mentioned in the documents (e.g. What are the healing properties of chamomile?).
Load text units and graph data tables as context for local search¶
- In this test we first load indexing outputs from parquet files to dataframes, then convert these dataframes into collections of data objects aligning with the knowledge model.
Load tables to dataframes¶
In [3]:
Copied!
INPUT_DIR = "./inputs/operation dulce"
LANCEDB_URI = f"{INPUT_DIR}/lancedb"
COMMUNITY_REPORT_TABLE = "community_reports"
ENTITY_TABLE = "entities"
COMMUNITY_TABLE = "communities"
RELATIONSHIP_TABLE = "relationships"
COVARIATE_TABLE = "covariates"
TEXT_UNIT_TABLE = "text_units"
COMMUNITY_LEVEL = 2
INPUT_DIR = "./inputs/operation dulce"
LANCEDB_URI = f"{INPUT_DIR}/lancedb"
COMMUNITY_REPORT_TABLE = "community_reports"
ENTITY_TABLE = "entities"
COMMUNITY_TABLE = "communities"
RELATIONSHIP_TABLE = "relationships"
COVARIATE_TABLE = "covariates"
TEXT_UNIT_TABLE = "text_units"
COMMUNITY_LEVEL = 2
Read entities¶
In [4]:
Copied!
# read nodes table to get community and degree data
entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet")
community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet")
entities = read_indexer_entities(entity_df, community_df, COMMUNITY_LEVEL)
# load description embeddings to an in-memory lancedb vectorstore
# to connect to a remote db, specify url and port values.
description_embedding_store = LanceDBVectorStore(
collection_name="default-entity-description",
)
description_embedding_store.connect(db_uri=LANCEDB_URI)
print(f"Entity count: {len(entity_df)}")
entity_df.head()
# read nodes table to get community and degree data
entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet")
community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet")
entities = read_indexer_entities(entity_df, community_df, COMMUNITY_LEVEL)
# load description embeddings to an in-memory lancedb vectorstore
# to connect to a remote db, specify url and port values.
description_embedding_store = LanceDBVectorStore(
collection_name="default-entity-description",
)
description_embedding_store.connect(db_uri=LANCEDB_URI)
print(f"Entity count: {len(entity_df)}")
entity_df.head()
Entity count: 18
Out[4]:
id | human_readable_id | title | type | description | text_unit_ids | frequency | degree | x | y | |
---|---|---|---|---|---|---|---|---|---|---|
0 | 425a7862-0aef-4f69-a4c8-8bd42151c9d4 | 0 | ALEX MERCER | PERSON | Agent Alex Mercer is a determined individual w... | [8e938693af886bfd081acbbe8384c3671446bff84a134... | 4 | 9 | 0 | 0 |
1 | bcdbf1fc-0dc1-460f-bc71-2781729c96ba | 1 | TAYLOR CRUZ | PERSON | Agent Taylor Cruz is a commanding and authorit... | [8e938693af886bfd081acbbe8384c3671446bff84a134... | 4 | 8 | 0 | 0 |
2 | ef02ef24-5762-46ce-93ce-7dea6fc86595 | 2 | JORDAN HAYES | PERSON | Dr. Jordan Hayes is a scientist and a member o... | [8e938693af886bfd081acbbe8384c3671446bff84a134... | 4 | 9 | 0 | 0 |
3 | 8b163d27-e43a-4a2c-a26f-866778d8720e | 3 | SAM RIVERA | PERSON | Sam Rivera is a cybersecurity expert and a tal... | [8e938693af886bfd081acbbe8384c3671446bff84a134... | 4 | 8 | 0 | 0 |
4 | 542aa5bd-ba2d-400a-8488-c52d50bc300d | 4 | PARANORMAL MILITARY SQUAD | ORGANIZATION | The PARANORMAL MILITARY SQUAD is an elite grou... | [8e938693af886bfd081acbbe8384c3671446bff84a134... | 2 | 6 | 0 | 0 |
Read relationships¶
In [5]:
Copied!
relationship_df = pd.read_parquet(f"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet")
relationships = read_indexer_relationships(relationship_df)
print(f"Relationship count: {len(relationship_df)}")
relationship_df.head()
relationship_df = pd.read_parquet(f"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet")
relationships = read_indexer_relationships(relationship_df)
print(f"Relationship count: {len(relationship_df)}")
relationship_df.head()
Relationship count: 54
Out[5]:
id | human_readable_id | source | target | description | weight | combined_degree | text_unit_ids | |
---|---|---|---|---|---|---|---|---|
0 | 2bfad9f4-5abd-48d0-8db3-a9cad9120413 | 0 | ALEX MERCER | TAYLOR CRUZ | Alex Mercer and Taylor Cruz are both agents wo... | 37.0 | 17 | [8e938693af886bfd081acbbe8384c3671446bff84a134... |
1 | 6cbb838f-9e83-4086-a684-15c8ed709e52 | 1 | ALEX MERCER | JORDAN HAYES | Alex Mercer and Jordan Hayes are both agents w... | 42.0 | 18 | [8e938693af886bfd081acbbe8384c3671446bff84a134... |
2 | bfdc25f1-80ca-477b-a304-94465b69e680 | 2 | ALEX MERCER | SAM RIVERA | Alex Mercer and Sam Rivera are both agents and... | 26.0 | 17 | [8e938693af886bfd081acbbe8384c3671446bff84a134... |
3 | 7a7e943d-a4f5-487b-9625-5d0907c4c26d | 3 | ALEX MERCER | PARANORMAL MILITARY SQUAD | Alex Mercer is a member of the Paranormal Mili... | 17.0 | 15 | [8e938693af886bfd081acbbe8384c3671446bff84a134... |
4 | 5e00bcb9-a17e-4c27-8241-6ebb286a7fc6 | 4 | ALEX MERCER | DULCE | Alex Mercer is preparing to lead the team into... | 15.0 | 14 | [8e938693af886bfd081acbbe8384c3671446bff84a134... |
In [6]:
Copied!
# NOTE: covariates are turned off by default, because they generally need prompt tuning to be valuable
# Please see the GRAPHRAG_CLAIM_* settings
covariate_df = pd.read_parquet(f"{INPUT_DIR}/{COVARIATE_TABLE}.parquet")
claims = read_indexer_covariates(covariate_df)
print(f"Claim records: {len(claims)}")
covariates = {"claims": claims}
# NOTE: covariates are turned off by default, because they generally need prompt tuning to be valuable
# Please see the GRAPHRAG_CLAIM_* settings
covariate_df = pd.read_parquet(f"{INPUT_DIR}/{COVARIATE_TABLE}.parquet")
claims = read_indexer_covariates(covariate_df)
print(f"Claim records: {len(claims)}")
covariates = {"claims": claims}
Claim records: 17
Read community reports¶
In [7]:
Copied!
report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet")
reports = read_indexer_reports(report_df, community_df, COMMUNITY_LEVEL)
print(f"Report records: {len(report_df)}")
report_df.head()
report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet")
reports = read_indexer_reports(report_df, community_df, COMMUNITY_LEVEL)
print(f"Report records: {len(report_df)}")
report_df.head()
Report records: 2
Out[7]:
id | human_readable_id | community | level | parent | children | title | summary | full_content | rank | rating_explanation | findings | full_content_json | period | size | |
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
0 | 6c3a555680d647ac8be866a129c7b0ea | 0 | 0 | 0 | -1 | [] | Operation: Dulce and Dulce Base Exploration | The community revolves around 'Operation: Dulc... | # Operation: Dulce and Dulce Base Exploration\... | 8.5 | The impact severity rating is high due to the ... | [{'explanation': 'Operation: Dulce is a signif... | {\n "title": "Operation: Dulce and Dulce Ba... | 2025-03-04 | 7 |
1 | 0127331a1ea34b8ba19de2c2a4cb3bc9 | 1 | 1 | 0 | -1 | [] | Paranormal Military Squad and Operation: Dulce | The community centers around the Paranormal Mi... | # Paranormal Military Squad and Operation: Dul... | 8.5 | The impact severity rating is high due to the ... | [{'explanation': 'Agent Alex Mercer is a key f... | {\n "title": "Paranormal Military Squad and... | 2025-03-04 | 9 |
Read text units¶
In [8]:
Copied!
text_unit_df = pd.read_parquet(f"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet")
text_units = read_indexer_text_units(text_unit_df)
print(f"Text unit records: {len(text_unit_df)}")
text_unit_df.head()
text_unit_df = pd.read_parquet(f"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet")
text_units = read_indexer_text_units(text_unit_df)
print(f"Text unit records: {len(text_unit_df)}")
text_unit_df.head()
Text unit records: 5
Out[8]:
id | human_readable_id | text | n_tokens | document_ids | entity_ids | relationship_ids | covariate_ids | |
---|---|---|---|---|---|---|---|---|
0 | 8e938693af886bfd081acbbe8384c3671446bff84a134a... | 1 | # Operation: Dulce\n\n## Chapter 1\n\nThe thru... | 1200 | [6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8... | [425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f... | [2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838... | [745d28dd-be20-411b-85ff-1c69ca70e7b3, 9cba185... |
1 | fd1f46d32e1df6cd429542aeda3d64ddf3745ccb80f443... | 2 | , the hollow echo of the bay a stark reminder ... | 1200 | [6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8... | [425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f... | [2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838... | [4f9b461f-5e8f-465d-9586-e2fc81787062, 0f74618... |
2 | 7296d9a1f046854d59079dc183de8a054c27c4843d2979... | 3 | differently than praise from others. This was... | 1200 | [6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8... | [425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f... | [2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838... | [3ef1be9c-4080-4fac-99bd-c4a636248904, 8730b20... |
3 | ac72722a02ac71242a2a91fca323198d04197daf60515d... | 4 | contrast to the rigid silence enveloping the ... | 1200 | [6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8... | [425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f... | [2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838... | [2c292047-b79a-4958-ab57-7bf7d7a22c92, 3cbd18a... |
4 | 4c277337d461a16aaf8f9760ddb8b44ef220e948a2341d... | 5 | a mask of duty.\n\nIn the midst of the descen... | 35 | [6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8... | [d084d615-3584-4ec8-9931-90aa6075c764, 4b84859... | [6efdc42e-69a2-47c0-97ec-4b296cd16d5e] | [db8da02f-f889-4bb5-8e81-ab2a72e380bb] |
In [9]:
Copied!
from graphrag.config.enums import ModelType
from graphrag.config.models.language_model_config import LanguageModelConfig
from graphrag.language_model.manager import ModelManager
api_key = os.environ["GRAPHRAG_API_KEY"]
llm_model = os.environ["GRAPHRAG_LLM_MODEL"]
embedding_model = os.environ["GRAPHRAG_EMBEDDING_MODEL"]
chat_config = LanguageModelConfig(
api_key=api_key,
type=ModelType.OpenAIChat,
model=llm_model,
max_retries=20,
)
chat_model = ModelManager().get_or_create_chat_model(
name="local_search",
model_type=ModelType.OpenAIChat,
config=chat_config,
)
token_encoder = tiktoken.encoding_for_model(llm_model)
embedding_config = LanguageModelConfig(
api_key=api_key,
type=ModelType.OpenAIEmbedding,
model=embedding_model,
max_retries=20,
)
text_embedder = ModelManager().get_or_create_embedding_model(
name="local_search_embedding",
model_type=ModelType.OpenAIEmbedding,
config=embedding_config,
)
from graphrag.config.enums import ModelType
from graphrag.config.models.language_model_config import LanguageModelConfig
from graphrag.language_model.manager import ModelManager
api_key = os.environ["GRAPHRAG_API_KEY"]
llm_model = os.environ["GRAPHRAG_LLM_MODEL"]
embedding_model = os.environ["GRAPHRAG_EMBEDDING_MODEL"]
chat_config = LanguageModelConfig(
api_key=api_key,
type=ModelType.OpenAIChat,
model=llm_model,
max_retries=20,
)
chat_model = ModelManager().get_or_create_chat_model(
name="local_search",
model_type=ModelType.OpenAIChat,
config=chat_config,
)
token_encoder = tiktoken.encoding_for_model(llm_model)
embedding_config = LanguageModelConfig(
api_key=api_key,
type=ModelType.OpenAIEmbedding,
model=embedding_model,
max_retries=20,
)
text_embedder = ModelManager().get_or_create_embedding_model(
name="local_search_embedding",
model_type=ModelType.OpenAIEmbedding,
config=embedding_config,
)
Create local search context builder¶
In [10]:
Copied!
context_builder = LocalSearchMixedContext(
community_reports=reports,
text_units=text_units,
entities=entities,
relationships=relationships,
# if you did not run covariates during indexing, set this to None
covariates=covariates,
entity_text_embeddings=description_embedding_store,
embedding_vectorstore_key=EntityVectorStoreKey.ID, # if the vectorstore uses entity title as ids, set this to EntityVectorStoreKey.TITLE
text_embedder=text_embedder,
token_encoder=token_encoder,
)
context_builder = LocalSearchMixedContext(
community_reports=reports,
text_units=text_units,
entities=entities,
relationships=relationships,
# if you did not run covariates during indexing, set this to None
covariates=covariates,
entity_text_embeddings=description_embedding_store,
embedding_vectorstore_key=EntityVectorStoreKey.ID, # if the vectorstore uses entity title as ids, set this to EntityVectorStoreKey.TITLE
text_embedder=text_embedder,
token_encoder=token_encoder,
)
Create local search engine¶
In [11]:
Copied!
# text_unit_prop: proportion of context window dedicated to related text units
# community_prop: proportion of context window dedicated to community reports.
# The remaining proportion is dedicated to entities and relationships. Sum of text_unit_prop and community_prop should be <= 1
# conversation_history_max_turns: maximum number of turns to include in the conversation history.
# conversation_history_user_turns_only: if True, only include user queries in the conversation history.
# top_k_mapped_entities: number of related entities to retrieve from the entity description embedding store.
# top_k_relationships: control the number of out-of-network relationships to pull into the context window.
# include_entity_rank: if True, include the entity rank in the entity table in the context window. Default entity rank = node degree.
# include_relationship_weight: if True, include the relationship weight in the context window.
# include_community_rank: if True, include the community rank in the context window.
# return_candidate_context: if True, return a set of dataframes containing all candidate entity/relationship/covariate records that
# could be relevant. Note that not all of these records will be included in the context window. The "in_context" column in these
# dataframes indicates whether the record is included in the context window.
# max_tokens: maximum number of tokens to use for the context window.
local_context_params = {
"text_unit_prop": 0.5,
"community_prop": 0.1,
"conversation_history_max_turns": 5,
"conversation_history_user_turns_only": True,
"top_k_mapped_entities": 10,
"top_k_relationships": 10,
"include_entity_rank": True,
"include_relationship_weight": True,
"include_community_rank": False,
"return_candidate_context": False,
"embedding_vectorstore_key": EntityVectorStoreKey.ID, # set this to EntityVectorStoreKey.TITLE if the vectorstore uses entity title as ids
"max_tokens": 12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
}
model_params = {
"max_tokens": 2_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000=1500)
"temperature": 0.0,
}
# text_unit_prop: proportion of context window dedicated to related text units
# community_prop: proportion of context window dedicated to community reports.
# The remaining proportion is dedicated to entities and relationships. Sum of text_unit_prop and community_prop should be <= 1
# conversation_history_max_turns: maximum number of turns to include in the conversation history.
# conversation_history_user_turns_only: if True, only include user queries in the conversation history.
# top_k_mapped_entities: number of related entities to retrieve from the entity description embedding store.
# top_k_relationships: control the number of out-of-network relationships to pull into the context window.
# include_entity_rank: if True, include the entity rank in the entity table in the context window. Default entity rank = node degree.
# include_relationship_weight: if True, include the relationship weight in the context window.
# include_community_rank: if True, include the community rank in the context window.
# return_candidate_context: if True, return a set of dataframes containing all candidate entity/relationship/covariate records that
# could be relevant. Note that not all of these records will be included in the context window. The "in_context" column in these
# dataframes indicates whether the record is included in the context window.
# max_tokens: maximum number of tokens to use for the context window.
local_context_params = {
"text_unit_prop": 0.5,
"community_prop": 0.1,
"conversation_history_max_turns": 5,
"conversation_history_user_turns_only": True,
"top_k_mapped_entities": 10,
"top_k_relationships": 10,
"include_entity_rank": True,
"include_relationship_weight": True,
"include_community_rank": False,
"return_candidate_context": False,
"embedding_vectorstore_key": EntityVectorStoreKey.ID, # set this to EntityVectorStoreKey.TITLE if the vectorstore uses entity title as ids
"max_tokens": 12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
}
model_params = {
"max_tokens": 2_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000=1500)
"temperature": 0.0,
}
In [12]:
Copied!
search_engine = LocalSearch(
model=chat_model,
context_builder=context_builder,
token_encoder=token_encoder,
model_params=model_params,
context_builder_params=local_context_params,
response_type="multiple paragraphs", # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report
)
search_engine = LocalSearch(
model=chat_model,
context_builder=context_builder,
token_encoder=token_encoder,
model_params=model_params,
context_builder_params=local_context_params,
response_type="multiple paragraphs", # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report
)
Run local search on sample queries¶
In [13]:
Copied!
result = await search_engine.search("Tell me about Agent Mercer")
print(result.response)
result = await search_engine.search("Tell me about Agent Mercer")
print(result.response)
Reached token limit - reverting to previous context state
### Overview of Agent Alex Mercer Agent Alex Mercer is a prominent member of the Paranormal Military Squad, an elite group tasked with executing missions such as Operation: Dulce. He is known for his determination and plays a crucial role in the exploration of the Dulce base, a mysterious and secretive location associated with advanced alien technology [Data: Entities (0, 4, 8); Relationships (23, 24, 37)]. ### Leadership and Mentorship Mercer is recognized for his leadership qualities, particularly his ability to provide guidance and emphasize the importance of intuition and trust. He serves as a mentor to Sam Rivera, offering valuable support and leadership, which highlights his role in fostering teamwork and collaboration within the squad [Data: Reports (1); Entities (0, 3); Relationships (2, 15)]. ### Professional Relationships Agent Mercer maintains professional relationships with other key members of the squad, including Taylor Cruz and Jordan Hayes. His interactions with Cruz are marked by a competitive undercurrent, as Cruz's authoritative nature often challenges Mercer's compliance. Despite this, Mercer acknowledges Cruz's authority and follows their lead during missions [Data: Reports (1); Relationships (0, 1)]. With Dr. Jordan Hayes, Mercer shares a mutual respect and understanding, particularly admiring each other's expertise and analytical abilities. Their collaboration is crucial to the success of Operation: Dulce, as they work together to explore the Dulce base and uncover its secrets [Data: Reports (1); Relationships (1, 26)]. ### Role in Operation: Dulce Agent Mercer's involvement in Operation: Dulce is significant, as he is one of the agents leading the mission into the Dulce base. His role in mission leadership and decision-making is underscored by his suspected internal conflict between compliance with protocols and his natural inclination to question and explore all details [Data: Claims (3, 5); Relationships (4, 24)]. In summary, Agent Alex Mercer is a key figure in the Paranormal Military Squad, known for his leadership, mentorship, and professional relationships with other agents. His contributions to Operation: Dulce and his interactions with team members highlight his importance in the mission's success.
In [14]:
Copied!
question = "Tell me about Dr. Jordan Hayes"
result = await search_engine.search(question)
print(result.response)
question = "Tell me about Dr. Jordan Hayes"
result = await search_engine.search(question)
print(result.response)
Reached token limit - reverting to previous context state
## Overview of Dr. Jordan Hayes Dr. Jordan Hayes is a prominent scientist and a key member of the Paranormal Military Squad, known for their expertise in physics and composed demeanor. They play a significant role in Operation: Dulce, particularly in working with alien technology, which is a central element of the mission [Data: Entities (2); Reports (1)]. ## Role in Operation: Dulce Dr. Hayes is deeply involved in the exploration of the Dulce base, where they contribute significantly to the team's efforts by analyzing and working on alien technology. Their analytical mind and reflective nature are crucial in navigating the complexities of the mission. Hayes is recognized for their ability to adapt to unknown variables, which is essential given the unpredictable nature of the operation [Data: Reports (1); Entities (2, 13); Relationships (5, 9, 18, 51)]. ## Professional Relationships Dr. Hayes maintains professional relationships with other key members of the Paranormal Military Squad, including Taylor Cruz and Sam Rivera. Their relationship with Taylor Cruz is marked by differing views on protocol and adaptability, highlighting a complex dynamic characterized by moments of mutual respect. With Sam Rivera, Hayes shares a common belief in the importance of adaptability, which is vital for the success of Operation: Dulce [Data: Reports (1); Relationships (5, 9, 25)]. ## Analytical Insights and Skepticism Dr. Hayes is portrayed as skeptical of strict adherence to protocols, emphasizing the need for adaptability and acknowledging the unknown variables that exceed the known. This skepticism is evident in their interactions and comments during briefings, where they provide analytical insights and express concerns about the mission. Hayes's ability to identify hidden elements within the Dulce base, such as a suspicious panel, further underscores their analytical prowess [Data: Claims (2, 6, 10); Sources (0, 2)]. ## Contribution to the Team As a member of the Paranormal Military Squad, Dr. Hayes's contributions extend beyond their scientific expertise. They are involved in the team's preparation and coordination efforts, ensuring that the mission is meticulously planned and executed. Their role in the lab, working on alien technology, is pivotal to understanding the implications of the mission for humanity [Data: Reports (1); Entities (2, 13); Relationships (10, 21, 27, 48)]. In summary, Dr. Jordan Hayes is a critical asset to the Paranormal Military Squad, bringing a unique blend of scientific expertise, analytical insight, and adaptability to Operation: Dulce. Their professional relationships and skepticism towards rigid protocols highlight their importance in navigating the complexities of the mission.
Inspecting the context data used to generate the response¶
In [15]:
Copied!
result.context_data["entities"].head()
result.context_data["entities"].head()
Out[15]:
id | entity | description | number of relationships | in_context | |
---|---|---|---|---|---|
0 | 2 | JORDAN HAYES | Dr. Jordan Hayes is a scientist and a member o... | 9 | True |
1 | 13 | LAB | A lab where Dr. Jordan Hayes works on alien te... | 1 | True |
2 | 10 | AGENT HAYES | Agent Hayes is a member of the team exploring ... | 5 | True |
3 | 12 | AGENT CRUZ | 5 | True | |
4 | 15 | BRIEFING ROOM | 2 | True |
In [16]:
Copied!
result.context_data["relationships"].head()
result.context_data["relationships"].head()
Out[16]:
id | source | target | description | weight | links | in_context | |
---|---|---|---|---|---|---|---|
0 | 5 | TAYLOR CRUZ | JORDAN HAYES | Taylor Cruz and Jordan Hayes have a profession... | 7.0 | 1 | True |
1 | 9 | JORDAN HAYES | SAM RIVERA | Jordan Hayes and Sam Rivera are both agents wo... | 14.0 | 4 | True |
2 | 25 | JORDAN HAYES | TAYLOR CRUZ | Jordan Hayes and Taylor Cruz are both agents w... | 18.0 | 1 | True |
3 | 6 | TAYLOR CRUZ | SAM RIVERA | Taylor Cruz and Sam Rivera are both agents wor... | 20.0 | 4 | True |
4 | 10 | JORDAN HAYES | PARANORMAL MILITARY SQUAD | Jordan Hayes is a member of the Paranormal Mil... | 9.0 | 3 | True |
In [17]:
Copied!
if "reports" in result.context_data:
result.context_data["reports"].head()
if "reports" in result.context_data:
result.context_data["reports"].head()
In [18]:
Copied!
result.context_data["sources"].head()
result.context_data["sources"].head()
Out[18]:
id | text | |
---|---|---|
0 | 0 | # Operation: Dulce\n\n## Chapter 1\n\nThe thru... |
1 | 2 | differently than praise from others. This was... |
2 | 3 | contrast to the rigid silence enveloping the ... |
In [19]:
Copied!
if "claims" in result.context_data:
print(result.context_data["claims"].head())
if "claims" in result.context_data:
print(result.context_data["claims"].head())
id entity object_id status start_date end_date \ 0 2 JORDAN HAYES NONE SUSPECTED NONE NONE 1 6 JORDAN HAYES NONE SUSPECTED NONE NONE 2 10 JORDAN HAYES NONE TRUE NONE NONE 3 1 TAYLOR CRUZ NONE SUSPECTED NONE NONE 4 7 TAYLOR CRUZ NONE SUSPECTED NONE NONE description in_context 0 Jordan Hayes is portrayed as skeptical of Tayl... True 1 Jordan Hayes is providing analytical insights ... True 2 Jordan Hayes identified a suspicious panel tha... True 3 Taylor Cruz's leadership style is described as... True 4 Taylor Cruz is asserting command over the miss... True
Question Generation¶
This function takes a list of user queries and generates the next candidate questions.
In [20]:
Copied!
question_generator = LocalQuestionGen(
model=chat_model,
context_builder=context_builder,
token_encoder=token_encoder,
model_params=model_params,
context_builder_params=local_context_params,
)
question_generator = LocalQuestionGen(
model=chat_model,
context_builder=context_builder,
token_encoder=token_encoder,
model_params=model_params,
context_builder_params=local_context_params,
)
In [21]:
Copied!
question_history = [
"Tell me about Agent Mercer",
"What happens in Dulce military base?",
]
candidate_questions = await question_generator.agenerate(
question_history=question_history, context_data=None, question_count=5
)
print(candidate_questions.response)
question_history = [
"Tell me about Agent Mercer",
"What happens in Dulce military base?",
]
candidate_questions = await question_generator.agenerate(
question_history=question_history, context_data=None, question_count=5
)
print(candidate_questions.response)
Reached token limit - reverting to previous context state
['- What is the role of Agent Alex Mercer in the Paranormal Military Squad?', '- How does Agent Mercer interact with other members of the team during Operation: Dulce?', '- What are the key responsibilities of Agent Mercer in exploring the Dulce base?', "- How does Agent Mercer's leadership style impact the success of Operation: Dulce?", '- What is the relationship between Agent Mercer and Sam Rivera within the Paranormal Military Squad?']