Local Search

In [1]:

# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License.

# Copyright (c) 2024 Microsoft Corporation. # Licensed under the MIT License.

In [2]:

import os

import pandas as pd
import tiktoken

from graphrag.query.context_builder.entity_extraction import EntityVectorStoreKey
from graphrag.query.indexer_adapters import (
    read_indexer_covariates,
    read_indexer_entities,
    read_indexer_relationships,
    read_indexer_reports,
    read_indexer_text_units,
)
from graphrag.query.question_gen.local_gen import LocalQuestionGen
from graphrag.query.structured_search.local_search.mixed_context import (
    LocalSearchMixedContext,
)
from graphrag.query.structured_search.local_search.search import LocalSearch
from graphrag.vector_stores.lancedb import LanceDBVectorStore

import os import pandas as pd import tiktoken from graphrag.query.context_builder.entity_extraction import EntityVectorStoreKey from graphrag.query.indexer_adapters import ( read_indexer_covariates, read_indexer_entities, read_indexer_relationships, read_indexer_reports, read_indexer_text_units, ) from graphrag.query.question_gen.local_gen import LocalQuestionGen from graphrag.query.structured_search.local_search.mixed_context import ( LocalSearchMixedContext, ) from graphrag.query.structured_search.local_search.search import LocalSearch from graphrag.vector_stores.lancedb import LanceDBVectorStore

Local Search Example¶

Local search method generates answers by combining relevant data from the AI-extracted knowledge-graph with text chunks of the raw documents. This method is suitable for questions that require an understanding of specific entities mentioned in the documents (e.g. What are the healing properties of chamomile?).

Load text units and graph data tables as context for local search¶

• In this test we first load indexing outputs from parquet files to dataframes, then convert these dataframes into collections of data objects aligning with the knowledge model.

Load tables to dataframes¶

In [3]:

INPUT_DIR = "./inputs/operation dulce"
LANCEDB_URI = f"{INPUT_DIR}/lancedb"

COMMUNITY_REPORT_TABLE = "community_reports"
ENTITY_TABLE = "entities"
COMMUNITY_TABLE = "communities"
RELATIONSHIP_TABLE = "relationships"
COVARIATE_TABLE = "covariates"
TEXT_UNIT_TABLE = "text_units"
COMMUNITY_LEVEL = 2

INPUT_DIR = "./inputs/operation dulce" LANCEDB_URI = f"{INPUT_DIR}/lancedb" COMMUNITY_REPORT_TABLE = "community_reports" ENTITY_TABLE = "entities" COMMUNITY_TABLE = "communities" RELATIONSHIP_TABLE = "relationships" COVARIATE_TABLE = "covariates" TEXT_UNIT_TABLE = "text_units" COMMUNITY_LEVEL = 2

Read entities¶

In [4]:

# read nodes table to get community and degree data
entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet")
community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet")

entities = read_indexer_entities(entity_df, community_df, COMMUNITY_LEVEL)

# load description embeddings to an in-memory lancedb vectorstore
# to connect to a remote db, specify url and port values.
description_embedding_store = LanceDBVectorStore(
    collection_name="default-entity-description",
)
description_embedding_store.connect(db_uri=LANCEDB_URI)

print(f"Entity count: {len(entity_df)}")
entity_df.head()

# read nodes table to get community and degree data entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet") community_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet") entities = read_indexer_entities(entity_df, community_df, COMMUNITY_LEVEL) # load description embeddings to an in-memory lancedb vectorstore # to connect to a remote db, specify url and port values. description_embedding_store = LanceDBVectorStore( collection_name="default-entity-description", ) description_embedding_store.connect(db_uri=LANCEDB_URI) print(f"Entity count: {len(entity_df)}") entity_df.head()

Entity count: 18

Out[4]:

	id	human_readable_id	title	type	description	text_unit_ids	frequency	degree	x	y
0	425a7862-0aef-4f69-a4c8-8bd42151c9d4	0	ALEX MERCER	PERSON	Agent Alex Mercer is a determined individual w...	[8e938693af886bfd081acbbe8384c3671446bff84a134...	4	9	0	0
1	bcdbf1fc-0dc1-460f-bc71-2781729c96ba	1	TAYLOR CRUZ	PERSON	Agent Taylor Cruz is a commanding and authorit...	[8e938693af886bfd081acbbe8384c3671446bff84a134...	4	8	0	0
2	ef02ef24-5762-46ce-93ce-7dea6fc86595	2	JORDAN HAYES	PERSON	Dr. Jordan Hayes is a scientist and a member o...	[8e938693af886bfd081acbbe8384c3671446bff84a134...	4	9	0	0
3	8b163d27-e43a-4a2c-a26f-866778d8720e	3	SAM RIVERA	PERSON	Sam Rivera is a cybersecurity expert and a tal...	[8e938693af886bfd081acbbe8384c3671446bff84a134...	4	8	0	0
4	542aa5bd-ba2d-400a-8488-c52d50bc300d	4	PARANORMAL MILITARY SQUAD	ORGANIZATION	The PARANORMAL MILITARY SQUAD is an elite grou...	[8e938693af886bfd081acbbe8384c3671446bff84a134...	2	6	0	0

Read relationships¶

In [5]:

relationship_df = pd.read_parquet(f"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet")
relationships = read_indexer_relationships(relationship_df)

print(f"Relationship count: {len(relationship_df)}")
relationship_df.head()

relationship_df = pd.read_parquet(f"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet") relationships = read_indexer_relationships(relationship_df) print(f"Relationship count: {len(relationship_df)}") relationship_df.head()

Relationship count: 54

Out[5]:

	id	human_readable_id	source	target	description	weight	combined_degree	text_unit_ids
0	2bfad9f4-5abd-48d0-8db3-a9cad9120413	0	ALEX MERCER	TAYLOR CRUZ	Alex Mercer and Taylor Cruz are both agents wo...	37.0	17	[8e938693af886bfd081acbbe8384c3671446bff84a134...
1	6cbb838f-9e83-4086-a684-15c8ed709e52	1	ALEX MERCER	JORDAN HAYES	Alex Mercer and Jordan Hayes are both agents w...	42.0	18	[8e938693af886bfd081acbbe8384c3671446bff84a134...
2	bfdc25f1-80ca-477b-a304-94465b69e680	2	ALEX MERCER	SAM RIVERA	Alex Mercer and Sam Rivera are both agents and...	26.0	17	[8e938693af886bfd081acbbe8384c3671446bff84a134...
3	7a7e943d-a4f5-487b-9625-5d0907c4c26d	3	ALEX MERCER	PARANORMAL MILITARY SQUAD	Alex Mercer is a member of the Paranormal Mili...	17.0	15	[8e938693af886bfd081acbbe8384c3671446bff84a134...
4	5e00bcb9-a17e-4c27-8241-6ebb286a7fc6	4	ALEX MERCER	DULCE	Alex Mercer is preparing to lead the team into...	15.0	14	[8e938693af886bfd081acbbe8384c3671446bff84a134...

In [6]:

# NOTE: covariates are turned off by default, because they generally need prompt tuning to be valuable
# Please see the GRAPHRAG_CLAIM_* settings
covariate_df = pd.read_parquet(f"{INPUT_DIR}/{COVARIATE_TABLE}.parquet")

claims = read_indexer_covariates(covariate_df)

print(f"Claim records: {len(claims)}")
covariates = {"claims": claims}

# NOTE: covariates are turned off by default, because they generally need prompt tuning to be valuable # Please see the GRAPHRAG_CLAIM_* settings covariate_df = pd.read_parquet(f"{INPUT_DIR}/{COVARIATE_TABLE}.parquet") claims = read_indexer_covariates(covariate_df) print(f"Claim records: {len(claims)}") covariates = {"claims": claims}

Claim records: 17

Read community reports¶

In [7]:

report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet")
reports = read_indexer_reports(report_df, community_df, COMMUNITY_LEVEL)

print(f"Report records: {len(report_df)}")
report_df.head()

report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet") reports = read_indexer_reports(report_df, community_df, COMMUNITY_LEVEL) print(f"Report records: {len(report_df)}") report_df.head()

Report records: 2

Out[7]:

	id	human_readable_id	community	level	parent	children	title	summary	full_content	rank	rating_explanation	findings	full_content_json	period	size
0	6c3a555680d647ac8be866a129c7b0ea	0	0	0	-1	[]	Operation: Dulce and Dulce Base Exploration	The community revolves around 'Operation: Dulc...	# Operation: Dulce and Dulce Base Exploration\...	8.5	The impact severity rating is high due to the ...	[{'explanation': 'Operation: Dulce is a signif...	{\n "title": "Operation: Dulce and Dulce Ba...	2025-03-04	7
1	0127331a1ea34b8ba19de2c2a4cb3bc9	1	1	0	-1	[]	Paranormal Military Squad and Operation: Dulce	The community centers around the Paranormal Mi...	# Paranormal Military Squad and Operation: Dul...	8.5	The impact severity rating is high due to the ...	[{'explanation': 'Agent Alex Mercer is a key f...	{\n "title": "Paranormal Military Squad and...	2025-03-04	9

Read text units¶

In [8]:

text_unit_df = pd.read_parquet(f"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet")
text_units = read_indexer_text_units(text_unit_df)

print(f"Text unit records: {len(text_unit_df)}")
text_unit_df.head()

text_unit_df = pd.read_parquet(f"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet") text_units = read_indexer_text_units(text_unit_df) print(f"Text unit records: {len(text_unit_df)}") text_unit_df.head()

Text unit records: 5

Out[8]:

	id	human_readable_id	text	n_tokens	document_ids	entity_ids	relationship_ids	covariate_ids
0	8e938693af886bfd081acbbe8384c3671446bff84a134a...	1	# Operation: Dulce\n\n## Chapter 1\n\nThe thru...	1200	[6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8...	[425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f...	[2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838...	[745d28dd-be20-411b-85ff-1c69ca70e7b3, 9cba185...
1	fd1f46d32e1df6cd429542aeda3d64ddf3745ccb80f443...	2	, the hollow echo of the bay a stark reminder ...	1200	[6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8...	[425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f...	[2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838...	[4f9b461f-5e8f-465d-9586-e2fc81787062, 0f74618...
2	7296d9a1f046854d59079dc183de8a054c27c4843d2979...	3	differently than praise from others. This was...	1200	[6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8...	[425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f...	[2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838...	[3ef1be9c-4080-4fac-99bd-c4a636248904, 8730b20...
3	ac72722a02ac71242a2a91fca323198d04197daf60515d...	4	contrast to the rigid silence enveloping the ...	1200	[6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8...	[425a7862-0aef-4f69-a4c8-8bd42151c9d4, bcdbf1f...	[2bfad9f4-5abd-48d0-8db3-a9cad9120413, 6cbb838...	[2c292047-b79a-4958-ab57-7bf7d7a22c92, 3cbd18a...
4	4c277337d461a16aaf8f9760ddb8b44ef220e948a2341d...	5	a mask of duty.\n\nIn the midst of the descen...	35	[6e81f882f89dd5596e1925dd3ae8a4f0a0edcb55b35a8...	[d084d615-3584-4ec8-9931-90aa6075c764, 4b84859...	[6efdc42e-69a2-47c0-97ec-4b296cd16d5e]	[db8da02f-f889-4bb5-8e81-ab2a72e380bb]

In [9]:

from graphrag.config.enums import ModelType
from graphrag.config.models.language_model_config import LanguageModelConfig
from graphrag.language_model.manager import ModelManager

api_key = os.environ["GRAPHRAG_API_KEY"]
llm_model = os.environ["GRAPHRAG_LLM_MODEL"]
embedding_model = os.environ["GRAPHRAG_EMBEDDING_MODEL"]

chat_config = LanguageModelConfig(
    api_key=api_key,
    type=ModelType.OpenAIChat,
    model=llm_model,
    max_retries=20,
)
chat_model = ModelManager().get_or_create_chat_model(
    name="local_search",
    model_type=ModelType.OpenAIChat,
    config=chat_config,
)

token_encoder = tiktoken.encoding_for_model(llm_model)

embedding_config = LanguageModelConfig(
    api_key=api_key,
    type=ModelType.OpenAIEmbedding,
    model=embedding_model,
    max_retries=20,
)

text_embedder = ModelManager().get_or_create_embedding_model(
    name="local_search_embedding",
    model_type=ModelType.OpenAIEmbedding,
    config=embedding_config,
)

from graphrag.config.enums import ModelType from graphrag.config.models.language_model_config import LanguageModelConfig from graphrag.language_model.manager import ModelManager api_key = os.environ["GRAPHRAG_API_KEY"] llm_model = os.environ["GRAPHRAG_LLM_MODEL"] embedding_model = os.environ["GRAPHRAG_EMBEDDING_MODEL"] chat_config = LanguageModelConfig( api_key=api_key, type=ModelType.OpenAIChat, model=llm_model, max_retries=20, ) chat_model = ModelManager().get_or_create_chat_model( name="local_search", model_type=ModelType.OpenAIChat, config=chat_config, ) token_encoder = tiktoken.encoding_for_model(llm_model) embedding_config = LanguageModelConfig( api_key=api_key, type=ModelType.OpenAIEmbedding, model=embedding_model, max_retries=20, ) text_embedder = ModelManager().get_or_create_embedding_model( name="local_search_embedding", model_type=ModelType.OpenAIEmbedding, config=embedding_config, )

Create local search context builder¶

In [10]:

context_builder = LocalSearchMixedContext(
    community_reports=reports,
    text_units=text_units,
    entities=entities,
    relationships=relationships,
    # if you did not run covariates during indexing, set this to None
    covariates=covariates,
    entity_text_embeddings=description_embedding_store,
    embedding_vectorstore_key=EntityVectorStoreKey.ID,  # if the vectorstore uses entity title as ids, set this to EntityVectorStoreKey.TITLE
    text_embedder=text_embedder,
    token_encoder=token_encoder,
)

context_builder = LocalSearchMixedContext( community_reports=reports, text_units=text_units, entities=entities, relationships=relationships, # if you did not run covariates during indexing, set this to None covariates=covariates, entity_text_embeddings=description_embedding_store, embedding_vectorstore_key=EntityVectorStoreKey.ID, # if the vectorstore uses entity title as ids, set this to EntityVectorStoreKey.TITLE text_embedder=text_embedder, token_encoder=token_encoder, )

Create local search engine¶

In [11]:

# text_unit_prop: proportion of context window dedicated to related text units
# community_prop: proportion of context window dedicated to community reports.
# The remaining proportion is dedicated to entities and relationships. Sum of text_unit_prop and community_prop should be <= 1
# conversation_history_max_turns: maximum number of turns to include in the conversation history.
# conversation_history_user_turns_only: if True, only include user queries in the conversation history.
# top_k_mapped_entities: number of related entities to retrieve from the entity description embedding store.
# top_k_relationships: control the number of out-of-network relationships to pull into the context window.
# include_entity_rank: if True, include the entity rank in the entity table in the context window. Default entity rank = node degree.
# include_relationship_weight: if True, include the relationship weight in the context window.
# include_community_rank: if True, include the community rank in the context window.
# return_candidate_context: if True, return a set of dataframes containing all candidate entity/relationship/covariate records that
# could be relevant. Note that not all of these records will be included in the context window. The "in_context" column in these
# dataframes indicates whether the record is included in the context window.
# max_tokens: maximum number of tokens to use for the context window.


local_context_params = {
    "text_unit_prop": 0.5,
    "community_prop": 0.1,
    "conversation_history_max_turns": 5,
    "conversation_history_user_turns_only": True,
    "top_k_mapped_entities": 10,
    "top_k_relationships": 10,
    "include_entity_rank": True,
    "include_relationship_weight": True,
    "include_community_rank": False,
    "return_candidate_context": False,
    "embedding_vectorstore_key": EntityVectorStoreKey.ID,  # set this to EntityVectorStoreKey.TITLE if the vectorstore uses entity title as ids
    "max_tokens": 12_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
}

model_params = {
    "max_tokens": 2_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000=1500)
    "temperature": 0.0,
}

# text_unit_prop: proportion of context window dedicated to related text units # community_prop: proportion of context window dedicated to community reports. # The remaining proportion is dedicated to entities and relationships. Sum of text_unit_prop and community_prop should be <= 1 # conversation_history_max_turns: maximum number of turns to include in the conversation history. # conversation_history_user_turns_only: if True, only include user queries in the conversation history. # top_k_mapped_entities: number of related entities to retrieve from the entity description embedding store. # top_k_relationships: control the number of out-of-network relationships to pull into the context window. # include_entity_rank: if True, include the entity rank in the entity table in the context window. Default entity rank = node degree. # include_relationship_weight: if True, include the relationship weight in the context window. # include_community_rank: if True, include the community rank in the context window. # return_candidate_context: if True, return a set of dataframes containing all candidate entity/relationship/covariate records that # could be relevant. Note that not all of these records will be included in the context window. The "in_context" column in these # dataframes indicates whether the record is included in the context window. # max_tokens: maximum number of tokens to use for the context window. local_context_params = { "text_unit_prop": 0.5, "community_prop": 0.1, "conversation_history_max_turns": 5, "conversation_history_user_turns_only": True, "top_k_mapped_entities": 10, "top_k_relationships": 10, "include_entity_rank": True, "include_relationship_weight": True, "include_community_rank": False, "return_candidate_context": False, "embedding_vectorstore_key": EntityVectorStoreKey.ID, # set this to EntityVectorStoreKey.TITLE if the vectorstore uses entity title as ids "max_tokens": 12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000) } model_params = { "max_tokens": 2_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000=1500) "temperature": 0.0, }

In [12]:

search_engine = LocalSearch(
    model=chat_model,
    context_builder=context_builder,
    token_encoder=token_encoder,
    model_params=model_params,
    context_builder_params=local_context_params,
    response_type="multiple paragraphs",  # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report
)

search_engine = LocalSearch( model=chat_model, context_builder=context_builder, token_encoder=token_encoder, model_params=model_params, context_builder_params=local_context_params, response_type="multiple paragraphs", # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report )

Run local search on sample queries¶

In [13]:

result = await search_engine.search("Tell me about Agent Mercer")
print(result.response)

result = await search_engine.search("Tell me about Agent Mercer") print(result.response)

### Overview of Agent Alex Mercer

Agent Alex Mercer is a pivotal member of the Paranormal Military Squad, a specialized team tasked with executing Operation: Dulce. His role is crucial to the mission's success, as he is one of the agents exploring the Dulce base, a mysterious and secretive location associated with advanced alien technology [Data: Reports (1); Entities (0, 8); Relationships (23, 24, 3)].

### Leadership and Mentorship

Alex Mercer is recognized for his leadership qualities and serves as a mentor to Sam Rivera, another key member of the squad. He provides guidance and emphasizes the importance of intuition and trust, which are essential traits for navigating the complexities of Operation: Dulce. His mentorship is characterized by a supportive and encouraging approach, helping Rivera develop their skills and confidence [Data: Reports (1); Entities (0, 3); Relationships (2, 15)].

### Professional Relationships

Mercer maintains professional relationships with other team members, including Taylor Cruz and Jordan Hayes. His interactions with Cruz are marked by a competitive undercurrent, as Cruz's authoritative nature often challenges Mercer's compliance. Despite this, Mercer acknowledges Cruz's authority and follows their lead during the mission. With Jordan Hayes, Mercer shares a mutual respect and understanding, particularly admiring each other's expertise and analytical abilities. Their collaboration is vital to the exploration of the Dulce base and the success of Operation: Dulce [Data: Reports (1); Entities (0, 1, 2); Relationships (0, 1)].

### Internal Conflict and Mission Involvement

Agent Mercer experiences an internal conflict between adhering to protocols and his natural inclination to question and explore all details. This conflict is highlighted through his interactions with Taylor Cruz and his own reflections. Despite this, Mercer is committed to the mission and plays a significant role in leading the team into the Dulce base, where he aims to navigate and uncover its secrets [Data: Claims (3, 5); Relationships (4)].

In summary, Agent Alex Mercer is a key figure in the Paranormal Military Squad, known for his leadership, mentorship, and professional relationships. His involvement in Operation: Dulce is marked by a balance of compliance with protocols and a drive to explore the unknown, making him an indispensable asset to the mission [Data: Reports (1); Entities (0); Relationships (3, 24)].

In [14]:

question = "Tell me about Dr. Jordan Hayes"
result = await search_engine.search(question)
print(result.response)

question = "Tell me about Dr. Jordan Hayes" result = await search_engine.search(question) print(result.response)

### Overview of Dr. Jordan Hayes

Dr. Jordan Hayes is a prominent scientist and a key member of the Paranormal Military Squad, known for their expertise in physics and composed demeanor. They play a significant role in Operation: Dulce, particularly in working with alien technology, which is a central element of the mission [Data: Entities (2); Reports (1)]. Dr. Hayes is recognized for their analytical mind and reflective nature, often contemplating the complexities of their missions and the layers of data they encounter [Data: Entities (2); Reports (1)].

### Role in Operation: Dulce

In Operation: Dulce, Dr. Hayes is involved in exploring the Dulce base and working on alien technology. Their role is crucial as they provide analytical insights and express concerns about the mission, indicating their involvement in the analytical assessment of the operation [Data: Reports (1); Claims (6)]. Dr. Hayes identified a suspicious panel within the base, suggesting a hidden element, which highlights their keen observational skills and attention to detail [Data: Claims (10)].

### Professional Relationships

Dr. Hayes maintains professional relationships with other key members of the Paranormal Military Squad, including Taylor Cruz and Sam Rivera. Their relationship with Taylor Cruz is marked by differing views on protocol and adaptability, with Dr. Hayes advocating for adaptability and acknowledging the unknown variables that exceed the known [Data: Relationships (5, 25); Claims (2)]. They share a common belief in the importance of adaptability with Sam Rivera, emphasizing the need for flexibility in the mission [Data: Relationships (9)].

### Analytical and Skeptical Nature

Dr. Hayes is portrayed as skeptical of strict adherence to protocols, emphasizing the need for adaptability in the face of unknown variables. This skepticism is evident in their interactions and comments during briefings, where they advocate for a more flexible approach to the mission [Data: Claims (2)]. Their analytical nature is further demonstrated by their work in the lab on alien technology, where they explore the potential implications of their findings for humanity [Data: Entities (13); Reports (1)].

In summary, Dr. Jordan Hayes is a critical member of the Paranormal Military Squad, contributing significantly to Operation: Dulce through their expertise in physics and alien technology. Their analytical insights and emphasis on adaptability make them a valuable asset to the team, as they navigate the complexities of the mission and the unknown challenges it presents.

Inspecting the context data used to generate the response¶

In [15]:

result.context_data["entities"].head()

result.context_data["entities"].head()

Out[15]:

	id	entity	description	number of relationships	in_context
0	2	JORDAN HAYES	Dr. Jordan Hayes is a scientist and a member o...	9	True
1	13	LAB	A lab where Dr. Jordan Hayes works on alien te...	1	True
2	10	AGENT HAYES	Agent Hayes is a member of the team exploring ...	5	True
3	12	AGENT CRUZ		5	True
4	15	BRIEFING ROOM		2	True

In [16]:

result.context_data["relationships"].head()

result.context_data["relationships"].head()

Out[16]:

	id	source	target	description	weight	links	in_context
0	24	ALEX MERCER	OPERATION: DULCE	Alex Mercer is involved in and is part of Oper...	24.0	10	True
1	27	JORDAN HAYES	OPERATION: DULCE	Jordan Hayes is part of Operation: Dulce	8.0	10	True
2	48	OPERATION: DULCE	JORDAN HAYES	Jordan Hayes is working on alien technology as...	16.0	10	True
3	1	ALEX MERCER	JORDAN HAYES	Alex Mercer and Jordan Hayes are both agents w...	42.0	7	True
4	29	TAYLOR CRUZ	OPERATION: DULCE	Taylor Cruz is part of Operation: Dulce	8.0	10	True

In [17]:

if "reports" in result.context_data:
    result.context_data["reports"].head()

if "reports" in result.context_data: result.context_data["reports"].head()

In [18]:

result.context_data["sources"].head()

result.context_data["sources"].head()

Out[18]:

	id	text
0	0	# Operation: Dulce\n\n## Chapter 1\n\nThe thru...
1	2	differently than praise from others. This was...
2	3	contrast to the rigid silence enveloping the ...
3	1	, the hollow echo of the bay a stark reminder ...
4	4	a mask of duty.\n\nIn the midst of the descen...

In [19]:

if "claims" in result.context_data:
    print(result.context_data["claims"].head())

if "claims" in result.context_data: print(result.context_data["claims"].head())

   id        entity object_id     status start_date end_date  \
0   2  JORDAN HAYES      NONE  SUSPECTED       NONE     NONE   
1   6  JORDAN HAYES      NONE  SUSPECTED       NONE     NONE   
2  10  JORDAN HAYES      NONE       TRUE       NONE     NONE   
3   1   TAYLOR CRUZ      NONE  SUSPECTED       NONE     NONE   
4   7   TAYLOR CRUZ      NONE  SUSPECTED       NONE     NONE   

                                         description  in_context  
0  Jordan Hayes is portrayed as skeptical of Tayl...        True  
1  Jordan Hayes is providing analytical insights ...        True  
2  Jordan Hayes identified a suspicious panel tha...        True  
3  Taylor Cruz's leadership style is described as...        True  
4  Taylor Cruz is asserting command over the miss...        True  

Question Generation¶

This function takes a list of user queries and generates the next candidate questions.

In [20]:

question_generator = LocalQuestionGen(
    model=chat_model,
    context_builder=context_builder,
    token_encoder=token_encoder,
    model_params=model_params,
    context_builder_params=local_context_params,
)

question_generator = LocalQuestionGen( model=chat_model, context_builder=context_builder, token_encoder=token_encoder, model_params=model_params, context_builder_params=local_context_params, )

In [21]:

question_history = [
    "Tell me about Agent Mercer",
    "What happens in Dulce military base?",
]
candidate_questions = await question_generator.agenerate(
    question_history=question_history, context_data=None, question_count=5
)
print(candidate_questions.response)

question_history = [ "Tell me about Agent Mercer", "What happens in Dulce military base?", ] candidate_questions = await question_generator.agenerate( question_history=question_history, context_data=None, question_count=5 ) print(candidate_questions.response)

['- What is the role of Agent Alex Mercer in the exploration of the Dulce base?', "- How does Agent Taylor Cruz's leadership style impact the mission at the Dulce base?", '- What expertise does Dr. Jordan Hayes bring to the team exploring the Dulce base?', '- How does Sam Rivera contribute to the mission at the Dulce base with their cybersecurity skills?', '- What are the main objectives of Operation: Dulce at the Dulce base?']