Local Search

In [1]:

# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License.

# Copyright (c) 2024 Microsoft Corporation. # Licensed under the MIT License.

In [2]:

import os

import pandas as pd
import tiktoken

from graphrag.query.context_builder.entity_extraction import EntityVectorStoreKey
from graphrag.query.indexer_adapters import (
    read_indexer_covariates,
    read_indexer_entities,
    read_indexer_relationships,
    read_indexer_reports,
    read_indexer_text_units,
)
from graphrag.query.input.loaders.dfs import (
    store_entity_semantic_embeddings,
)
from graphrag.query.llm.oai.chat_openai import ChatOpenAI
from graphrag.query.llm.oai.embedding import OpenAIEmbedding
from graphrag.query.llm.oai.typing import OpenaiApiType
from graphrag.query.question_gen.local_gen import LocalQuestionGen
from graphrag.query.structured_search.local_search.mixed_context import (
    LocalSearchMixedContext,
)
from graphrag.query.structured_search.local_search.search import LocalSearch
from graphrag.vector_stores.lancedb import LanceDBVectorStore

import os import pandas as pd import tiktoken from graphrag.query.context_builder.entity_extraction import EntityVectorStoreKey from graphrag.query.indexer_adapters import ( read_indexer_covariates, read_indexer_entities, read_indexer_relationships, read_indexer_reports, read_indexer_text_units, ) from graphrag.query.input.loaders.dfs import ( store_entity_semantic_embeddings, ) from graphrag.query.llm.oai.chat_openai import ChatOpenAI from graphrag.query.llm.oai.embedding import OpenAIEmbedding from graphrag.query.llm.oai.typing import OpenaiApiType from graphrag.query.question_gen.local_gen import LocalQuestionGen from graphrag.query.structured_search.local_search.mixed_context import ( LocalSearchMixedContext, ) from graphrag.query.structured_search.local_search.search import LocalSearch from graphrag.vector_stores.lancedb import LanceDBVectorStore

Local Search Example¶

Local search method generates answers by combining relevant data from the AI-extracted knowledge-graph with text chunks of the raw documents. This method is suitable for questions that require an understanding of specific entities mentioned in the documents (e.g. What are the healing properties of chamomile?).

Load text units and graph data tables as context for local search¶

• In this test we first load indexing outputs from parquet files to dataframes, then convert these dataframes into collections of data objects aligning with the knowledge model.

Load tables to dataframes¶

In [3]:

INPUT_DIR = "./inputs/operation dulce"
LANCEDB_URI = f"{INPUT_DIR}/lancedb"

COMMUNITY_REPORT_TABLE = "create_final_community_reports"
ENTITY_TABLE = "create_final_nodes"
ENTITY_EMBEDDING_TABLE = "create_final_entities"
RELATIONSHIP_TABLE = "create_final_relationships"
COVARIATE_TABLE = "create_final_covariates"
TEXT_UNIT_TABLE = "create_final_text_units"
COMMUNITY_LEVEL = 2

INPUT_DIR = "./inputs/operation dulce" LANCEDB_URI = f"{INPUT_DIR}/lancedb" COMMUNITY_REPORT_TABLE = "create_final_community_reports" ENTITY_TABLE = "create_final_nodes" ENTITY_EMBEDDING_TABLE = "create_final_entities" RELATIONSHIP_TABLE = "create_final_relationships" COVARIATE_TABLE = "create_final_covariates" TEXT_UNIT_TABLE = "create_final_text_units" COMMUNITY_LEVEL = 2

Read entities¶

In [4]:

# read nodes table to get community and degree data
entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet")
entity_embedding_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_EMBEDDING_TABLE}.parquet")

entities = read_indexer_entities(entity_df, entity_embedding_df, COMMUNITY_LEVEL)

# load description embeddings to an in-memory lancedb vectorstore
# to connect to a remote db, specify url and port values.
description_embedding_store = LanceDBVectorStore(
    collection_name="default-entity-description",
)
description_embedding_store.connect(db_uri=LANCEDB_URI)
entity_description_embeddings = store_entity_semantic_embeddings(
    entities=entities, vectorstore=description_embedding_store
)

print(f"Entity count: {len(entity_df)}")
entity_df.head()

# read nodes table to get community and degree data entity_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_TABLE}.parquet") entity_embedding_df = pd.read_parquet(f"{INPUT_DIR}/{ENTITY_EMBEDDING_TABLE}.parquet") entities = read_indexer_entities(entity_df, entity_embedding_df, COMMUNITY_LEVEL) # load description embeddings to an in-memory lancedb vectorstore # to connect to a remote db, specify url and port values. description_embedding_store = LanceDBVectorStore( collection_name="default-entity-description", ) description_embedding_store.connect(db_uri=LANCEDB_URI) entity_description_embeddings = store_entity_semantic_embeddings( entities=entities, vectorstore=description_embedding_store ) print(f"Entity count: {len(entity_df)}") entity_df.head()

Entity count: 178

Out[4]:

	level	title	type	description	source_id	community	degree	human_readable_id	id	size	graph_embedding	top_level_node_id	x	y
0	0	ALEX MERCER	PERSON	Alex Mercer is a commanding and strategic figu...	06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...	3	57	0	b45241d70f0e43fca764df95b2b81f77	57	None	b45241d70f0e43fca764df95b2b81f77	0	0
1	0	TAYLOR CRUZ	PERSON	Taylor Cruz is a central and commanding figure...	06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...	3	52	1	4119fd06010c494caa07f439b333f4c5	52	None	4119fd06010c494caa07f439b333f4c5	0	0
2	0	JORDAN HAYES	PERSON	Dr. Jordan Hayes is a central figure at Dulce ...	06bdff339c02ab35c80fa49320d5da66,1f2c6c263f168...	3	48	2	d3835bf3dda84ead99deadbeac5d0d7d	48	None	d3835bf3dda84ead99deadbeac5d0d7d	0	0
3	0	SAM RIVERA	PERSON	Sam Rivera is a key operator and technologist ...	06bdff339c02ab35c80fa49320d5da66,1b603cdb97651...	3	48	3	077d2820ae1845bcbb1803379a3d1eae	48	None	077d2820ae1845bcbb1803379a3d1eae	0	0
4	0	PARANORMAL MILITARY SQUAD	ORGANIZATION	The Paranormal Military Squad is a specialized...	06bdff339c02ab35c80fa49320d5da66,2db9206de77cf...	3	48	4	3671ea0dd4e84c1a9b02c5ab2c8f4bac	48	None	3671ea0dd4e84c1a9b02c5ab2c8f4bac	0	0

Read relationships¶

In [5]:

relationship_df = pd.read_parquet(f"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet")
relationships = read_indexer_relationships(relationship_df)

print(f"Relationship count: {len(relationship_df)}")
relationship_df.head()

relationship_df = pd.read_parquet(f"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet") relationships = read_indexer_relationships(relationship_df) print(f"Relationship count: {len(relationship_df)}") relationship_df.head()

Relationship count: 373

Out[5]:

	source	target	weight	description	text_unit_ids	id	human_readable_id	source_degree	target_degree	rank
0	ALEX MERCER	TAYLOR CRUZ	21.0	Alex Mercer and Taylor Cruz are integral membe...	[06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...	148fffeb994541b2b4b6dcefda7001a8	0	57	52	109
1	ALEX MERCER	JORDAN HAYES	25.0	Alex Mercer and Jordan Hayes are integral team...	[06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...	89c08e793298442686292454a1abff31	1	57	48	105
2	ALEX MERCER	SAM RIVERA	20.0	Alex Mercer and Sam Rivera are integral member...	[06bdff339c02ab35c80fa49320d5da66, 1f2c6c263f1...	0467928aa65e4a4fba62bdb1467e3a54	2	57	48	105
3	ALEX MERCER	OPERATION: DULCE BRIEFING ROOM	1.0	Alex Mercer was present in the Operation: Dulc...	[06bdff339c02ab35c80fa49320d5da66]	43c3390303c6476cb65f584e37c3e81c	3	57	4	61
4	ALEX MERCER	PARANORMAL MILITARY SQUAD	20.0	Alex Mercer is a prominent and influential lea...	[06bdff339c02ab35c80fa49320d5da66, 2db9206de77...	fa14b16c17e3417dba5a4b473ea5b18d	4	57	48	105

In [6]:

# NOTE: covariates are turned off by default, because they generally need prompt tuning to be valuable
# Please see the GRAPHRAG_CLAIM_* settings
covariate_df = pd.read_parquet(f"{INPUT_DIR}/{COVARIATE_TABLE}.parquet")

claims = read_indexer_covariates(covariate_df)

print(f"Claim records: {len(claims)}")
covariates = {"claims": claims}

# NOTE: covariates are turned off by default, because they generally need prompt tuning to be valuable # Please see the GRAPHRAG_CLAIM_* settings covariate_df = pd.read_parquet(f"{INPUT_DIR}/{COVARIATE_TABLE}.parquet") claims = read_indexer_covariates(covariate_df) print(f"Claim records: {len(claims)}") covariates = {"claims": claims}

Claim records: 156

Read community reports¶

In [7]:

report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet")
reports = read_indexer_reports(report_df, entity_df, COMMUNITY_LEVEL)

print(f"Report records: {len(report_df)}")
report_df.head()

report_df = pd.read_parquet(f"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet") reports = read_indexer_reports(report_df, entity_df, COMMUNITY_LEVEL) print(f"Report records: {len(report_df)}") report_df.head()

Report records: 20

Out[7]:

	community	full_content	level	rank	title	rank_explanation	summary	findings	full_content_json	id
0	10	# Paranormal Military Squad at Dulce Base: Dec...	1	8.5	Paranormal Military Squad at Dulce Base: Decod...	The impact severity rating is high due to the ...	The Paranormal Military Squad, stationed at Du...	[{'explanation': 'Jordan is a central figure i...	{\n "title": "Paranormal Military Squad at ...	1ba2d200-dd26-4693-affe-a5539d0a0e0d
1	11	# Dulce and Paranormal Military Squad Operatio...	1	8.5	Dulce and Paranormal Military Squad Operations	The impact severity rating is high due to the ...	The community centers around Dulce, a secretiv...	[{'explanation': 'Dulce is described as a top-...	{\n "title": "Dulce and Paranormal Military...	a8a530b0-ae6b-44ea-b11c-9f70d138298d
2	12	# Paranormal Military Squad and Dulce Base Ope...	1	7.5	Paranormal Military Squad and Dulce Base Opera...	The impact severity rating is relatively high ...	The community centers around the Paranormal Mi...	[{'explanation': 'Taylor is a central figure w...	{\n "title": "Paranormal Military Squad and...	0478975b-c805-4cc1-b746-82f3e689e2f3
3	13	# Mission Dynamics and Leadership: Cruz and Wa...	1	7.5	Mission Dynamics and Leadership: Cruz and Wash...	The impact severity rating is relatively high ...	This report explores the intricate dynamics of...	[{'explanation': 'Cruz is a central figure in ...	{\n "title": "Mission Dynamics and Leadersh...	b56f6e68-3951-4f07-8760-63700944a375
4	14	# Dulce Base and Paranormal Military Squad: Br...	1	8.5	Dulce Base and Paranormal Military Squad: Brid...	The impact severity rating is high due to the ...	The community centers around the Dulce Base, a...	[{'explanation': 'Sam Rivera, a member of the ...	{\n "title": "Dulce Base and Paranormal Mil...	736e7006-d050-4abb-a122-00febf3f540f

Read text units¶

In [8]:

text_unit_df = pd.read_parquet(f"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet")
text_units = read_indexer_text_units(text_unit_df)

print(f"Text unit records: {len(text_unit_df)}")
text_unit_df.head()

text_unit_df = pd.read_parquet(f"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet") text_units = read_indexer_text_units(text_unit_df) print(f"Text unit records: {len(text_unit_df)}") text_unit_df.head()

Text unit records: 50

Out[8]:

	id	text	n_tokens	document_ids	entity_ids	relationship_ids	covariate_ids
0	06bdff339c02ab35c80fa49320d5da66	# Operation: Dulce\n\n## Chapter 1\n\nThe thru...	600	[958fdd043f17ade63cb13570b59df295]	[b45241d70f0e43fca764df95b2b81f77, 4119fd06010...	[148fffeb994541b2b4b6dcefda7001a8, 89c08e79329...	[439081a3-bfeb-4693-968a-0f6189d8fa50, 3399e3d...
1	28dc4aa41a3e99deb7f354682f3e9904	's authoritarian performance. _Protocols_, Jor...	600	[958fdd043f17ade63cb13570b59df295]	[4119fd06010c494caa07f439b333f4c5, 077d2820ae1...	[9a6f414210e14841a5b0e661aedc898d, db541b72609...	[b09594cb-d4b7-4de4-a1af-97778300eb1b, f7c5ea4...
2	813db3138ef511c34be86f841f68aa8f	, rehearsing the speech for their subordinates...	600	[958fdd043f17ade63cb13570b59df295]	[b45241d70f0e43fca764df95b2b81f77, 077d2820ae1...	[0467928aa65e4a4fba62bdb1467e3a54, 7cc3356d38d...	[7642fe21-7cb9-4428-848b-d2e3f5ab10ca, 6fe3d6c...
3	d0afd106abf3a2966ff88a99eef710db	, weighing his words carefully. "Our tech is t...	600	[958fdd043f17ade63cb13570b59df295]	[b45241d70f0e43fca764df95b2b81f77, 4119fd06010...	[148fffeb994541b2b4b6dcefda7001a8, 89c08e79329...	[956e4c22-e343-4b5d-ad3c-d44ca3ce5fb5]
4	f4c7c95d7bf621c6eb73d331cbf0b608	stepping into their exchange. The authority i...	600	[958fdd043f17ade63cb13570b59df295]	[077d2820ae1845bcbb1803379a3d1eae, 1fd3fa8bb5a...	[478e4c72d8fb46dd8cc9f0691c9878fd, 82b0446e7c9...	[a54eda4d-b5bf-471d-989f-370eb9aff961, 9f85274...

In [9]:

api_key = os.environ["GRAPHRAG_API_KEY"]
llm_model = os.environ["GRAPHRAG_LLM_MODEL"]
embedding_model = os.environ["GRAPHRAG_EMBEDDING_MODEL"]

llm = ChatOpenAI(
    api_key=api_key,
    model=llm_model,
    api_type=OpenaiApiType.OpenAI,  # OpenaiApiType.OpenAI or OpenaiApiType.AzureOpenAI
    max_retries=20,
)

token_encoder = tiktoken.get_encoding("cl100k_base")

text_embedder = OpenAIEmbedding(
    api_key=api_key,
    api_base=None,
    api_type=OpenaiApiType.OpenAI,
    model=embedding_model,
    deployment_name=embedding_model,
    max_retries=20,
)

api_key = os.environ["GRAPHRAG_API_KEY"] llm_model = os.environ["GRAPHRAG_LLM_MODEL"] embedding_model = os.environ["GRAPHRAG_EMBEDDING_MODEL"] llm = ChatOpenAI( api_key=api_key, model=llm_model, api_type=OpenaiApiType.OpenAI, # OpenaiApiType.OpenAI or OpenaiApiType.AzureOpenAI max_retries=20, ) token_encoder = tiktoken.get_encoding("cl100k_base") text_embedder = OpenAIEmbedding( api_key=api_key, api_base=None, api_type=OpenaiApiType.OpenAI, model=embedding_model, deployment_name=embedding_model, max_retries=20, )

Create local search context builder¶

In [10]:

context_builder = LocalSearchMixedContext(
    community_reports=reports,
    text_units=text_units,
    entities=entities,
    relationships=relationships,
    # if you did not run covariates during indexing, set this to None
    covariates=covariates,
    entity_text_embeddings=description_embedding_store,
    embedding_vectorstore_key=EntityVectorStoreKey.ID,  # if the vectorstore uses entity title as ids, set this to EntityVectorStoreKey.TITLE
    text_embedder=text_embedder,
    token_encoder=token_encoder,
)

context_builder = LocalSearchMixedContext( community_reports=reports, text_units=text_units, entities=entities, relationships=relationships, # if you did not run covariates during indexing, set this to None covariates=covariates, entity_text_embeddings=description_embedding_store, embedding_vectorstore_key=EntityVectorStoreKey.ID, # if the vectorstore uses entity title as ids, set this to EntityVectorStoreKey.TITLE text_embedder=text_embedder, token_encoder=token_encoder, )

Create local search engine¶

In [11]:

# text_unit_prop: proportion of context window dedicated to related text units
# community_prop: proportion of context window dedicated to community reports.
# The remaining proportion is dedicated to entities and relationships. Sum of text_unit_prop and community_prop should be <= 1
# conversation_history_max_turns: maximum number of turns to include in the conversation history.
# conversation_history_user_turns_only: if True, only include user queries in the conversation history.
# top_k_mapped_entities: number of related entities to retrieve from the entity description embedding store.
# top_k_relationships: control the number of out-of-network relationships to pull into the context window.
# include_entity_rank: if True, include the entity rank in the entity table in the context window. Default entity rank = node degree.
# include_relationship_weight: if True, include the relationship weight in the context window.
# include_community_rank: if True, include the community rank in the context window.
# return_candidate_context: if True, return a set of dataframes containing all candidate entity/relationship/covariate records that
# could be relevant. Note that not all of these records will be included in the context window. The "in_context" column in these
# dataframes indicates whether the record is included in the context window.
# max_tokens: maximum number of tokens to use for the context window.


local_context_params = {
    "text_unit_prop": 0.5,
    "community_prop": 0.1,
    "conversation_history_max_turns": 5,
    "conversation_history_user_turns_only": True,
    "top_k_mapped_entities": 10,
    "top_k_relationships": 10,
    "include_entity_rank": True,
    "include_relationship_weight": True,
    "include_community_rank": False,
    "return_candidate_context": False,
    "embedding_vectorstore_key": EntityVectorStoreKey.ID,  # set this to EntityVectorStoreKey.TITLE if the vectorstore uses entity title as ids
    "max_tokens": 12_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000)
}

llm_params = {
    "max_tokens": 2_000,  # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000=1500)
    "temperature": 0.0,
}

# text_unit_prop: proportion of context window dedicated to related text units # community_prop: proportion of context window dedicated to community reports. # The remaining proportion is dedicated to entities and relationships. Sum of text_unit_prop and community_prop should be <= 1 # conversation_history_max_turns: maximum number of turns to include in the conversation history. # conversation_history_user_turns_only: if True, only include user queries in the conversation history. # top_k_mapped_entities: number of related entities to retrieve from the entity description embedding store. # top_k_relationships: control the number of out-of-network relationships to pull into the context window. # include_entity_rank: if True, include the entity rank in the entity table in the context window. Default entity rank = node degree. # include_relationship_weight: if True, include the relationship weight in the context window. # include_community_rank: if True, include the community rank in the context window. # return_candidate_context: if True, return a set of dataframes containing all candidate entity/relationship/covariate records that # could be relevant. Note that not all of these records will be included in the context window. The "in_context" column in these # dataframes indicates whether the record is included in the context window. # max_tokens: maximum number of tokens to use for the context window. local_context_params = { "text_unit_prop": 0.5, "community_prop": 0.1, "conversation_history_max_turns": 5, "conversation_history_user_turns_only": True, "top_k_mapped_entities": 10, "top_k_relationships": 10, "include_entity_rank": True, "include_relationship_weight": True, "include_community_rank": False, "return_candidate_context": False, "embedding_vectorstore_key": EntityVectorStoreKey.ID, # set this to EntityVectorStoreKey.TITLE if the vectorstore uses entity title as ids "max_tokens": 12_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 5000) } llm_params = { "max_tokens": 2_000, # change this based on the token limit you have on your model (if you are using a model with 8k limit, a good setting could be 1000=1500) "temperature": 0.0, }

In [12]:

search_engine = LocalSearch(
    llm=llm,
    context_builder=context_builder,
    token_encoder=token_encoder,
    llm_params=llm_params,
    context_builder_params=local_context_params,
    response_type="multiple paragraphs",  # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report
)

search_engine = LocalSearch( llm=llm, context_builder=context_builder, token_encoder=token_encoder, llm_params=llm_params, context_builder_params=local_context_params, response_type="multiple paragraphs", # free form text describing the response type and format, can be anything, e.g. prioritized list, single paragraph, multiple paragraphs, multiple-page report )

Run local search on sample queries¶

In [13]:

result = await search_engine.asearch("Tell me about Agent Mercer")
print(result.response)

result = await search_engine.asearch("Tell me about Agent Mercer") print(result.response)

### Overview of Agent Mercer

Agent Alex Mercer is a prominent figure within the Paranormal Military Squad, playing a crucial role in Operation: Dulce at the Dulce Base. He is recognized for his commanding and strategic presence, particularly in overseeing operations related to communication with extraterrestrial intelligence. His responsibilities include initiating broadcasts to communicate with extraterrestrial beings, decoding alien messages, and leading the team in understanding and responding to these communications [Data: Entities (0)].

### Role and Responsibilities

Mercer is deeply involved in the philosophical and strategic aspects of interstellar communication, viewing these interactions as a form of cosmic dialogue. His leadership style is characterized by a blend of determination, compliance with mission protocols, and a protective approach towards his team. He collaborates with team members like Jordan Hayes, exploring secured areas and engaging in high-stakes, secretive operations. Mercer emphasizes the importance of intuition and trust beyond protocol, particularly in his mentorship of Sam Rivera [Data: Entities (0)].

### Relationships and Influence

Agent Mercer is known for his intellectual curiosity and is a thoughtful mentor, providing belief and guidance to his team. His experiences during encounters with alien signals have led to profound changes, reinforcing his role as a key decision-maker and guardian in missions that transcend traditional boundaries. He shares a deep commitment to discovery with colleagues like Dr. Jordan Hayes, working together on alien technology during Operation: Dulce [Data: Entities (0); Relationships (336)].

### Challenges and Philosophical Approach

Despite outward compliance, Mercer harbors a subtle form of determination that resists easy submission to authority. This is evident during the briefing for Operation: Dulce, where he is observed to be unfailingly determined on paper, yet dwarfed by the enormity of the sterile briefing room. His approach to the mission is not just about adhering to parameters but understanding the broader implications for humanity [Data: Claims (1); Sources (40)].

In summary, Agent Alex Mercer is a pivotal figure in the Paranormal Military Squad, known for his strategic leadership, mentorship, and philosophical approach to interstellar communication. His role in Operation: Dulce highlights his commitment to understanding and responding to extraterrestrial intelligence, making him a key player in the team's efforts to bridge the gap between human and alien communication.

In [14]:

question = "Tell me about Dr. Jordan Hayes"
result = await search_engine.asearch(question)
print(result.response)

question = "Tell me about Dr. Jordan Hayes" result = await search_engine.asearch(question) print(result.response)

### Overview of Dr. Jordan Hayes

Dr. Jordan Hayes is a prominent scientist at Dulce Base, playing a crucial role in the Paranormal Military Squad's efforts to understand and communicate with extraterrestrial entities. Dr. Hayes's work is primarily focused on decoding and analyzing alien signals and codes, which are essential for interstellar communication and potentially interspecies interaction. This involves decrypting algorithms, interpreting cosmic signals, and analyzing the implications of alien society [Data: Entities (2, 17); Relationships (201, 322, 283)].

### Key Contributions and Expertise

Dr. Hayes is noted for their adaptability and skepticism, qualities that are essential given the uncertainties and unknown challenges of their mission. They are deeply involved in the scientific exploration aspects of Operation: Dulce, where their expertise in deciphering alien codes is crucial. Dr. Hayes's efforts are on the verge of a significant scientific breakthrough, as they lead efforts in isolating and understanding complex alien signals that resemble human cognition. Their work suggests that these signals are artificial and patterned, indicating a tandem evolution with humanity [Data: Entities (2, 17); Claims (60, 83, 91, 134)].

### Collaborative Efforts and Leadership

Dr. Hayes collaborates closely with colleagues such as Alex Mercer and Taylor Cruz, engaging in thoughtful dialogue and showing analytical thinking about the mission's uncertainties. Despite some tension with Taylor Cruz due to differing leadership styles, Dr. Hayes continues to work effectively within the team, contributing to the strategic discussions and decisions made in the briefing room. Their role in the command center at Dulce Base involves setting up lab stations, operating the mainframe, and playing a crucial role in the command center [Data: Entities (2, 17); Relationships (26, 82, 175, 270); Claims (2, 13)].

### Scientific Breakthroughs and Challenges

Dr. Hayes's work is characterized by a focus on the unknown variables and challenges beyond established protocols. They are involved in crafting humanity's responses to cosmic alignments with stars and responsive galactic signals, navigating the dark corridors of Dulce with a focus on the unknown. Their scientific endeavors are on the brink of a breakthrough, as they lead efforts in isolating and understanding complex alien signals that resemble human cognition. This work suggests that these signals are artificial and patterned, indicating a tandem evolution with humanity [Data: Entities (2, 17); Claims (60, 83, 91, 134)].

In summary, Dr. Jordan Hayes is a key figure in the scientific exploration of extraterrestrial communication, contributing significantly to the understanding and interpretation of alien messages. Their work at Dulce Base is pivotal in advancing the mission's objectives and potentially reshaping humanity's understanding of its place in the universe.

Inspecting the context data used to generate the response¶

In [15]:

result.context_data["entities"].head()

result.context_data["entities"].head()

Out[15]:

	id	entity	description	number of relationships	in_context
0	17	DR. JORDAN HAYES	Dr. Jordan Hayes is a key scientist at Dulce B...	18	True
1	7	JORDAN	Jordan Hayes is a key member of the Paranormal...	22	True
2	2	JORDAN HAYES	Dr. Jordan Hayes is a central figure at Dulce ...	48	True
3	47	DR. HAYES	Dr. Hayes is a scientist working on understand...	1	True
4	10	AGENT HAYES	Agent Hayes is known for emphasizing empowerme...	4	True

In [16]:

result.context_data["relationships"].head()

result.context_data["relationships"].head()

Out[16]:

	id	source	target	description	weight	links	in_context
0	260	JORDAN	DR. JORDAN HAYES	Dr. Jordan Hayes and Jordan refer to the same ...	1.0	1	True
1	50	ALEX MERCER	JORDAN	Alex Mercer values Jordan's analytical approac...	1.0	2	True
2	26	ALEX MERCER	DR. JORDAN HAYES	Alex Mercer and Dr. Jordan Hayes are collabora...	5.0	2	True
3	77	TAYLOR CRUZ	JORDAN	Jordan and Taylor Cruz are team members workin...	4.0	2	True
4	82	TAYLOR CRUZ	DR. JORDAN HAYES	Dr. Jordan Hayes and Taylor Cruz are colleague...	4.0	2	True

In [17]:

result.context_data["reports"].head()

result.context_data["reports"].head()

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In[17], line 1
----> 1 result.context_data["reports"].head()

KeyError: 'reports'

In [18]:

result.context_data["sources"].head()

result.context_data["sources"].head()

Out[18]:

	id	text
0	40	and the distant hum of machinery hinted at Du...
1	18	the universe.\n\nIn a symphony of clicks and ...
2	37	from across the console, their eyes not leavi...
3	19	extraterrestrial sentience that had outreache...
4	44	It's a... symphony. It's as if they're teachi...

In [19]:

if "claims" in result.context_data:
    print(result.context_data["claims"].head())

if "claims" in result.context_data: print(result.context_data["claims"].head())

   id            entity object_id status start_date end_date  \
0   2  DR. JORDAN HAYES      NONE   TRUE       NONE     NONE   
1  13  DR. JORDAN HAYES      NONE   TRUE       NONE     NONE   
2  60  DR. JORDAN HAYES      NONE   TRUE       NONE     NONE   
3  83  DR. JORDAN HAYES      NONE   TRUE       NONE     NONE   
4  91  DR. JORDAN HAYES      NONE   TRUE       NONE     NONE   

                                         description  in_context  
0  Dr. Jordan Hayes emphasizes the importance of ...        True  
1  Dr. Jordan Hayes reflected on their own skepti...        True  
2  Dr. Jordan Hayes was focused on deciphering al...        True  
3  Dr. Jordan Hayes successfully worked through t...        True  
4  Dr. Jordan Hayes led the efforts in isolating ...        True  

Question Generation¶

This function takes a list of user queries and generates the next candidate questions.

In [20]:

question_generator = LocalQuestionGen(
    llm=llm,
    context_builder=context_builder,
    token_encoder=token_encoder,
    llm_params=llm_params,
    context_builder_params=local_context_params,
)

question_generator = LocalQuestionGen( llm=llm, context_builder=context_builder, token_encoder=token_encoder, llm_params=llm_params, context_builder_params=local_context_params, )

In [21]:

question_history = [
    "Tell me about Agent Mercer",
    "What happens in Dulce military base?",
]
candidate_questions = await question_generator.agenerate(
    question_history=question_history, context_data=None, question_count=5
)
print(candidate_questions.response)

question_history = [ "Tell me about Agent Mercer", "What happens in Dulce military base?", ] candidate_questions = await question_generator.agenerate( question_history=question_history, context_data=None, question_count=5 ) print(candidate_questions.response)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[21], line 5
      1 question_history = [
      2     "Tell me about Agent Mercer",
      3     "What happens in Dulce military base?",
      4 ]
----> 5 candidate_questions = await question_generator.agenerate(
      6     question_history=question_history, context_data=None, question_count=5
      7 )
      8 print(candidate_questions.response)

File ~/work/graphrag/graphrag/graphrag/query/question_gen/local_gen.py:74, in LocalQuestionGen.agenerate(self, question_history, context_data, question_count, **kwargs)
     70     conversation_history = ConversationHistory.from_list(history)
     72 if context_data is None:
     73     # generate context data based on the question history
---> 74     context_data, context_records = self.context_builder.build_context(
     75         query=question_text,
     76         conversation_history=conversation_history,
     77         **kwargs,
     78         **self.context_builder_params,
     79     )  # type: ignore
     80 else:
     81     context_records = {"context_data": context_data}

TypeError: cannot unpack non-iterable ContextBuilderResult object