autogen.retrieve_utils

num_tokens_from_text

def num_tokens_from_text(
    text: str,
    model: str = "gpt-3.5-turbo-0613",
    return_tokens_per_name_and_message: bool = False
) -> Union[int, Tuple[int, int, int]]

Return the number of tokens used by a text.

num_tokens_from_messages

def num_tokens_from_messages(messages: dict,
                             model: str = "gpt-3.5-turbo-0613")

Return the number of tokens used by a list of messages.

split_text_to_chunks

def split_text_to_chunks(text: str,
                         max_tokens: int = 4000,
                         chunk_mode: str = "multi_lines",
                         must_break_at_empty_line: bool = True,
                         overlap: int = 10)

Split a long text into chunks of max_tokens.

split_files_to_chunks

def split_files_to_chunks(files: list,
                          max_tokens: int = 4000,
                          chunk_mode: str = "multi_lines",
                          must_break_at_empty_line: bool = True)

Split a list of files into chunks of max_tokens.

get_files_from_dir

def get_files_from_dir(dir_path: str,
                       types: list = TEXT_FORMATS,
                       recursive: bool = True)

Return a list of all the files in a given directory.

get_file_from_url

def get_file_from_url(url: str, save_path: str = None)

Download a file from a URL.

is_url

def is_url(string: str)

Return True if the string is a valid URL.

create_vector_db_from_dir

def create_vector_db_from_dir(dir_path: str,
                              max_tokens: int = 4000,
                              client: API = None,
                              db_path: str = "/tmp/chromadb.db",
                              collection_name: str = "all-my-documents",
                              get_or_create: bool = False,
                              chunk_mode: str = "multi_lines",
                              must_break_at_empty_line: bool = True,
                              embedding_model: str = "all-MiniLM-L6-v2")

Create a vector db from all the files in a given directory.

query_vector_db

def query_vector_db(
        query_texts: List[str],
        n_results: int = 10,
        client: API = None,
        db_path: str = "/tmp/chromadb.db",
        collection_name: str = "all-my-documents",
        search_string: str = "",
        embedding_model: str = "all-MiniLM-L6-v2") -> Dict[str, List[str]]

Query a vector db.