o Ag\@snddlmZddlZddlmZmZmZmZmZddl m Z m Z Gddde Z dddZ Gddde ZdS)) annotationsN)AnyListLiteralOptionalUnion)Language TextSplittercs0eZdZdZ ddfd d ZdddZZS)CharacterTextSplitterz(Splitting text that looks at characters. F separatorstris_separator_regexboolkwargsrreturnNonec s"tjdi|||_||_dS)Create a new TextSplitter.N)super__init__ _separator_is_separator_regex)selfr rr __class__ri/var/www/html/development/chatbot/venv/lib/python3.10/site-packages/langchain_text_splitters/character.pyr s zCharacterTextSplitter.__init__text List[str]cCsB|jr|jnt|j}t|||j}|jrdn|j}|||S)&Split incoming text and return chunks.)rrreescape_split_text_with_regex_keep_separator _merge_splits)rrr splitsrrrr split_texts  z CharacterTextSplitter.split_text)r F)r r rrrrrrrr rr)__name__ __module__ __qualname____doc__rr' __classcell__rrrrr s r rr r keep_separator$Union[bool, Literal['start', 'end']]rrcs|r\|rUtd|d||dkr#fddtdtddDnfd dtdtdD}tddkrB|d d7}|dkrM|d gndg|}n t||}nt|}d d|DS) N()endc g|] }||dqSr.0i_splitsrr ( z*_split_text_with_regex..rr5cr3r4rr6r9rrr;*r<cSsg|]}|dkr|qS)r r)r7srrrr;7s)r!splitrangelenlist)rr r.r&rr9rr#s " r#csZeZdZdZ   ddfdd Zd ddZd!ddZed"ddZe d#ddZ Z S)$RecursiveCharacterTextSplitterzSplitting text by recursively look at characters. Recursively tries to split by different characters to find one that works. NTF separatorsOptional[List[str]]r.r/rrrrrrc s.tjdd|i||pgd|_||_dS)rr.)r   r Nr)rr _separatorsr)rrEr.rrrrrrAs z'RecursiveCharacterTextSplitter.__init__rr rcCs&g}|d}g}t|D](\}}|jr|nt|}|dkr"|}nt||r4|}||dd}nq |jr:|nt|}t|||j} g} |jrMdn|}| D]2} || |jkra| | qQ| rp| | |} | | g} |sx| | qQ| | |} | | qQ| r| | |} | | |S)rr>r r5N) enumeraterr!r"searchr#r$_length_function _chunk_sizeappendr%extend _split_text)rrrE final_chunksr new_separatorsr8_srr& _good_splitsr? merged_text other_inforrrrPMs@         z*RecursiveCharacterTextSplitter._split_textcCs|||jS)zSplit the input text into smaller chunks based on predefined separators. Args: text (str): The input text to be split. Returns: List[str]: A list of text chunks obtained after splitting. )rPrI)rrrrrr'us z)RecursiveCharacterTextSplitter.split_textlanguagercKs||}|d|dd|S)aReturn an instance of this class based on a specific language. This method initializes the text splitter with language-specific separators. Args: language (Language): The language to configure the text splitter for. **kwargs (Any): Additional keyword arguments to customize the splitter. Returns: RecursiveCharacterTextSplitter: An instance of the text splitter configured for the specified language. T)rErNr)get_separators_for_language)clsrWrrErrr from_languages z,RecursiveCharacterTextSplitter.from_languagecCs|tjks |tjkrgdS|tjkrgdS|tjkr gdS|tjkr)gdS|tjkr2gdS|tjkr;gdS|tjkrDgdS|tj krMgdS|tj krVgd S|tj kr_gd S|tj krhgd S|tj krqgd S|tjkrzgd S|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjkrgdS|tjvrtd|dtd|dtt)a Retrieve a list of separators specific to the given language. Args: language (Language): The language for which to get the separators. Returns: List[str]: A list of separators appropriate for the specified language. ) class z void z int z float z double  if  for  while  switch  case r rGrHr )  func  var  const  type r\r]r_r`r rGrHr )r[ public protected private  static r\r]r^r_r`r rGrHr )r[rerfrgz internal z companion z fun  val rbr\r]r^z when r` else r rGrHr ) function rc let rbr[r\r]r^r_r` default r rGrHr ) enum interface z namespace rdr[rkrcrlrbr\r]r^r_r`rmr rGrHr ) rkr[r\ foreach r^ do r_r`r rGrHr ) z message z service rnz option  import z syntax r rGrHr )r[ def z def r rGrHr )z =+ z -+ z \*+ z .. * r rGrHr ) rsr[r\ unless r^r]rqz begin z rescue r rGrHr )rsz defp z defmodule z defprotocol z defmacro z defmacrop r\rtr^r`z cond z with r]rqr rGrHr ) z fn rcrlr\r^r]z loop  match rcr rGrHr )r[z object rsrirbr\r]r^rur`r rGrHr )rar[ struct rnr\r]r^rqr_r`r rGrHr ) z #{1,6} z``` z \*\*\*+ z ---+ z ___+ r rGrHr )z \\chapter{z \\section{z \\subsection{z \\subsubsection{z \\begin{enumerate}z \\begin{itemize}z \\begin{description}z \\begin{list}z \\begin{quote}z \\begin{quotation}z \\begin{verse}z \\begin{verbatim}z \\begin{align}z$$$rHr )zs