o AgQ@sddlZddlZddlZddlmZmZddlmZmZm Z m Z m Z ddl m Z ddlmZGdddeeZGdd d eZGd d d eZdS) N)ABCabstractmethod)DictIteratorOptionalTupleUnion)Document) BaseLoaderc!@s2eZdZdZdddddddd d dddd d d d ed ededeeefdededededededeeefdeeefdeeefdeedeedd f ddZ de e fddZ e defd d!Zd"edefd#d$Zd%ed&ede e fd'd(Z d/d%ed ed)eede e fd*d+Zd,edeeeffd-d.Zd S)0DedocBaseLoadera Base Loader that uses `dedoc` (https://dedoc.readthedocs.io). Loader enables extracting text, tables and attached files from the given file: * `Text` can be split by pages, `dedoc` tree nodes, textual lines (according to the `split` parameter). * `Attached files` (when with_attachments=True) are split according to the `split` parameter. For attachments, langchain Document object has an additional metadata field `type`="attachment". * `Tables` (when with_tables=True) are not split - each table corresponds to one langchain Document object. For tables, Document object has additional metadata fields `type`="table" and `text_as_html` with table HTML representation. documentTF auto_tabbyrus+eng:autoN)split with_tableswith_attachmentsrecursion_deep_attachmentspdf_with_text_layerlanguagepagesis_one_column_documentdocument_orientationneed_header_footer_analysisneed_binarizationneed_pdf_table_analysis delimiterencoding file_pathrrrrrrrrrrrrrrreturncCs~ddtD|_hd|_||jvr!td|d|jd||_||_||_|jdkr1dnd }||jd <||jd <d S) a Initialize with file path and parsing parameters. Args: file_path: path to the file for processing split: type of document splitting into parts (each part is returned separately), default value "document" "document": document text is returned as a single langchain Document object (don't split) "page": split document text into pages (works for PDF, DJVU, PPTX, PPT, ODP) "node": split document text into tree nodes (title nodes, list item nodes, raw text nodes) "line": split document text into lines with_tables: add tables to the result - each table is returned as a single langchain Document object Parameters used for document parsing via `dedoc` (https://dedoc.readthedocs.io/en/latest/parameters/parameters.html): with_attachments: enable attached files extraction recursion_deep_attachments: recursion level for attached files extraction, works only when with_attachments==True pdf_with_text_layer: type of handler for parsing PDF documents, available options ["true", "false", "tabby", "auto", "auto_tabby" (default)] language: language of the document for PDF without a textual layer and images, available options ["eng", "rus", "rus+eng" (default)], the list of languages can be extended, please see https://dedoc.readthedocs.io/en/latest/tutorials/add_new_language.html pages: page slice to define the reading range for parsing PDF documents is_one_column_document: detect number of columns for PDF without a textual layer and images, available options ["true", "false", "auto" (default)] document_orientation: fix document orientation (90, 180, 270 degrees) for PDF without a textual layer and images, available options ["auto" (default), "no_change"] need_header_footer_analysis: remove headers and footers from the output result for parsing PDF and images need_binarization: clean pages background (binarize) for PDF without a textual layer and images need_pdf_table_analysis: parse tables for PDF without a textual layer and images delimiter: column separator for CSV, TSV files encoding: encoding of TXT, CSV, TSV cSsi|] \}}|dvr||qS)>selfrr r).0keyvaluer#r#q/var/www/html/development/chatbot/venv/lib/python3.10/site-packages/langchain_community/document_loaders/dedoc.py ds z,DedocBaseLoader.__init__..>linenodepager Got $ for `split`, but should be one of ``r*treelinearstructure_typeneed_content_analysisN)localsitemsparsing_parametersvalid_split_values ValueErrorrrr )r"r rrrrrrrrrrrrrrr1r#r#r'__init__#s A   zDedocBaseLoader.__init__ccsddl}zddlm}Wn tytdw||d}d|jd_|}|j|j i|j d|id }Wdn1sDwY|j | |jd EdHdS) Lazily load documents.rN) DedocManagerzE`dedoc` package not found, please install it with `pip install dedoc`)manager_configTloggerattachments_dir)r parameters document_treer)tempfilededocr: ImportError _make_configconfigdisabledTemporaryDirectoryparser r5_split_document to_api_schemadictr)r"rAr: dedoc_managertmpdirr@r#r#r' lazy_loadws(   zDedocBaseLoader.lazy_loadcCsdS)zu Make configuration for DedocManager according to the file extension and parsing parameters. Nr#r"r#r#r'rDszDedocBaseLoader._make_config paragraphcsBdfdd|dD}|r|dd|}|S|d}|S)z1Get text (recursively) of the document tree node. csg|]}|qSr#) _json2txt)r$ subparagraphrOr#r' sz-DedocBaseLoader._json2txt.. subparagraphstext)join)r"rPsubparagraphs_textrVr#rOr'rRs zDedocBaseLoader._json2txtr@document_metadataccsZt|ddkr|dD] }|j||dEdHq dSt|di||ddVdS)z4Parse recursively document tree obtained by `dedoc`.rUrr@rYNrVmetadata page_contentr[)len_parse_subparagraphsr )r"r@rYrSr#r#r'r_s  z$DedocBaseLoader._parse_subparagraphsadditional_metadataccs|d}|r i||}|dkr#|j|ddd}t||dVn|dkrt|ddd}|d dd }d }|D]*} | dd |krM||| 7}q;t|i|d |idV| dd }|| }q;t|i|d |idVnA|d kr|dddD]} | d} t|| i|| dVqn|d kr|j|dd|dEdHn td|d|jd|jr|ddD]} || \} } t| i| dd| ddVq|dD]}|j||jddidEdHqdS)z=Split document into parts according to the `split` parameter.r[r content structure)rPr\r+rUrpage_idr)r*rZNr,r-r.tablestable)type text_as_html attachmentsrg attachment)r@rr`) rRr r_r7r6r _get_tablerIr)r"r@rr`rYrVnodesrc page_textr* line_metadatarf table_text table_htmlrjr#r#r'rIsz        zDedocBaseLoader._split_documentrfc Csd}|dD]}|D]}|ddd|dD7}|d7}q |d7}qd }|dD]>}|d 7}|D]1}dd d|dD}t|}|d 7}|d rQ|d7}|d|dd|dd|d7}q2|d7}q*|d7}||fS)z.Get text and HTML representation of the table.rdcells cs|]}|dVqdSrVNr#r$r)r#r#r' z-DedocBaseLoader._get_table..lines rQzK z csrsrtr#rur#r#r'rvrwz z z
z
)rWhtmlescape)r"rfrorowcellrp cell_textr#r#r'rks4       zDedocBaseLoader._get_tableN)__name__ __module__ __qualname____doc__strboolrintrr8rr rNrrKrDrRr_rIrrkr#r#r#r'r s         T  Kr c@seZdZdZdefddZdS)DedocFileLoaderaw DedocFileLoader document loader integration to load files using `dedoc`. The file loader automatically detects the file type (with the correct extension). The list of supported file types is gives at https://dedoc.readthedocs.io/en/latest/index.html#id1. Please see the documentation of DedocBaseLoader to get more details. Setup: Install ``dedoc`` package. .. code-block:: bash pip install -U dedoc Instantiate: .. code-block:: python from langchain_community.document_loaders import DedocFileLoader loader = DedocFileLoader( file_path="example.pdf", # split=..., # with_tables=..., # pdf_with_text_layer=..., # pages=..., # ... ) Load: .. code-block:: python docs = loader.load() print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python Some text { 'file_name': 'example.pdf', 'file_type': 'application/pdf', # ... } Lazy load: .. code-block:: python docs = [] docs_lazy = loader.lazy_load() for doc in docs_lazy: docs.append(doc) print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python Some text { 'file_name': 'example.pdf', 'file_type': 'application/pdf', # ... } r!cCs ddlm}||j|j|jdS)Nr)make_manager_config)r parsing_paramsr)dedoc.utils.langchainrr r5r)r"rr#r#r'rD`s zDedocFileLoader._make_configN)rrrrrKrDr#r#r#r'rsBrc#seZdZdZdddddddd d d dddd d d d ededededeeefdededededededeeefdeeefdeeefdeedeedd f"fddZ de e fd d!Z de fd"d#Zded ed$e deeeee efffd%d&ZZS)'DedocAPIFileLoaderaU Load files using `dedoc` API. The file loader automatically detects the file type (even with the wrong extension). By default, the loader makes a call to the locally hosted `dedoc` API. More information about `dedoc` API can be found in `dedoc` documentation: https://dedoc.readthedocs.io/en/latest/dedoc_api_usage/api.html Please see the documentation of DedocBaseLoader to get more details. Setup: You don't need to install `dedoc` library for using this loader. Instead, the `dedoc` API needs to be run. You may use Docker container for this purpose. Please see `dedoc` documentation for more details: https://dedoc.readthedocs.io/en/latest/getting_started/installation.html#install-and-run-dedoc-using-docker .. code-block:: bash docker pull dedocproject/dedoc docker run -p 1231:1231 Instantiate: .. code-block:: python from langchain_community.document_loaders import DedocAPIFileLoader loader = DedocAPIFileLoader( file_path="example.pdf", # url=..., # split=..., # with_tables=..., # pdf_with_text_layer=..., # pages=..., # ... ) Load: .. code-block:: python docs = loader.load() print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python Some text { 'file_name': 'example.pdf', 'file_type': 'application/pdf', # ... } Lazy load: .. code-block:: python docs = [] docs_lazy = loader.lazy_load() for doc in docs_lazy: docs.append(doc) print(docs[0].page_content[:100]) print(docs[0].metadata) .. code-block:: python Some text { 'file_name': 'example.pdf', 'file_type': 'application/pdf', # ... } zhttp://0.0.0.0:1231r TFr rrrrN)urlrrrrrrrrrrrrrrr rrrrrrrrrrrrrrrr!cs>tj|||||||| | | | | |||d||_d|jd<dS)a Initialize with file path, API url and parsing parameters. Args: file_path: path to the file for processing url: URL to call `dedoc` API split: type of document splitting into parts (each part is returned separately), default value "document" "document": document is returned as a single langchain Document object (don't split) "page": split document into pages (works for PDF, DJVU, PPTX, PPT, ODP) "node": split document into tree nodes (title nodes, list item nodes, raw text nodes) "line": split document into lines with_tables: add tables to the result - each table is returned as a single langchain Document object Parameters used for document parsing via `dedoc` (https://dedoc.readthedocs.io/en/latest/parameters/parameters.html): with_attachments: enable attached files extraction recursion_deep_attachments: recursion level for attached files extraction, works only when with_attachments==True pdf_with_text_layer: type of handler for parsing PDF documents, available options ["true", "false", "tabby", "auto", "auto_tabby" (default)] language: language of the document for PDF without a textual layer and images, available options ["eng", "rus", "rus+eng" (default)], the list of languages can be extended, please see https://dedoc.readthedocs.io/en/latest/tutorials/add_new_language.html pages: page slice to define the reading range for parsing PDF documents is_one_column_document: detect number of columns for PDF without a textual layer and images, available options ["true", "false", "auto" (default)] document_orientation: fix document orientation (90, 180, 270 degrees) for PDF without a textual layer and images, available options ["auto" (default), "no_change"] need_header_footer_analysis: remove headers and footers from the output result for parsing PDF and images need_binarization: clean pages background (binarize) for PDF without a textual layer and images need_pdf_table_analysis: parse tables for PDF without a textual layer and images delimiter: column separator for CSV, TSV files encoding: encoding of TXT, CSV, TSV )r rrrrrrrrrrrrrrjson return_formatN)superr8rr5)r"r rrrrrrrrrrrrrrr __class__r#r'r8s&AzDedocAPIFileLoader.__init__ccs2|j|j|j|jd}|j||jdEdHdS)r9)rr r>r?N) _send_filerr r5rIr)r"doc_treer#r#r'rN s  zDedocAPIFileLoader.lazy_loadcCsiSrr#rOr#r#r'rDszDedocAPIFileLoader._make_configr>c Csddl}tj|}t|d}d||fi}|j|d||d}Wdn1s+wY|jdkr?td|j t |j } | S) z7Send POST-request to `dedoc` API and return the resultsrNrbfilez/upload)filesdatazError during file handling: ) requestsospathbasenameopenpost status_coder7radecoderloads) r"rr r>r file_namerrrresultr#r#r'rs    zDedocAPIFileLoader._send_file)rrrrrrrrrr8rr rNrKrDrlistr __classcell__r#r#rr'rjs~M         Ur)r}rrabcrrtypingrrrrrlangchain_core.documentsr )langchain_community.document_loaders.baser r rrr#r#r#r's   M