From acc1822adaf9ef4e9ea6b16b77c6601c55d9a216 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Thu, 30 Jul 2026 18:26:16 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A7=BB=E9=99=A4=20langchain=5Fcommunity=20?= =?UTF-8?q?=E4=BE=9D=E8=B5=96=EF=BC=8C=E5=86=85=E8=81=94=20CSVLoader/TextL?= =?UTF-8?q?oader=EF=BC=8C=E5=8F=AF=E9=80=89=E5=8C=96=20ebooklib/bs4/mobi/h?= =?UTF-8?q?tml2text=20=E5=AF=BC=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- filetxt/loader.py | 62 +++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/filetxt/loader.py b/filetxt/loader.py index 1d05bfd..ee66538 100644 --- a/filetxt/loader.py +++ b/filetxt/loader.py @@ -1,16 +1,31 @@ import os +import csv import codecs from datetime import datetime -from langchain_community.document_loaders.csv_loader import CSVLoader -from langchain_community.document_loaders.text import TextLoader from docx import Document -from ebooklib import epub -from bs4 import BeautifulSoup from pptx import Presentation from openpyxl import load_workbook -import mobi from pypdf import PdfReader -import html2text + +try: + from ebooklib import epub +except ImportError: + epub = None + +try: + from bs4 import BeautifulSoup +except ImportError: + BeautifulSoup = None + +try: + import mobi +except ImportError: + mobi = None + +try: + import html2text +except ImportError: + html2text = None class BaseLoader: def __init__(self, file_path): @@ -26,11 +41,15 @@ class BaseLoader: raise Exception('Not implement') class MyMobiLoader(BaseLoader): - def load_pagetext(self): - tempdir, filepath = mobi.extract(self.filepath) - with codecs.open(filepath, "r", "utf-8") as f: - content=f.read() - yield (0, html2text.html2text(content)) + def load_pagetext(self): + if mobi is None: + raise ImportError("mobi not installed. Run: pip install mobi") + if html2text is None: + raise ImportError("html2text not installed. Run: pip install html2text") + tempdir, filepath = mobi.extract(self.filepath) + with codecs.open(filepath, "r", "utf-8") as f: + content=f.read() + yield (0, html2text.html2text(content)) class MyPdfLoader(BaseLoader): def load_pagetext(self): @@ -59,12 +78,11 @@ class MyPptLoader(BaseLoader): yield (i,'\n'.join(txts)) class MyCsvLoader(BaseLoader): - def load_pagetext(self): - loader = CSVLoader(self.filepath) - docs = loader.load() - for i, d in enumerate(docs): - dat = (i, d.page_content) - yield dat + def load_pagetext(self): + with open(self.filepath, 'r', newline='', encoding='utf-8') as f: + reader = csv.reader(f) + for i, row in enumerate(reader): + yield (i, ' '.join(row)) class MyExcelLoader(BaseLoader): def load_pagetext(self): @@ -97,12 +115,10 @@ class MyEpubLoader(BaseLoader): class MyTextLoader(BaseLoader): - def load_pagetext(self): - loader = TextLoader(self.filepath) - docs = loader.load() - for i, d in enumerate(docs): - dat = (i, d.page_content) - yield dat + def load_pagetext(self): + with open(self.filepath, 'r', encoding='utf-8') as f: + content = f.read() + yield (0, content) class File2Text: all_loaders = {