diff --git a/filetxt/loader.py b/filetxt/loader.py index 1d05bfd..ee66538 100644 --- a/filetxt/loader.py +++ b/filetxt/loader.py @@ -1,16 +1,31 @@ import os +import csv import codecs from datetime import datetime -from langchain_community.document_loaders.csv_loader import CSVLoader -from langchain_community.document_loaders.text import TextLoader from docx import Document -from ebooklib import epub -from bs4 import BeautifulSoup from pptx import Presentation from openpyxl import load_workbook -import mobi from pypdf import PdfReader -import html2text + +try: + from ebooklib import epub +except ImportError: + epub = None + +try: + from bs4 import BeautifulSoup +except ImportError: + BeautifulSoup = None + +try: + import mobi +except ImportError: + mobi = None + +try: + import html2text +except ImportError: + html2text = None class BaseLoader: def __init__(self, file_path): @@ -26,11 +41,15 @@ class BaseLoader: raise Exception('Not implement') class MyMobiLoader(BaseLoader): - def load_pagetext(self): - tempdir, filepath = mobi.extract(self.filepath) - with codecs.open(filepath, "r", "utf-8") as f: - content=f.read() - yield (0, html2text.html2text(content)) + def load_pagetext(self): + if mobi is None: + raise ImportError("mobi not installed. Run: pip install mobi") + if html2text is None: + raise ImportError("html2text not installed. Run: pip install html2text") + tempdir, filepath = mobi.extract(self.filepath) + with codecs.open(filepath, "r", "utf-8") as f: + content=f.read() + yield (0, html2text.html2text(content)) class MyPdfLoader(BaseLoader): def load_pagetext(self): @@ -59,12 +78,11 @@ class MyPptLoader(BaseLoader): yield (i,'\n'.join(txts)) class MyCsvLoader(BaseLoader): - def load_pagetext(self): - loader = CSVLoader(self.filepath) - docs = loader.load() - for i, d in enumerate(docs): - dat = (i, d.page_content) - yield dat + def load_pagetext(self): + with open(self.filepath, 'r', newline='', encoding='utf-8') as f: + reader = csv.reader(f) + for i, row in enumerate(reader): + yield (i, ' '.join(row)) class MyExcelLoader(BaseLoader): def load_pagetext(self): @@ -97,12 +115,10 @@ class MyEpubLoader(BaseLoader): class MyTextLoader(BaseLoader): - def load_pagetext(self): - loader = TextLoader(self.filepath) - docs = loader.load() - for i, d in enumerate(docs): - dat = (i, d.page_content) - yield dat + def load_pagetext(self): + with open(self.filepath, 'r', encoding='utf-8') as f: + content = f.read() + yield (0, content) class File2Text: all_loaders = {