移除 langchain_community 依赖,内联 CSVLoader/TextLoader,可选化 ebooklib/bs4/mobi/html2text 导入
This commit is contained in:
parent
30ff027a51
commit
acc1822ada
@ -1,16 +1,31 @@
|
|||||||
import os
|
import os
|
||||||
|
import csv
|
||||||
import codecs
|
import codecs
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from langchain_community.document_loaders.csv_loader import CSVLoader
|
|
||||||
from langchain_community.document_loaders.text import TextLoader
|
|
||||||
from docx import Document
|
from docx import Document
|
||||||
from ebooklib import epub
|
|
||||||
from bs4 import BeautifulSoup
|
|
||||||
from pptx import Presentation
|
from pptx import Presentation
|
||||||
from openpyxl import load_workbook
|
from openpyxl import load_workbook
|
||||||
import mobi
|
|
||||||
from pypdf import PdfReader
|
from pypdf import PdfReader
|
||||||
import html2text
|
|
||||||
|
try:
|
||||||
|
from ebooklib import epub
|
||||||
|
except ImportError:
|
||||||
|
epub = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
except ImportError:
|
||||||
|
BeautifulSoup = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
import mobi
|
||||||
|
except ImportError:
|
||||||
|
mobi = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
import html2text
|
||||||
|
except ImportError:
|
||||||
|
html2text = None
|
||||||
|
|
||||||
class BaseLoader:
|
class BaseLoader:
|
||||||
def __init__(self, file_path):
|
def __init__(self, file_path):
|
||||||
@ -27,6 +42,10 @@ class BaseLoader:
|
|||||||
|
|
||||||
class MyMobiLoader(BaseLoader):
|
class MyMobiLoader(BaseLoader):
|
||||||
def load_pagetext(self):
|
def load_pagetext(self):
|
||||||
|
if mobi is None:
|
||||||
|
raise ImportError("mobi not installed. Run: pip install mobi")
|
||||||
|
if html2text is None:
|
||||||
|
raise ImportError("html2text not installed. Run: pip install html2text")
|
||||||
tempdir, filepath = mobi.extract(self.filepath)
|
tempdir, filepath = mobi.extract(self.filepath)
|
||||||
with codecs.open(filepath, "r", "utf-8") as f:
|
with codecs.open(filepath, "r", "utf-8") as f:
|
||||||
content=f.read()
|
content=f.read()
|
||||||
@ -60,11 +79,10 @@ class MyPptLoader(BaseLoader):
|
|||||||
|
|
||||||
class MyCsvLoader(BaseLoader):
|
class MyCsvLoader(BaseLoader):
|
||||||
def load_pagetext(self):
|
def load_pagetext(self):
|
||||||
loader = CSVLoader(self.filepath)
|
with open(self.filepath, 'r', newline='', encoding='utf-8') as f:
|
||||||
docs = loader.load()
|
reader = csv.reader(f)
|
||||||
for i, d in enumerate(docs):
|
for i, row in enumerate(reader):
|
||||||
dat = (i, d.page_content)
|
yield (i, ' '.join(row))
|
||||||
yield dat
|
|
||||||
|
|
||||||
class MyExcelLoader(BaseLoader):
|
class MyExcelLoader(BaseLoader):
|
||||||
def load_pagetext(self):
|
def load_pagetext(self):
|
||||||
@ -98,11 +116,9 @@ class MyEpubLoader(BaseLoader):
|
|||||||
|
|
||||||
class MyTextLoader(BaseLoader):
|
class MyTextLoader(BaseLoader):
|
||||||
def load_pagetext(self):
|
def load_pagetext(self):
|
||||||
loader = TextLoader(self.filepath)
|
with open(self.filepath, 'r', encoding='utf-8') as f:
|
||||||
docs = loader.load()
|
content = f.read()
|
||||||
for i, d in enumerate(docs):
|
yield (0, content)
|
||||||
dat = (i, d.page_content)
|
|
||||||
yield dat
|
|
||||||
|
|
||||||
class File2Text:
|
class File2Text:
|
||||||
all_loaders = {
|
all_loaders = {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user