Spaces:
Running
Running
File size: 3,508 Bytes
54155f3 407c79e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
import io
import pdfplumber
import pandas as pd
import json
from docx import Document
from openpyxl import load_workbook
import re
import uuid
class FileReader:
def __init__(self):
self.allowed_files = ["txt", "pdf", "docx", "md", "json", "csv", "xlsx", "xls"]
self.max_chars_per_file = 5000000
def calc_chars(self, files, allowed_chars):
total_chars = 0
clean_contents = []
for file in files:
file_extension = file.filename.split('.')[-1].lower()
if file_extension not in self.allowed_files:
return {"error": "unsupported file type uploaded"}, 400
try:
if file_extension == 'txt' or file_extension=="md":
text = self._read_txt(file)
elif file_extension == 'pdf':
text = self._read_pdf(file)
elif file_extension == 'docx':
text = self._read_docx(file)
elif file_extension == 'json':
text = self._read_json(file)
elif file_extension == 'csv':
text = self._read_csv(file)
elif file_extension in ['xlsx', 'xls']:
text = self._read_excel(file)
if(len(text)>self.max_chars_per_file):
return {"error": "max 5 million characters per file allowed."} , 400
clean_contents.append({
"type": file_extension,
"content": text,
"name": file.filename,
"id": str(uuid.uuid4()),
"total_chars": len(text)
})
total_chars += len(text)
if(total_chars>int(allowed_chars)):
return {"error": "Total allowed characters limit reached"}, 400
except Exception as e:
return {"error": f"Error reading file {file.filename}: {e}"}, 500
return {"total_chars": total_chars, "clean_contents": clean_contents}, 200
def _read_txt(self, file):
file_content = file.read().decode("utf-8")
return self._clean_text(file_content)
def _read_pdf(self, file):
with pdfplumber.open(file) as pdf:
text = ''
for page in pdf.pages:
text += page.extract_text() or ''
return self._clean_text(text)
def _read_docx(self, file):
doc = Document(file)
text = ''
for para in doc.paragraphs:
text += para.text + "\n"
return self._clean_text(text)
def _read_json(self, file):
content = json.load(file)
text = json.dumps(content, ensure_ascii=False)
return self._clean_text(text)
def _read_csv(self, file):
df = pd.read_csv(file)
text = df.to_string(index=False)
return self._clean_text(text)
def _read_excel(self, file):
wb = load_workbook(file)
text = ''
for sheet in wb.sheetnames:
ws = wb[sheet]
for row in ws.iter_rows(values_only=True):
text += ' | '.join(str(cell) if cell is not None else '' for cell in row) + "\n"
return self._clean_text(text)
def _clean_text(self, text):
text = re.sub(r'\s+', ' ', text)
text = re.sub(r'[^\x00-\x7F]+', '', text)
text = text.strip()
return text
file_reader = FileReader() |