Spaces:
Runtime error
Runtime error
File size: 2,389 Bytes
82515cb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
#!/usr/bin/env python
# -*- coding:utf-8 _*-
"""
@author:quincy qiang
@license: Apache Licence
@file: clean_corpus.py.py
@time: 2023/04/19
@contact: yanqiangmiffy@gamil.com
@software: PyCharm
@description: coding..
"""
"""
FILE : clean_corpus.py
FUNCTION : None
"""
import sys
import os
from optparse import OptionParser
class Clean(object):
def __init__(self, infile, outfile):
self.infile = infile
self.outfile = outfile
self.corpus = []
self.remove_corpus = []
self.read(self.infile)
self.remove(self.corpus)
self.write(self.remove_corpus, self.outfile)
def read(self, path):
print("reading now......")
if os.path.isfile(path) is False:
print("path is not a file")
exit()
now_line = 0
with open(path, encoding="UTF-8") as f:
for line in f:
now_line += 1
line = line.replace("\n", "").replace("\t", "")
self.corpus.append(line)
print("read finished.")
def remove(self, list):
print("removing now......")
for line in list:
re_list = []
for word in line:
if self.is_chinese(word) is False:
continue
re_list.append(word)
self.remove_corpus.append("".join(re_list))
print("remove finished.")
def write(self, list, path):
print("writing now......")
if os.path.exists(path):
os.remove(path)
file = open(path, encoding="UTF-8", mode="w")
for line in list:
file.writelines(line + "\n")
file.close()
print("writing finished")
def is_chinese(self, uchar):
"""判断一个unicode是否是汉字"""
if (uchar >= u'\u4e00') and (uchar <= u'\u9fa5'):
return True
else:
return False
if __name__ == "__main__":
print("clean corpus")
parser = OptionParser()
parser.add_option("--input", dest="input", default="", help="input file")
parser.add_option("--output", dest="output", default="", help="output file")
(options, args) = parser.parse_args()
input = options.input
output = options.output
try:
Clean(infile=input, outfile=output)
print("All Finished.")
except Exception as err:
print(err) |