今天处理了一下，转换gbk编码为utf-8，1998-01-2003版-带音.txt，我重新命令为1998-01-2003.txt
def convert_encoding(input_file, output_file,
input_encoding='gbk', output_encoding='utf8'):
with open(input_file, 'rb') as input_fd, open(output_file, 'wb') as output_fd:
file_content = input_fd.read()
unicode_file_content = file_content.decode(input_encoding)
output_fd.write(unicode_file_content.encode(output_encoding))
def create_if_dir_not_exists(path_dir):
if os.path.exists(path_dir):
if not os.path.isdir(path_dir):
raise ValueError("path: {} exits, but is not a directory!".format(path_dir))
else:
os.makedirs(path_dir, exist_ok=True)
current_dir='.'
data_dir = os.path.join(current_dir, 'data')
create_if_dir_not_exists(data_dir)
input_file = os.path.join(data_dir, '1998-01-2003.txt')
output_file = os.path.join(data_dir, 'raw_data.txt')
convert_encoding(input_file, output_file)
with open(output_file) as f:
list_data=f.readlines()
from tqdm import tqdm
list_strip_data=[]
for line in tqdm(list_data):
if(not line.strip()):
continue
line_arr=line.strip().split()
# print(line_arr)
key=line_arr[0].split('/')[0]
values=line_arr[1:]
list_strip_data.append([key,values])
processed_data=[]
for item in tqdm(list_strip_data):
key=item[0].split('-')[0]
values=item[1]
words=[item.split('/')[0] for item in values]
label=['B' if item.split('/')[1]=='nt' else 'O' for item in values]
processed_data.append([key,' '.join(words),' '.join(label)])
train_data=[]
valid_data=[]
test_data=[]
for item in processed_data:
if(item[0]<='19980120'):
train_data.append(item)
elif(item[0]>='19980120' and item[0]<='19980125'):
valid_data.append(item)
else:
test_data.append(item)
import pandas as pd
data=pd.DataFrame(train_data,columns=['id','text','label'])
data.to_csv('train.csv',index=False,sep='\t')
data=pd.DataFrame(valid_data,columns=['id','text','label'])
data.to_csv('valid.csv',index=False,sep='\t')
data=pd.DataFrame(test_data,columns=['id','text','label'])
data.to_csv('test.csv',index=False,sep='\t')
