from pandas import read_csv import numpy as np from sklearn.datasets.base import Bunch import pickle #导入cPickle包并且取一个别名pickle #持久化类 from sklearn.feature_extraction.text import TfidfVectorizer import jieba import xlwt import operator#排序用 from sklearn import metrics Straindata=[] Strainlabel=[] Sart_train=[] Stestdata=[] Stestlabel=[] Sart_test=[] Slast=[] Snew=[] class obj: def __init__(self): self.key=0 self.weight=0.0 def importSmallContentdata(file,data,art,label,f): dataset=read_csv(file) Sdata = dataset.values[:,:] print(type(Sdata)) if f==1: for line in Sdata: ls=[] ls.append(line[14]) ls.append(line[15]) ls.append(line[16]) ls.append(line[17]) Slast.append(ls) #print(len(Slast)) #print("需要对照的小类数据准备完毕") '''找到smalli不为0的装入Straindata,把数据分开''' for smalli in range(14,18): #print(smalli) count=0 for line in Sdata: count=count+1 if line[smalli]!='0' and line[smalli]!=0 : k=1 ls=[] for i in line: if k==1: art.append(i) k=k+1 continue if k==11:#k为14并不代表是line[14]，因为line是从0开始 break ls.append(float(i)) k=k+1 data.append(ls) label.append(line[smalli]) if f==1: Snew.append(count) #print("为什么都超限",len(Snew)) def getKvector(train_set,vec,n): nonzero=train_set.tdm.nonzero() k=0 lis=[] gather=[] p=-1 for i in nonzero[0]: p=p+1 if k==i: a=obj() a.key=nonzero[1][p] a.weight=train_set.tdm[i,nonzero[1][p]] lis.append(a) else: lis.sort(key=lambda obj: obj.weight, reverse=True)#对链表内为类对象的排序 gather.append(lis) while k < i: k=k+1 lis=[] a=obj() a.key=nonzero[1][p] a.weight=train_set.tdm[i,nonzero[1][p]] lis.append(a) gather.append(lis)#gather存储的是每条数据的事实描述的特征向量，已经从小到大排好了，只不过每个存既有key又有weight #我们只要key，不再需要weight sj=1 for i in gather: ls=[] for j in i: sj=sj+1 ls.append(float(j.key)) while sj<=n: sj=sj+1 ls.append(-1) sj=1 vec.append(ls) '''读取停用词''' def _readfile(path): with open(path, "rb") as fp: content = fp.read() return content ''' 读取bunch对象''' def _readbunchobj(path): with open(path, "rb") as file_obj: bunch = pickle.load(file_obj) return bunch '''写入bunch对象''' def _writebunchobj(path, bunchobj): with open(path, "wb") as file_obj: pickle.dump(bunchobj, file_obj) def buildtrainbunch(bunch_path,art_train,trainlabel): bunch = Bunch(label=[],contents=[]) for item1 in trainlabel: bunch.label.append(item1) #trainContentdatasave=[] #存储所有训练和测试数据的分词 for item2 in art_train: item2=str(item2) item2 = item2.replace("\r\n", "") item2 = item2.replace(" ", "") content_seg=jieba.cut(item2) save2='' for item3 in content_seg: if len(item3) > 1 and item3!='\r\n': #trainContentdatasave.append(item3) save2=save2+","+item3 bunch.contents.append(save2) with open(bunch_path, "wb") as file_obj: pickle.dump(bunch, file_obj) print("构建训练数据文本对象结束！！！") def buildtestbunch(bunch_path,art_test,testlabel): bunch = Bunch(label=[],contents=[]) for item1 in testlabel: bunch.label.append(item1) #testContentdatasave=[] #存储所有训练和测试数据的分词 for item2 in art_test: item2=str(item2) item2 = item2.replace("\r\n", "") item2 = item2.replace(" ", "") content_seg=jieba.cut(item2) save2='' for item3 in content_seg: if len(item3) > 1 and item3!='\r\n': #testContentdatasave.append(item3) save2=save2+","+item3 bunch.contents.append(save2) with open(bunch_path, "wb") as file_obj: pickle.dump(bunch, file_obj) print("构建测试数据文本对象结束！！！") def vector_space(stopword_path,bunch_path,space_path): stpwrdlst = _readfile(stopword_path).splitlines()#读取停用词 bunch = _readbunchobj(bunch_path)#导入分词后的词向量bunch对象 #构建tf-idf词向量空间对象 tfidfspace = Bunch(label=bunch.label,tdm=[], vocabulary={}) #权重矩阵tdm，其中，权重矩阵是一个二维矩阵，tdm[i][j]表示，第j个词（即词典中的序号）在第i个类别中的IF-IDF值 #使用TfidVectorizer初始化向量空间模型 vectorizer = TfidfVectorizer(stop_words=stpwrdlst, sublinear_tf=True, max_df=0.5, min_df=0.0001,use_idf=True,max_features=15000) #print(vectorizer) #文本转为词频矩阵，单独保存字典文件 tfidfspace.tdm = vectorizer.fit_transform(bunch.contents) tfidfspace.vocabulary = vectorizer.vocabulary_ #创建词袋的持久化 _writebunchobj(space_path, tfidfspace) print("if-idf词向量空间实例创建成功！！！") def testvector_space(stopword_path,bunch_path,space_path,train_tfidf_path): stpwrdlst = _readfile(stopword_path).splitlines()#把停用词变成列表 bunch = _readbunchobj(bunch_path) tfidfspace = Bunch(label=bunch.label,tdm=[], vocabulary={}) #导入训练集的TF-IDF词向量空间 ★★ trainbunch = _readbunchobj(train_tfidf_path) tfidfspace.vocabulary = trainbunch.vocabulary vectorizer = TfidfVectorizer(stop_words=stpwrdlst, sublinear_tf=True, max_df=0.7, vocabulary=trainbunch.vocabulary, min_df=0.001) tfidfspace.tdm = vectorizer.fit_transform(bunch.contents) _writebunchobj(space_path, tfidfspace) print("if-idf词向量空间实例创建成功！！！") if __name__=="__main__": '''============================先导入数据==================================''' file_train = 'F:/goverment/exceloperating/all_tocai_train.csv' file_test = 'F:/goverment/exceloperating/all_tocai_test.csv' importSmallContentdata(file_train,Straindata,Sart_train,Strainlabel,0) importSmallContentdata(file_test,Stestdata,Sart_test,Stestlabel,1) #print("Stestlabel" ,len(Stestlabel)) #print("小类导入数据完毕") #print("大类标签导入完毕")#共1329*4 '''==========================================================tf-idf对Bar进行文本特征提取============================================================================''' #导入分词后的词向量bunch对象 train_bunch_path ="F:/goverment/exceloperating/trainbunch.bat"#Bunch保存路径 train_space_path = "F:/goverment/exceloperating/traintfdifspace.dat" test_bunch_path ="F:/goverment/exceloperating/testbunch.bat" test_space_path = "F:/goverment/exceloperating/testtfdifspace.dat" stopword_path ="F:/goverment/exceloperating/hlt_stop_words.txt" '''============================================================tf-idf对Sart进行文本特征提取==============================================================================''' buildtrainbunch(train_bunch_path,Sart_train,Strainlabel) buildtestbunch(test_bunch_path,Sart_test,Stestlabel) vector_space(stopword_path,train_bunch_path,train_space_path) testvector_space(stopword_path,test_bunch_path,test_space_path,train_space_path) train_set=_readbunchobj(train_space_path) test_set=_readbunchobj(test_space_path) '''训练数据''' S_vec_train=[] getKvector(train_set,S_vec_train,76) '''测试数据''' S_vec_test=[] getKvector(test_set,S_vec_test,76) '''=================将得到的61个特征和之前的其它特征合并Btraindata==================''' '''小类训练数据''' S_vec_train=np.array(S_vec_train) #print(type(S_vec_train)) #print(S_vec_train.shape) Straindata=np.array(Straindata) #print(type(Straindata)) #print(Straindata.shape) Straindata=np.hstack((S_vec_train,Straindata)) #print(Straindata) '''小类测试数据''' S_vec_test=np.array(S_vec_test) Stestdata=np.array(Stestdata) Stestdata=np.hstack((S_vec_test,Stestdata)) print("分类算小类精度") Strainlabel=np.array(Strainlabel) Strainlabel=np.array(Strainlabel) from xgboost import XGBClassifier clf= XGBClassifier(learning_rate =0.1, n_estimators=1150, max_depth=2, min_child_weight=1, gamma=0, subsample=0.8, colsample_bytree=0.8, objective= 'binary:logistic', nthread=4,#没用 scale_pos_weight=1,#没用 seed=27) clf.fit(Straindata, Strainlabel) predict=clf.predict(Stestdata) aa=metrics.accuracy_score(Stestlabel, predict) print(aa)#40.09 ''''============================输出技术问题及其可能性================''' class attri: def __init__(self): self.key=0 self.weight=0.0 '''====================小类=======================''' attribute_proba=clf.predict_proba(Stestdata) label=[] for i in attribute_proba: lis=[] k=0 while k<4: k=k+1 p=1 mm=0 sj=-1 for j in i: sj=sj+1 if j>mm: mm=j p=sj i[p]=0#难道是从1开始？ a=attri() a.key=p a.weight=mm lis.append(a) #lis.append(p) label.append(lis) #接下来将label和snew结合，再排序去重就可以和slast比较了 #print("为什么都超限",len(Snew)) print("label",len(label)) count=0 for lis in label: lis.append(Snew[count]) count=count+1 print("结合完成，准备去重！")#此时label和Snew的长度都为1439 bol=np.zeros(len(label)+1) Snew=[] for lis in label: if bol[lis[4]]==0: Snew.append(lis) bol[lis[4]]=1 #print(len(Snew))#去重后为1162 for i in range(len(Slast)+1): if i==0: continue if bol[i]==0: ls=[] a=attri() a.weight=1 a.key=0 ls.append(a) ls.append(a) ls.append(a) ls.append(a) ls.append(i) Snew.append(ls) #print("Snew",len(Snew)) #为1329 print("去重完毕，准备排序！") Snew.sort(key=operator.itemgetter(4)) print("排序完毕，准备比较!") myexcel = xlwt.Workbook() sheet = myexcel.add_sheet('sheet') si=-2 sj=-1 #cys=1 #print(Snew) for i in Snew: si=si+2 #print(si) #print("对于记录 %d:" % cys) #cys=cys+1 for j in range(len(i)): if(j==len(i)-1): continue sj=sj+1 #sheet.write(si,sj,str(j)) sheet.write(si,sj,str(i[j].key)) sheet.write(si+1,sj,str(i[j].weight*100)) #print ("发生技术问题 %d 的可能性是:%.2f %%" % (j.key,j.weight*100)) sj=-1 myexcel.save("Snew.xls")
