通过Python代码封装评分卡设计中经常使用的方法
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import math
from xgboost import XGBClassifier
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
from sklearn.model_selection import cross_val_score
import statsmodels.api as sm
from sklearn.linear_model import LogisticRegression
from sklearn import metrics
# EDA分析
# 类别型变量的分布
def plot_cate_var(df, col_list, hspace=0.4, wspace=0.4, plt_size=None, plt_num=None, x=None, y=None):
"""
df:数据集
col_list:变量list集合
hspace :子图之间的间隔(y轴方向)
wspace :子图之间的间隔(x轴方向)
plt_size :图纸的尺寸
plt_num :子图的数量
x :子图矩阵中一行子图的数量
y :子图矩阵中一列子图的数量
return :变量的分布图（柱状图形式）
"""
plt.figure(figsize=plt_size)
plt.subplots_adjust(hspace=hspace, wspace=wspace)
plt.rcParams['font.sans-serif'] = ['Microsoft YaHei']
plt.rcParams['axes.unicode_minus'] = False
for i, col in zip(range(1, plt_num + 1, 1), col_list):
plt.subplot(x, y, i)
plt.title(col)
sns.countplot(data=df, y=col)
plt.ylabel('')
return plt.show()
# 数值型变量的分布
def plot_num_col(df, col_list, hspace=0.4, wspace=0.4, plt_type=None, plt_size=None, plt_num=None, x=None, y=None):
"""
df:数据集
col_list:变量list集合
hspace :子图之间的间隔(y轴方向)
wspace :子图之间的间隔(x轴方向)
plt_type: 选择直方图/箱线图
plt_size :图纸的尺寸
plt_num :子图的数量
x :子图矩阵中一行子图的数量
y :子图矩阵中一列子图的数量
return :变量的分布图（箱线图/直方图）
"""
plt.figure(figsize=plt_size)
plt.subplots_adjust(hspace=hspace, wspace=wspace)
if plt_type == 'hist':
for i, col in zip(range(1, plt_num + 1, 1), col_list):
plt.subplot(x, y, i)
plt.title(col)
sns.distplot(df[col].dropna())
plt.xlabel('')
if plt_type == 'box':
for i, col in zip(range(1, plt_num + 1, 1), col_list):
plt.subplot(x, y, i)
plt.title(col)
sns.boxplot(data=df, x=col)
plt.xlabel('')
return plt.show()
# 类别型变量的违约率分析
def plot_default_cate(df, col_list, target, hspace=0.4, wspace=0.4, plt_size=None, plt_num=None, x=None, y=None):
"""
df:数据集
col_list:变量list集合
target ：目标变量的字段名
hspace :子图之间的间隔(y轴方向)
wspace :子图之间的间隔(x轴方向)
plt_size :图纸的尺寸
plt_num :子图的数量
x :子图矩阵中一行子图的数量
y :子图矩阵中一列子图的数量
return :违约率分布图（柱状图形式）
"""
all_bad = df[target].sum()
total = df[target].count()
all_default_rate = all_bad * 1.0 / total
plt.figure(figsize=plt_size)
plt.subplots_adjust(hspace=hspace, wspace=wspace)
plt.rcParams['font.sans-serif'] = ['Microsoft YaHei']
plt.rcParams['axes.unicode_minus'] = False
for i, col in zip(range(1, plt_num + 1, 1), col_list):
d1 = df.groupby(col)
d2 = pd.DataFrame()
d2['total'] = d1[target].count()
d2['bad'] = d1[target].sum()
d2['default_rate'] = d2['bad'] / d2['total']
d2 = d2.reset_index()
plt.subplot(x, y, i)
plt.title(col)
plt.axvline(x=all_default_rate)
sns.barplot(data=d2, y=col, x='default_rate')
plt.ylabel('')
return plt.show()
# 数值型变量的违约率分析
def plot_default_num(df, col_list, target, hspace=0.4, wspace=0.4, q=None, plt_size=None, plt_num=None, x=None, y=None):
"""
df:数据集
col_list:变量list集合
target ：目标变量的字段名
hspace :子图之间的间隔(y轴方向)
wspace :子图之间的间隔(x轴方向)
q :等深分箱的箱体个数
plt_size :图纸的尺寸
plt_num :子图的数量
x :子图矩阵中一行子图的数量
y :子图矩阵中一列子图的数量
return :违约率分布图（折线图形式）
"""
all_bad = df[target].sum()
total = df[target].count()
all_default_rate = all_bad * 1.0 / total
plt.figure(figsize=plt_size)
plt.subplots_adjust(hspace=hspace, wspace=wspace)
for i, col in zip(range(1, plt_num + 1, 1), col_list):
bucket = pd.qcut(df[col], q=q, duplicates='drop')
d1 = df.groupby(bucket)
d2 = pd.DataFrame()
d2['total'] = d1[target].count()
d2['bad'] = d1[target].sum()
d2['default_rate'] = d2['bad'] / d2['total']
d2 = d2.reset_index()
plt.subplot(x, y, i)
plt.title(col)
plt.axhline(y=all_default_rate)
sns.pointplot(data=d2, x=col, y='default_rate', color='hotpink')
plt.xticks(rotation=60)
plt.xlabel('')
return plt.show()
# 变量woe离散化
# 变量woe结果表
def woe_df_concat(bin_df):
"""
bin_df:list形式，里面存储每个变量的分箱结果
return :woe结果表
"""
woe_df_list = []
for df in bin_df:
woe_df = df.reset_index().assign(col=df.index.name).rename(columns={df.index.name: 'bin'})
woe_df_list.append(woe_df)
woe_result = pd.concat(woe_df_list, axis=0)
# 为了便于查看，将字段名列移到第一列的位置上
woe_result1 = woe_result['col']
woe_result2 = woe_result.iloc[:, :-1]
woe_result_df = pd.concat([woe_result1, woe_result2], axis=1)
woe_result_df = woe_result_df.reset_index(drop=True)
return woe_result_df
# woe转换
def woe_transform(df, target, df_woe):
"""
df:数据集
target:目标变量的字段名
df_woe:woe结果表
return:woe转化之后的数据集
"""
df2 = df.copy()
for col in df2.drop([target], axis=1).columns:
x = df2[col]
bin_map = df_woe[df_woe.col == col]
bin_res = np.array([0] * x.shape[0], dtype=float)
for i in bin_map.index:
lower = bin_map['min_bin'][i]
upper = bin_map['max_bin'][i]
if lower == upper:
x1 = x[np.where(x == lower)[0]]
else:
x1 = x[np.where((x >= lower) & (x <= upper))[0]]
mask = np.in1d(x, x1)
bin_res[mask] = bin_map['woe'][i]
bin_res = pd.Series(bin_res, index=x.index)
bin_res.name = x.name
df2[col] = bin_res
return df2
# 变量分箱
# 类别性变量的分箱
def binning_cate(df, col_list, target):
"""
df:数据集
col_list:变量list集合
target:目标变量的字段名
return:
bin_df :list形式，里面存储每个变量的分箱结果
iv_value:list形式，里面存储每个变量的IV值
"""
total = df[target].count()
bad = df[target].sum()
good = total - bad
all_odds = good * 1.0 / bad
bin_df = []
iv_value = []
for col in col_list:
d1 = df.groupby([col], as_index=True)
d2 = pd.DataFrame()
d2['min_bin'] = d1[col].min()
d2['max_bin'] = d1[col].max()
d2['total'] = d1[target].count()
d2['totalrate'] = d2['total'] / total
d2['bad'] = d1[target].sum()
d2['badrate'] = d2['bad'] / d2['total']
d2['good'] = d2['total'] - d2['bad']
d2['goodrate'] = d2['good'] / d2['total']
d2['badattr'] = d2['bad'] / bad
d2['goodattr'] = (d2['total'] - d2['bad']) / good
d2['odds'] = d2['good'] / d2['bad']
GB_list = []
for i in d2.odds:
if i >= all_odds:
GB_index = str(round((i / all_odds) * 100, 0)) + str('G')
else:
GB_index = str(round((all_odds / i) * 100, 0)) + str('B')
GB_list.append(GB_index)
d2['GB_index'] = GB_list
d2['woe'] = np.log(d2['badattr'] / d2['goodattr'])
d2['bin_iv'] = (d2['badattr'] - d2['goodattr']) * d2['woe']
d2['IV'] = d2['bin_iv'].sum()
iv = d2['bin_iv'].sum().round(3)
print('变量名:{}'.format(col))
print('IV:{}'.format(iv))
print('\t')
bin_df.append(d2)
iv_value.append(iv)
return bin_df, iv_value
# 类别性变量iv的明细表
def iv_cate(df, col_list, target):
"""
df:数据集
col_list:变量list集合
target:目标变量的字段名
return:变量的iv明细表
"""
bin_df, iv_value = binning_cate(df, col_list, target)
iv_df = pd.DataFrame({'col': col_list,
'iv': iv_value})
iv_df = iv_df.sort_values('iv', ascending=False)
return iv_df
# 数值型变量的分箱
# 先用卡方分箱输出变量的分割点
def split_data(df, col, split_num):
"""
df: 原始数据集
col:需要分箱的变量
split_num:分割点的数量
"""
df2 = df.copy()
count = df2.shape[0] # 总样本数
n = math.floor(count / split_num) # 按照分割点数目等分后每组的样本数
split_index = [i * n for i in range(1, split_num)] # 分割点的索引
values = sorted(list(df2[col])) # 对变量的值从小到大进行排序
split_value = [values[i] for i in split_index] # 分割点对应的value
split_value = sorted(list(set(split_value))) # 分割点的value去重排序
return split_value
def assign_group(x, split_bin):
"""
x:变量的value
split_bin:split_data得出的分割点list
"""
n = len(split_bin)
if x <= min(split_bin):
return min(split_bin) # 如果x小于分割点的最小值，则x映射为分割点的最小值
elif x > max(split_bin): # 如果x大于分割点的最大值，则x映射为分割点的最大值
return 10e10
else:
for i in range(n - 1):
if split_bin[i] < x <= split_bin[i + 1]: # 如果x在两个分割点之间，则x映射为分割点较大的值
return split_bin[i + 1]
def bin_bad_rate(df, col, target, grantRateIndicator=0):
"""
df:原始数据集
col:原始变量/变量映射后的字段
target:目标变量的字段
grantRateIndicator:是否输出总体的违约率
"""
total = df.groupby([col])[target].count()
bad = df.groupby([col])[target].sum()
total_df = pd.DataFrame({'total': total})
bad_df = pd.DataFrame({'bad': bad})
regroup = pd.merge(total_df, bad_df, left_index=True, right_index=True, how='left')
regroup = regroup.reset_index()
regroup['bad_rate'] = regroup['bad'] / regroup['total'] # 计算根据col分组后每组的违约率
dict_bad = dict(zip(regroup[col], regroup['bad_rate'])) # 转为字典形式
if grantRateIndicator == 0:
return (dict_bad, regroup)
total_all = df.shape[0]
bad_all = df[target].sum()
all_bad_rate = bad_all / total_all # 计算总体的违约率
return (dict_bad, regroup, all_bad_rate)
def cal_chi2(df, all_bad_rate):
"""
df:bin_bad_rate得出的regroup
all_bad_rate:bin_bad_rate得出的总体违约率
"""
df2 = df.copy()
df2['expected'] = df2['total'] * all_bad_rate # 计算每组的坏用户期望数量
combined = zip(df2['expected'], df2['bad']) # 遍历每组的坏用户期望数量和实际数量
chi = [(i[0] - i[1]) ** 2 / i[0] for i in combined] # 计算每组的卡方值
chi2 = sum(chi) # 计算总的卡方值
return chi2
def assign_bin(x, cutoffpoints):
"""
x:变量的value
cutoffpoints:分箱的切割点
"""
bin_num = len(cutoffpoints) + 1 # 箱体个数
if x <= cutoffpoints[0]: # 如果x小于最小的cutoff点，则映射为Bin 0
return 'Bin 0'
elif x > cutoffpoints[-1]: # 如果x大于最大的cutoff点，则映射为Bin(bin_num-1)
return 'Bin {}'.format(bin_num - 1)
else:
for i in range(0, bin_num - 1):
if cutoffpoints[i] < x <= cutoffpoints[i + 1]: # 如果x在两个cutoff点之间，则x映射为Bin(i+1)
return 'Bin {}'.format(i + 1)
def ChiMerge(df, col, target, max_bin=5, min_binpct=0):
col_unique = sorted(list(set(df[col]))) # 变量的唯一值并排序
n = len(col_unique) # 变量唯一值得个数
df2 = df.copy()
if n > 100: # 如果变量的唯一值数目超过100，则将通过split_data和assign_group将x映射为split对应的value
split_col = split_data(df2, col, 100) # 通过这个目的将变量的唯一值数目人为设定为100
df2['col_map'] = df2[col].map(lambda x: assign_group(x, split_col))
else:
df2['col_map'] = df2[col] # 变量的唯一值数目没有超过100，则不用做映射
# 生成dict_bad,regroup,all_bad_rate的元组
(dict_bad, regroup, all_bad_rate) = bin_bad_rate(df2, 'col_map', target, grantRateIndicator=1)
col_map_unique = sorted(list(set(df2['col_map']))) # 对变量映射后的value进行去重排序
group_interval = [[i] for i in col_map_unique] # 对col_map_unique中每个值创建list并存储在group_interval中
while (len(group_interval) > max_bin): # 当group_interval的长度大于max_bin时，执行while循环
chi_list = []
for i in range(len(group_interval) - 1):
temp_group = group_interval[i] + group_interval[i + 1] # temp_group 为生成的区间,list形式，例如[1,3]
chi_df = regroup[regroup['col_map'].isin(temp_group)]
chi_value = cal_chi2(chi_df, all_bad_rate) # 计算每一对相邻区间的卡方值
chi_list.append(chi_value)
best_combined = chi_list.index(min(chi_list)) # 最小的卡方值的索引
# 将卡方值最小的一对区间进行合并
group_interval[best_combined] = group_interval[best_combined] + group_interval[best_combined + 1]
# 删除合并前的右区间
group_interval.remove(group_interval[best_combined + 1])
# 对合并后每个区间进行排序
group_interval = [sorted(i) for i in group_interval]
# cutoff点为每个区间的最大值
cutoffpoints = [max(i) for i in group_interval[:-1]]
# 检查是否有箱只有好样本或者只有坏样本
df2['col_map_bin'] = df2['col_map'].apply(lambda x: assign_bin(x, cutoffpoints)) # 将col_map映射为对应的区间Bin
# 计算每个区间的违约率
(dict_bad, regroup) = bin_bad_rate(df2, 'col_map_bin', target)
# 计算最小和最大的违约率
[min_bad_rate, max_bad_rate] = [min(dict_bad.values()), max(dict_bad.values())]
# 当最小的违约率等于0，说明区间内只有好样本，当最大的违约率等于1，说明区间内只有坏样本
while min_bad_rate == 0 or max_bad_rate == 1:
bad01_index = regroup[regroup['bad_rate'].isin([0, 1])].col_map_bin.tolist() # 违约率为1或0的区间
bad01_bin = bad01_index[0]
if bad01_bin == max(regroup.col_map_bin):
cutoffpoints = cutoffpoints[:-1] # 当bad01_bin是最大的区间时，删除最大的cutoff点
elif bad01_bin == min(regroup.col_map_bin):
cutoffpoints = cutoffpoints[1:] # 当bad01_bin是最小的区间时，删除最小的cutoff点
else:
bad01_bin_index = list(regroup.col_map_bin).index(bad01_bin) # 找出bad01_bin的索引
prev_bin = list(regroup.col_map_bin)[bad01_bin_index - 1] # bad01_bin前一个区间
df3 = df2[df2.col_map_bin.isin([prev_bin, bad01_bin])]
(dict_bad, regroup1) = bin_bad_rate(df3, 'col_map_bin', target)
chi1 = cal_chi2(regroup1, all_bad_rate) # 计算前一个区间和bad01_bin的卡方值
later_bin = list(regroup.col_map_bin)[bad01_bin_index + 1] # bin01_bin的后一个区间
df4 = df2[df2.col_map_bin.isin([later_bin, bad01_bin])]
(dict_bad, regroup2) = bin_bad_rate(df4, 'col_map_bin', target)
chi2 = cal_chi2(regroup2, all_bad_rate) # 计算后一个区间和bad01_bin的卡方值
if chi1 < chi2: # 当chi1<chi2时,删除前一个区间对应的cutoff点
cutoffpoints.remove(cutoffpoints[bad01_bin_index - 1])
else: # 当chi1>=chi2时,删除bin01对应的cutoff点
cutoffpoints.remove(cutoffpoints[bad01_bin_index])
df2['col_map_bin'] = df2['col_map'].apply(lambda x: assign_bin(x, cutoffpoints))
(dict_bad, regroup) = bin_bad_rate(df2, 'col_map_bin', target)
# 重新将col_map映射至区间，并计算最小和最大的违约率，直达不再出现违约率为0或1的情况，循环停止
[min_bad_rate, max_bad_rate] = [min(dict_bad.values()), max(dict_bad.values())]
# 检查分箱后的最小占比
if min_binpct > 0:
group_values = df2['col_map'].apply(lambda x: assign_bin(x, cutoffpoints))
df2['col_map_bin'] = group_values # 将col_map映射为对应的区间Bin
group_df = group_values.value_counts().to_frame()
group_df['bin_pct'] = group_df['col_map'] / n # 计算每个区间的占比
min_pct = group_df.bin_pct.min() # 得出最小的区间占比
while min_pct < min_binpct and len(cutoffpoints) > 2: # 当最小的区间占比小于min_pct且cutoff点的个数大于2，执行循环
# 下面的逻辑基本与“检验是否有箱体只有好/坏样本”的一致
min_pct_index = group_df[group_df.bin_pct == min_pct].index.tolist()
min_pct_bin = min_pct_index[0]
if min_pct_bin == max(group_df.index):
cutoffpoints = cutoffpoints[:-1]
elif min_pct_bin == min(group_df.index):
cutoffpoints = cutoffpoints[1:]
else:
minpct_bin_index = list(group_df.index).index(min_pct_bin)
prev_pct_bin = list(group_df.index)[minpct_bin_index - 1]
df5 = df2[df2['col_map_bin'].isin([min_pct_bin, prev_pct_bin])]
(dict_bad, regroup3) = bin_bad_rate(df5, 'col_map_bin', target)
chi3 = cal_chi2(regroup3, all_bad_rate)
later_pct_bin = list(group_df.index)[minpct_bin_index + 1]
df6 = df2[df2['col_map_bin'].isin([min_pct_bin, later_pct_bin])]
(dict_bad, regroup4) = bin_bad_rate(df6, 'col_map_bin', target)
chi4 = cal_chi2(regroup4, all_bad_rate)
if chi3 < chi4:
cutoffpoints.remove(cutoffpoints[minpct_bin_index - 1])
else:
cutoffpoints.remove(cutoffpoints[minpct_bin_index])
return cutoffpoints
# 数值型变量的分箱（卡方分箱）
def binning_num(df, target, col_list, max_bin=None, min_binpct=None):
"""
df:数据集
target:目标变量的字段名
col_list:变量list集合
max_bin:最大的分箱个数
min_binpct:区间内样本所占总体的最小比
return:
bin_df :list形式，里面存储每个变量的分箱结果
iv_value:list形式，里面存储每个变量的IV值
"""
total = df[target].count()
bad = df[target].sum()
good = total - bad
all_odds = good / bad
inf = float('inf')
ninf = float('-inf')
bin_df = []
iv_value = []
for col in col_list:
cut = ChiMerge(df, col, target, max_bin=max_bin, min_binpct=min_binpct)
cut.insert(0, ninf)
cut.append(inf)
bucket = pd.cut(df[col], cut)
d1 = df.groupby(bucket)
d2 = pd.DataFrame()
d2['min_bin'] = d1[col].min()
d2['max_bin'] = d1[col].max()
d2['total'] = d1[target].count()
d2['totalrate'] = d2['total'] / total
d2['bad'] = d1[target].sum()
d2['badrate'] = d2['bad'] / d2['total']
d2['good'] = d2['total'] - d2['bad']
d2['goodrate'] = d2['good'] / d2['total']
d2['badattr'] = d2['bad'] / bad
d2['goodattr'] = (d2['total'] - d2['bad']) / good
d2['odds'] = d2['good'] / d2['bad']
GB_list = []
for i in d2.odds:
if i >= all_odds:
GB_index = str(round((i / all_odds) * 100, 0)) + str('G')
else:
GB_index = str(round((all_odds / i) * 100, 0)) + str('B')
GB_list.append(GB_index)
d2['GB_index'] = GB_list
d2['woe'] = np.log(d2['badattr'] / d2['goodattr'])
d2['bin_iv'] = (d2['badattr'] - d2['goodattr']) * d2['woe']
d2['IV'] = d2['bin_iv'].sum()
iv = d2['bin_iv'].sum().round(3)
print('变量名:{}'.format(col))
print('IV:{}'.format(iv))
print('\t')
bin_df.append(d2)
iv_value.append(iv)
return bin_df, iv_value
# 数值型变量的iv明细表
def iv_num(df, target, col_list, max_bin=None, min_binpct=None):
"""
df:数据集
target:目标变量的字段名
col_list:变量list集合
max_bin:最大的分箱个数
min_binpct:区间内样本所占总体的最小比
return :变量的iv明细表
"""
bin_df, iv_value = binning_num(df, target, col_list, max_bin=max_bin, min_binpct=min_binpct)
iv_df = pd.DataFrame({'col': col_list,
'iv': iv_value})
iv_df = iv_df.sort_values('iv', ascending=False)
return iv_df
# 自定义分箱
def binning_self(df, col, target, cut=None, right_border=True):
"""
df: 数据集
col:分箱的单个变量名
cut:划分区间的list
right_border：设定左开右闭、左闭右开
return:
bin_df: df形式，单个变量的分箱结果
iv_value: 单个变量的iv
"""
total = df[target].count()
bad = df[target].sum()
good = total - bad
all_odds = good / bad
bucket = pd.cut(df[col], cut, right=right_border)
d1 = df.groupby(bucket)
d2 = pd.DataFrame()
d2['min_bin'] = d1[col].min()
d2['max_bin'] = d1[col].max()
d2['total'] = d1[target].count()
d2['totalrate'] = d2['total'] / total
d2['bad'] = d1[target].sum()
d2['badrate'] = d2['bad'] / d2['total']
d2['good'] = d2['total'] - d2['bad']
d2['goodrate'] = d2['good'] / d2['total']
d2['badattr'] = d2['bad'] / bad
d2['goodattr'] = (d2['total'] - d2['bad']) / good
d2['odds'] = d2['good'] / d2['bad']
GB_list = []
for i in d2.odds:
if i >= all_odds:
GB_index = str(round((i / all_odds) * 100, 0)) + str('G')
else:
GB_index = str(round((all_odds / i) * 100, 0)) + str('B')
GB_list.append(GB_index)
d2['GB_index'] = GB_list
d2['woe'] = np.log(d2['badattr'] / d2['goodattr'])
d2['bin_iv'] = (d2['badattr'] - d2['goodattr']) * d2['woe']
d2['IV'] = d2['bin_iv'].sum()
iv_value = d2['bin_iv'].sum().round(3)
print('变量名:{}'.format(col))
print('IV:{}'.format(iv_value))
bin_df = d2.copy()
return bin_df, iv_value
# 变量分箱结果的检查
# woe的可视化
def plot_woe(bin_df, hspace=0.4, wspace=0.4, plt_size=None, plt_num=None, x=None, y=None):
"""
bin_df:list形式，里面存储每个变量的分箱结果
hspace :子图之间的间隔(y轴方向)
wspace :子图之间的间隔(x轴方向)
plt_size :图纸的尺寸
plt_num :子图的数量
x :子图矩阵中一行子图的数量
y :子图矩阵中一列子图的数量
return :每个变量的woe变化趋势图
"""
plt.figure(figsize=plt_size)
plt.subplots_adjust(hspace=hspace, wspace=wspace)
for i, df in zip(range(1, plt_num + 1, 1), bin_df):
col_name = df.index.name
df = df.reset_index()
plt.subplot(x, y, i)
plt.title(col_name)
sns.barplot(data=df, x=col_name, y='woe')
plt.xlabel('')
plt.xticks(rotation=30)
return plt.show()
# 检验woe是否单调
def woe_monoton(bin_df):
"""
bin_df:list形式，里面存储每个变量的分箱结果
return :
woe_notmonoton_col :woe没有呈单调变化的变量，list形式
woe_judge_df :df形式，每个变量的检验结果
"""
woe_notmonoton_col = []
col_list = []
woe_judge = []
for woe_df in bin_df:
col_name = woe_df.index.name
woe_list = list(woe_df.woe)
if woe_df.shape[0] == 2:
# print('{}是否单调: True'.format(col_name))
col_list.append(col_name)
woe_judge.append('True')
else:
woe_not_monoton = [(woe_list[i] < woe_list[i + 1] and woe_list[i] < woe_list[i - 1]) or (
woe_list[i] > woe_list[i + 1] and woe_list[i] > woe_list[i - 1]) for i in
range(1, len(woe_list) - 1, 1)]
if True in woe_not_monoton:
# print('{}是否单调: False'.format(col_name))
woe_notmonoton_col.append(col_name)
col_list.append(col_name)
woe_judge.append('False')
else:
# print('{}是否单调: True'.format(col_name))
col_list.append(col_name)
woe_judge.append('True')
woe_judge_df = pd.DataFrame({'col': col_list,
'judge_monoton': woe_judge})
return woe_notmonoton_col, woe_judge_df
# 检查某个区间的woe是否大于1
def woe_large(bin_df):
"""
bin_df:list形式，里面存储每个变量的分箱结果
return:
woe_large_col: 某个区间woe大于1的变量，list集合
woe_judge_df :df形式，每个变量的检验结果
"""
woe_large_col = []
col_list = []
woe_judge = []
for woe_df in bin_df:
col_name = woe_df.index.name
woe_list = list(woe_df.woe)
woe_large = list(filter(lambda x: x >= 1, woe_list))
if len(woe_large) > 0:
col_list.append(col_name)
woe_judge.append('True')
woe_large_col.append(col_name)
else:
col_list.append(col_name)
woe_judge.append('False')
woe_judge_df = pd.DataFrame({'col': col_list,
'judge_large': woe_judge})
return woe_large_col, woe_judge_df
# 变量筛选
# xgboost筛选变量
def select_xgboost(df, target, imp_num=None):
"""
df:数据集
target:目标变量的字段名
imp_num:筛选变量的个数
return:
xg_fea_imp:变量的特征重要性
xg_select_col:筛选出的变量
"""
x = df.drop([target], axis=1)
y = df[target]
xgmodel = XGBClassifier(random_state=0)
xgmodel = xgmodel.fit(x, y, eval_metric='auc')
xg_fea_imp = pd.DataFrame({'col': list(x.columns),
'imp': xgmodel.feature_importances_})
xg_fea_imp = xg_fea_imp.sort_values('imp', ascending=False).reset_index(drop=True).iloc[:imp_num, :]
xg_select_col = list(xg_fea_imp.col)
return xg_fea_imp, xg_select_col
# 随机森林筛选变量
def select_rf(df, target, imp_num=None):
"""
df:数据集
target:目标变量的字段名
imp_num:筛选变量的个数
return:
rf_fea_imp:变量的特征重要性
rf_select_col:筛选出的变量
"""
x = df.drop([target], axis=1)
y = df[target]
rfmodel = RandomForestClassifier(random_state=0)
rfmodel = rfmodel.fit(x, y)
rf_fea_imp = pd.DataFrame({'col': list(x.columns),
'imp': rfmodel.feature_importances_})
rf_fea_imp = rf_fea_imp.sort_values('imp', ascending=False).reset_index(drop=True).iloc[:imp_num, :]
rf_select_col = list(rf_fea_imp.col)
return rf_fea_imp, rf_select_col
# 相关性可视化
def plot_corr(df, col_list, threshold=None, plt_size=None, is_annot=True):
"""
df:数据集
col_list:变量list集合
threshold: 相关性设定的阈值
plt_size:图纸尺寸
is_annot:是否显示相关系数值
return :相关性热力图
"""
corr_df = df.loc[:, col_list].corr()
plt.figure(figsize=plt_size)
sns.heatmap(corr_df, annot=is_annot, cmap='rainbow', vmax=1, vmin=-1, mask=np.abs(corr_df) <= threshold)
return plt.show()
# 相关性剔除
def forward_delete_corr(df, col_list, threshold=None):
"""
df:数据集
col_list:变量list集合
threshold: 相关性设定的阈值
return:相关性剔除后的变量
"""
list_corr = col_list[:]
for col in list_corr:
corr = df.loc[:, list_corr].corr()[col]
corr_index = [x for x in corr.index if x != col]
corr_values = [x for x in corr.values if x != 1]
for i, j in zip(corr_index, corr_values):
if abs(j) >= threshold:
list_corr.remove(i)
return list_corr
# 相关性变量映射关系
def corr_mapping(df, col_list, threshold=None):
"""
df:数据集
col_list:变量list集合
threshold: 相关性设定的阈值
return:强相关性变量之间的映射关系表
"""
corr_df = df.loc[:, col_list].corr()
col_a = []
col_b = []
corr_value = []
for col, i in zip(col_list[:-1], range(1, len(col_list), 1)):
high_corr_col = []
high_corr_value = []
corr_series = corr_df[col][i:]
for i, j in zip(corr_series.index, corr_series.values):
if abs(j) >= threshold:
high_corr_col.append(i)
high_corr_value.append(j)
col_a.extend([col] * len(high_corr_col))
col_b.extend(high_corr_col)
corr_value.extend(high_corr_value)
corr_map_df = pd.DataFrame({'col_A': col_a,
'col_B': col_b,
'corr': corr_value})
return corr_map_df
# 显著性筛选,在筛选前需要做woe转换
def forward_delete_pvalue(x_train, y_train):
"""
x_train -- x训练集
y_train -- y训练集
return :显著性筛选后的变量
"""
col_list = list(x_train.columns)
pvalues_col = []
for col in col_list:
pvalues_col.append(col)
x_train2 = sm.add_constant(x_train.loc[:, pvalues_col])
sm_lr = sm.Logit(y_train, x_train2)
sm_lr = sm_lr.fit()
for i, j in zip(sm_lr.pvalues.index[1:], sm_lr.pvalues.values[1:]):
if j >= 0.05:
pvalues_col.remove(i)
x_new_train = x_train.loc[:, pvalues_col]
x_new_train2 = sm.add_constant(x_new_train)
lr = sm.Logit(y_train, x_new_train2)
lr = lr.fit()
print(lr.summary2())
return pvalues_col
# 逻辑回归系数符号筛选,在筛选前需要做woe转换
def forward_delete_coef(x_train, y_train):
"""
x_train -- x训练集
y_train -- y训练集
return :
coef_col回归系数符号筛选后的变量
lr_coe：每个变量的系数值
"""
col_list = list(x_train.columns)
coef_col = []
for col in col_list:
coef_col.append(col)
x_train2 = x_train.loc[:, coef_col]
sk_lr = LogisticRegression(random_state=0).fit(x_train2, y_train)
coef_df = pd.DataFrame({'col': coef_col, 'coef': sk_lr.coef_[0]})
if coef_df[coef_df.coef < 0].shape[0] > 0:
coef_col.remove(col)
x_new_train = x_train.loc[:, coef_col]
lr = LogisticRegression(random_state=0).fit(x_new_train, y_train)
lr_coe = pd.DataFrame({'col': coef_col,
'coef': lr.coef_[0]})
return coef_col, lr_coe
# 数据预处理
# 每个变量缺失率的计算
def missing_cal(df):
"""
df :数据集
return：每个变量的缺失率
"""
missing_series = df.isnull().sum() / df.shape[0]
missing_df = pd.DataFrame(missing_series).reset_index()
missing_df = missing_df.rename(columns={'index': 'col',
0: 'missing_pct'})
missing_df = missing_df.sort_values('missing_pct', ascending=False).reset_index(drop=True)
return missing_df
# 变量的缺失分布图
def plot_missing_var(df, plt_size=None):
"""
df: 数据集
plt_size :图纸的尺寸
return: 缺失分布图（直方图形式)
"""
missing_df = missing_cal(df)
plt.figure(figsize=plt_size)
plt.rcParams['font.sans-serif'] = ['Microsoft YaHei']
plt.rcParams['axes.unicode_minus'] = False
x = missing_df['missing_pct']
plt.hist(x=x, bins=np.arange(0, 1.1, 0.1), color='hotpink', ec='k', alpha=0.8)
plt.ylabel('缺失值个数')
plt.xlabel('缺失率')
return plt.show()
# 单个样本的缺失分布
def plot_missing_user(df, plt_size=None):
"""
df: 数据集
plt_size: 图纸的尺寸
return :缺失分布图（折线图形式）
"""
missing_series = df.isnull().sum(axis=1)
list_missing_num = sorted(list(missing_series.values))
plt.figure(figsize=plt_size)
plt.rcParams['font.sans-serif'] = ['Microsoft YaHei']
plt.rcParams['axes.unicode_minus'] = False
plt.plot(range(df.shape[0]), list_missing_num)
plt.ylabel('缺失变量个数')
plt.xlabel('sanples')
return plt.show()
# 缺失值剔除（单个变量）
def missing_delete_var(df, threshold=None):
"""
df:数据集
threshold:缺失率删除的阈值
return :删除缺失后的数据集
"""
df2 = df.copy()
missing_df = missing_cal(df)
missing_col_num = missing_df[missing_df.missing_pct >= threshold].shape[0]
missing_col = list(missing_df[missing_df.missing_pct >= threshold].col)
df2 = df2.drop(missing_col, axis=1)
print('缺失率超过{}的变量个数为{}'.format(threshold, missing_col_num))
return df2
# 缺失值剔除（单个样本）
def missing_delete_user(df, threshold=None):
"""
df:数据集
threshold:缺失个数删除的阈值
return :删除缺失后的数据集
"""
df2 = df.copy()
missing_series = df.isnull().sum(axis=1)
missing_list = list(missing_series)
missing_index_list = []
for i, j in enumerate(missing_list):
if j >= threshold:
missing_index_list.append(i)
df2 = df2[~(df2.index.isin(missing_index_list))]
print('缺失变量个数在{}以上的用户数有{}个'.format(threshold, len(missing_index_list)))
return df2
# 缺失值填充（类别型变量）
def fillna_cate_var(df, col_list, fill_type=None):
"""
df:数据集
col_list:变量list集合
fill_type: 填充方式：众数/当做一个类别
return :填充后的数据集
"""
df2 = df.copy()
for col in col_list:
if fill_type == 'class':
df2[col] = df2[col].fillna('unknown')
if fill_type == 'mode':
df2[col] = df2[col].fillna(df2[col].mode()[0])
return df2
# 数值型变量的填充
# 针对缺失率在5%以下的变量用中位数填充
# 缺失率在5%--15%的变量用随机森林填充,可先对缺失率较低的变量先用中位数填充，在用没有缺失的样本来对变量作随机森林填充
# 缺失率超过15%的变量建议当做一个类别
def fillna_num_var(df, col_list, fill_type=None, filled_df=None):
"""
df:数据集
col_list:变量list集合
fill_type:填充方式：中位数/随机森林/当做一个类别
filled_df :已填充好的数据集，当填充方式为随机森林时 使用
return:已填充好的数据集
"""
df2 = df.copy()
for col in col_list:
if fill_type == 'median':
df2[col] = df2[col].fillna(df2[col].median())
if fill_type == 'class':
df2[col] = df2[col].fillna(-999)
if fill_type == 'rf':
rf_df = pd.concat([df2[col], filled_df], axis=1)
known = rf_df[rf_df[col].notnull()]
unknown = rf_df[rf_df[col].isnull()]
x_train = known.drop([col], axis=1)
y_train = known[col]
x_pre = unknown.drop([col], axis=1)
rf = RandomForestRegressor(random_state=0)
rf.fit(x_train, y_train)
y_pre = rf.predict(x_pre)
df2.loc[df2[col].isnull(), col] = y_pre
return df2
# 常变量/同值化处理
def const_delete(df, col_list, threshold=None):
"""
df:数据集
col_list:变量list集合
threshold:同值化处理的阈值
return :处理后的数据集
"""
df2 = df.copy()
const_col = []
for col in col_list:
const_pct = df2[col].value_counts().iloc[0] / df2[df2[col].notnull()].shape[0]
if const_pct >= threshold:
const_col.append(col)
df2 = df2.drop(const_col, axis=1)
print('常变量/同值化处理的变量个数为{}'.format(len(const_col)))
return df2
# 分类型变量的降基处理
def descending_cate(df, col_list, threshold=None):
"""
df: 数据集
col_list:变量list集合
threshold:降基处理的阈值
return :处理后的数据集
"""
df2 = df.copy()
for col in col_list:
value_series = df[col].value_counts() / df[df[col].notnull()].shape[0]
small_value = []
for value_name, value_pct in zip(value_series.index, value_series.values):
if value_pct <= threshold:
small_value.append(value_name)
df2.loc[df2[col].isin(small_value), col] = 'other'
return df2
# 模型评估
# AUC
def plot_roc(y_label, y_pred):
"""
y_label:测试集的y
y_pred:对测试集预测后的概率
return:ROC曲线
"""
tpr, fpr, threshold = metrics.roc_curve(y_label, y_pred)
AUC = metrics.roc_auc_score(y_label, y_pred)
fig = plt.figure(figsize=(6, 4))
ax = fig.add_subplot(1, 1, 1)
ax.plot(tpr, fpr, color='blue', label='AUC=%.3f' % AUC)
ax.plot([0, 1], [0, 1], 'r--')
ax.set_ylim(0, 1)
ax.set_xlim(0, 1)
ax.set_title('ROC')
ax.legend(loc='best')
return plt.show(ax)
# KS
def plot_model_ks(y_label, y_pred):
"""
y_label:测试集的y
y_pred:对测试集预测后的概率
return:KS曲线
"""
pred_list = list(y_pred)
label_list = list(y_label)
total_bad = sum(label_list)
total_good = len(label_list) - total_bad
items = sorted(zip(pred_list, label_list), key=lambda x: x[0])
step = (max(pred_list) - min(pred_list)) / 200
pred_bin = []
good_rate = []
bad_rate = []
ks_list = []
for i in range(1, 201):
idx = min(pred_list) + i * step
pred_bin.append(idx)
label_bin = [x[1] for x in items if x[0] < idx]
bad_num = sum(label_bin)
good_num = len(label_bin) - bad_num
goodrate = good_num / total_good
badrate = bad_num / total_bad
ks = abs(goodrate - badrate)
good_rate.append(goodrate)
bad_rate.append(badrate)
ks_list.append(ks)
fig = plt.figure(figsize=(8, 6))
ax = fig.add_subplot(1, 1, 1)
ax.plot(pred_bin, good_rate, color='green', label='good_rate')
ax.plot(pred_bin, bad_rate, color='red', label='bad_rate')
ax.plot(pred_bin, ks_list, color='blue', label='good-bad')
ax.set_title('KS:{:.3f}'.format(max(ks_list)))
ax.legend(loc='best')
return plt.show(ax)
# 交叉验证
def cross_verify(x, y, estimators, fold, scoring='roc_auc'):
"""
x:自变量的数据集
y:target的数据集
estimators：验证的模型
fold：交叉验证的策略
scoring:评级指标，默认auc
return:交叉验证的结果
"""
cv_result = cross_val_score(estimator=estimators, X=x, y=y, cv=fold, n_jobs=-1, scoring=scoring)
print('CV的最大AUC为:{}'.format(cv_result.max()))
print('CV的最小AUC为:{}'.format(cv_result.min()))
print('CV的平均AUC为:{}'.format(cv_result.mean()))
plt.figure(figsize=(6, 4))
plt.title('交叉验证的评价指标分布图')
plt.boxplot(cv_result, patch_artist=True, showmeans=True,
boxprops={'color': 'black', 'facecolor': 'yellow'},
meanprops={'marker': 'D', 'markerfacecolor': 'tomato'},
flierprops={'marker': 'o', 'markerfacecolor': 'red', 'color': 'black'},
medianprops={'linestyle': '--', 'color': 'orange'})
return plt.show()
# 学习曲线
def plot_learning_curve(estimator, x, y, cv=None, train_size=np.linspace(0.1, 1.0, 5), plt_size=None):
"""
estimator :画学习曲线的基模型
x:自变量的数据集
y:target的数据集
cv:交叉验证的策略
train_size:训练集划分的策略
plt_size:画图尺寸
return:学习曲线
"""
from sklearn.model_selection import learning_curve
train_sizes, train_scores, test_scores = learning_curve(estimator=estimator,
X=x,
y=y,
cv=cv,
n_jobs=-1,
train_sizes=train_size)
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
plt.figure(figsize=plt_size)
plt.xlabel('Training-example')
plt.ylabel('score')
plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
train_scores_mean + train_scores_std, alpha=0.1, color='r')
plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
test_scores_mean + test_scores_std, alpha=0.1, color='g')
plt.plot(train_sizes, train_scores_mean, 'o-', color='r', label='Training-score')
plt.plot(train_sizes, test_scores_mean, 'o-', color='g', label='cross-val-score')
plt.legend(loc='best')
return plt.show()
# 混淆矩阵 /分类报告
def plot_matrix_report(y_label, y_pred):
"""
y_label:测试集的y
y_pred:对测试集预测后的概率
return:混淆矩阵
"""
matrix_array = metrics.confusion_matrix(y_label, y_pred)
plt.matshow(matrix_array, cmap=plt.cm.summer_r)
plt.colorbar()
for x in range(len(matrix_array)):
for y in range(len(matrix_array)):
plt.annotate(matrix_array[x, y], xy=(x, y), ha='center', va='center')
plt.xlabel('True label')
plt.ylabel('Predict label')
print(metrics.classification_report(y_label, y_pred))
return plt.show()
# 评分卡实现
# 评分卡刻度
def cal_scale(score, odds, PDO, model):
"""
odds：设定的坏好比
score:在这个odds下的分数
PDO: 好坏翻倍比
model:逻辑回归模型
return :A,B,base_score
"""
B = 20 / (np.log(odds) - np.log(2 * odds))
A = score - B * np.log(odds)
base_score = A + B * model.intercept_[0]
print('B: {:.2f}'.format(B))
print('A: {:.2f}'.format(A))
print('基础分为：{:.2f}'.format(base_score))
return A, B, base_score
# 变量得分表
def score_df_concat(woe_df, model, B):
"""
woe_df: woe结果表
model:逻辑回归模型
return:变量得分结果表
"""
coe = list(model.coef_[0])
columns = list(woe_df.col.unique())
scores = []
for c, col in zip(coe, columns):
score = []
for w in list(woe_df[woe_df.col == col].woe):
s = round(c * w * B, 0)
score.append(s)
scores.extend(score)
woe_df['score'] = scores
score_df = woe_df.copy()
return score_df
# 分数转换
def score_transform(df, target, df_score):
"""
df:数据集
target:目标变量的字段名
df_score:得分结果表
return:得分转化之后的数据集
"""
df2 = df.copy()
for col in df2.drop([target], axis=1).columns:
x = df2[col]
bin_map = df_score[df_score.col == col]
bin_res = np.array([0] * x.shape[0], dtype=float)
for i in bin_map.index:
lower = bin_map['min_bin'][i]
upper = bin_map['max_bin'][i]
if lower == upper:
x1 = x[np.where(x == lower)[0]]
else:
x1 = x[np.where((x >= lower) & (x <= upper))[0]]
mask = np.in1d(x, x1)
bin_res[mask] = bin_map['score'][i]
bin_res = pd.Series(bin_res, index=x.index)
bin_res.name = x.name
df2[col] = bin_res
return df2
# 得分的KS
def plot_score_ks(df, score_col, target):
"""
df:数据集
target:目标变量的字段名
score_col:最终得分的字段名
"""
total_bad = df[target].sum()
total_good = df[target].count() - total_bad
score_list = list(df[score_col])
target_list = list(df[target])
items = sorted(zip(score_list, target_list), key=lambda x: x[0])
step = (max(score_list) - min(score_list)) / 200
score_bin = []
good_rate = []
bad_rate = []
ks_list = []
for i in range(1, 201):
idx = min(score_list) + i * step
score_bin.append(idx)
target_bin = [x[1] for x in items if x[0] < idx]
bad_num = sum(target_bin)
good_num = len(target_bin) - bad_num
goodrate = good_num / total_good
badrate = bad_num / total_bad
ks = abs(goodrate - badrate)
good_rate.append(goodrate)
bad_rate.append(badrate)
ks_list.append(ks)
fig = plt.figure(figsize=(8, 6))
ax = fig.add_subplot(1, 1, 1)
ax.plot(score_bin, good_rate, color='green', label='good_rate')
ax.plot(score_bin, bad_rate, color='red', label='bad_rate')
ax.plot(score_bin, ks_list, color='blue', label='good-bad')
ax.set_title('KS:{:.3f}'.format(max(ks_list)))
ax.legend(loc='best')
return plt.show(ax)
# PR曲线
def plot_PR(df, score_col, target, plt_size=None):
"""
df:得分的数据集
score_col:分数的字段名
target:目标变量的字段名
plt_size:绘图尺寸
return: PR曲线
"""
total_bad = df[target].sum()
score_list = list(df[score_col])
target_list = list(df[target])
score_unique_list = sorted(set(list(df[score_col])))
items = sorted(zip(score_list, target_list), key=lambda x: x[0])
precison_list = []
tpr_list = []
for score in score_unique_list:
target_bin = [x[1] for x in items if x[0] <= score]
bad_num = sum(target_bin)
total_num = len(target_bin)
precison = bad_num / total_num
tpr = bad_num / total_bad
precison_list.append(precison)
tpr_list.append(tpr)
plt.figure(figsize=plt_size)
plt.title('PR曲线')
plt.xlabel('查全率')
plt.ylabel('精确率')
plt.plot(tpr_list, precison_list, color='tomato', label='PR曲线')
plt.legend(loc='best')
return plt.show()
# 得分分布图
def plot_score_hist(df, target, score_col, plt_size=None, cutoff=None):
"""
df:数据集
target:目标变量的字段名
score_col:最终得分的字段名
plt_size:图纸尺寸
cutoff :划分拒绝/通过的点
return :好坏用户的得分分布图
"""
plt.figure(figsize=plt_size)
x1 = df[df[target] == 1][score_col]
x2 = df[df[target] == 0][score_col]
sns.kdeplot(x1, shade=True, label='坏用户', color='hotpink')
sns.kdeplot(x2, shade=True, label='好用户', color='seagreen')
plt.axvline(x=cutoff)
plt.legend()
return plt.show()
# 得分明细表
def score_info(df, score_col, target, x=None, y=None, step=None):
"""
df:数据集
target:目标变量的字段名
score_col:最终得分的字段名
x:最小区间的左值
y:最大区间的右值
step:区间的分数间隔
return :得分明细表
"""
df['score_bin'] = pd.cut(df[score_col], bins=np.arange(x, y, step), right=True)
total = df[target].count()
bad = df[target].sum()
good = total - bad
group = df.groupby('score_bin')
score_info_df = pd.DataFrame()
score_info_df['用户数'] = group[target].count()
score_info_df['坏用户'] = group[target].sum()
score_info_df['好用户'] = score_info_df['用户数'] - score_info_df['坏用户']
score_info_df['违约占比'] = score_info_df['坏用户'] / score_info_df['用户数']
score_info_df['累计用户'] = score_info_df['用户数'].cumsum()
score_info_df['坏用户累计'] = score_info_df['坏用户'].cumsum()
score_info_df['好用户累计'] = score_info_df['好用户'].cumsum()
score_info_df['坏用户累计占比'] = score_info_df['坏用户累计'] / bad
score_info_df['好用户累计占比'] = score_info_df['好用户累计'] / good
score_info_df['累计用户占比'] = score_info_df['累计用户'] / total
score_info_df['累计违约占比'] = score_info_df['坏用户累计'] / score_info_df['累计用户']
score_info_df = score_info_df.reset_index()
return score_info_df
# 绘制提升图和洛伦兹曲线
def plot_lifting(df, score_col, target, bins=10, plt_size=None):
"""
df:数据集，包含最终的得分
score_col:最终分数的字段名
target:目标变量名
bins:分数划分成的等份数
plt_size:绘图尺寸
return:提升图和洛伦兹曲线
"""
score_list = list(df[score_col])
label_list = list(df[target])
items = sorted(zip(score_list, label_list), key=lambda x: x[0])
step = round(df.shape[0] / bins, 0)
bad = df[target].sum()
all_badrate = float(1 / bins)
all_badrate_list = [all_badrate] * bins
all_badrate_cum = list(np.cumsum(all_badrate_list))
all_badrate_cum.insert(0, 0)
score_bin_list = []
bad_rate_list = []
for i in range(0, bins, 1):
index_a = int(i * step)
index_b = int((i + 1) * step)
score = [x[0] for x in items[index_a:index_b]]
tup1 = (min(score),)
tup2 = (max(score),)
score_bin = tup1 + tup2
score_bin_list.append(score_bin)
label_bin = [x[1] for x in items[index_a:index_b]]
bin_bad = sum(label_bin)
bin_bad_rate = bin_bad / bad
bad_rate_list.append(bin_bad_rate)
bad_rate_cumsum = list(np.cumsum(bad_rate_list))
bad_rate_cumsum.insert(0, 0)
plt.figure(figsize=plt_size)
x = score_bin_list
y1 = bad_rate_list
y2 = all_badrate_list
y3 = bad_rate_cumsum
y4 = all_badrate_cum
plt.subplot(1, 2, 1)
plt.title('提升图')
plt.xticks(np.arange(bins) + 0.15, x, rotation=90)
bar_width = 0.3
plt.bar(np.arange(bins), y1, width=bar_width, color='hotpink', label='score_card')
plt.bar(np.arange(bins) + bar_width, y2, width=bar_width, color='seagreen', label='random')
plt.legend(loc='best')
plt.subplot(1, 2, 2)
plt.title('洛伦兹曲线图')
plt.plot(y3, color='hotpink', label='score_card')
plt.plot(y4, color='seagreen', label='random')
plt.xticks(np.arange(bins + 1), rotation=0)
plt.legend(loc='best')
return plt.show()
# 设定cutoff点，衡量有效性
def rule_verify(df, col_score, target, cutoff):
"""
df:数据集
target:目标变量的字段名
col_score:最终得分的字段名
cutoff :划分拒绝/通过的点
return :混淆矩阵
"""
df['result'] = df.apply(lambda x: 30 if x[col_score] <= cutoff else 10, axis=1)
TP = df[(df['result'] == 30) & (df[target] == 1)].shape[0]
FN = df[(df['result'] == 30) & (df[target] == 0)].shape[0]
bad = df[df[target] == 1].shape[0]
good = df[df[target] == 0].shape[0]
refuse = df[df['result'] == 30].shape[0]
passed = df[df['result'] == 10].shape[0]
acc = round(TP / refuse, 3)
tpr = round(TP / bad, 3)
fpr = round(FN / good, 3)
pass_rate = round(refuse / df.shape[0], 3)
matrix_df = pd.pivot_table(df, index='result', columns=target, aggfunc={col_score: pd.Series.count},
values=col_score)
print('精确率:{}'.format(acc))
print('查全率:{}'.format(tpr))
print('误伤率:{}'.format(fpr))
print('规则拒绝率:{}'.format(pass_rate))
return matrix_df
# 绘制变量的得分占比偏移图
def plot_var_shift(df, day_col, score_col, plt_size=None):
"""
df:变量在一段时间内，每个区间上的得分
day_col:时间的字段名（天）
score_col:得分的字段名
plt_size: 绘图尺寸
return:变量区间得分的偏移图
"""
day_list = sorted(set(list(df[day_col])))
score_list = sorted(set(list(df[score_col])))
# 计算每天各个区间得分的占比
prop_day_list = []
for day in day_list:
prop_list = []
for score in score_list:
prop = df[(df[day_col] == day) & (df[score_col] == score)].shape[0] / df[df[day_col] == day].shape[0]
prop_list.append(prop)
prop_day_list.append(prop_list)
# 将得分占比的转化为画图的格式
sub_list = []
for p in prop_day_list:
p_cumsum = list(np.cumsum(p))
p_cumsum = p_cumsum[:-1]
p_cumsum.insert(0, 0)
bar1_list = [1] * int(len(p_cumsum))
sub = [bar1_list[i] - p_cumsum[i] for i in range(len(p_cumsum))]
sub_list.append(sub)
array = np.array(sub_list)
stack_prop_list = [] # 面积图的y值
bar_prop_list = [] # 堆积柱状图的y
for i in range(len(score_list)):
bar_prop = array[:, i]
bar_prop_list.append(bar_prop)
stack_prop = []
for j in bar_prop:
a = j
b = j
stack_prop.append(a)
stack_prop.append(b)
stack_prop_list.append(stack_prop)
# 画图的x坐标轴
x_bar = list(range(1, len(day_list) * 2, 2)) # 堆积柱状图的x值
x_stack = [] # 面积图的x值
for i in x_bar:
c = i - 0.5
d = i + 0.5
x_stack.append(c)
x_stack.append(d)
# 绘图
fig = plt.figure(figsize=plt_size)
ax1 = fig.add_subplot(1, 1, 1)
# 先清除x轴的刻度
ax1.xaxis.set_major_formatter(plt.FuncFormatter(''.format))
ax1.set_xticks(range(1, len(day_list) * 2, 2))
# 将y轴的刻度设置为百分比形式
def to_percent(temp, position):
return '%1.0f' % (100 * temp) + '%'
plt.gca().yaxis.set_major_formatter(plt.FuncFormatter(to_percent))
# 自定义x轴刻度标签
for a, b in zip(x_bar, day_list):
ax1.text(a, -0.08, b, ha='center', va='bottom')
# 绘制面积图和堆积柱状图
for i, s in zip(range(len(day_list)), score_list):
ax1.stackplot(x_stack, stack_prop_list[i], alpha=0.25)
ax1.bar(x_bar, bar_prop_list[i], width=1, label='得分:{}'.format(s))
# 添加y轴刻度虚线
ax1.grid(True, 'major', 'y', ls='--', lw=.5, c='black', alpha=.3)
ax1.legend(loc='best')
plt.show()
# 计算评分的PSI
def score_psi(df1, df2, id_col, score_col, x, y, step=None):
"""
df1:建模样本的得分,包含用户id,得分
df2:上线样本的得分，包含用户id，得分
id_col:用户id字段名
score_col:得分的字段名
x:划分得分区间的left值
y:划分得分区间的right值
step:步长
return: 得分psi表
"""
df1['score_bin'] = pd.cut(df1[score_col], bins=np.arange(x, y, step))
model_score_group = df1.groupby('score_bin', as_index=False)[id_col].count().assign(
pct=lambda x: x[id_col] / x[id_col].sum()).rename(columns={id_col: '建模样本户数',
'pct': '建模户数占比'})
df2['score_bin'] = pd.cut(df2[score_col], bins=np.arange(x, y, step))
online_score_group = df2.groupby('score_bin', as_index=False)[id_col].count().assign(
pct=lambda x: x[id_col] / x[id_col].sum()).rename(columns={id_col: '线上样本户数',
'pct': '线上户数占比'})
score_compare = pd.merge(model_score_group, online_score_group, on='score_bin', how='inner')
score_compare['占比差异'] = score_compare['线上户数占比'] - score_compare['建模户数占比']
score_compare['占比权重'] = np.log(score_compare['线上户数占比'] / score_compare['建模户数占比'])
score_compare['Index'] = score_compare['占比差异'] * score_compare['占比权重']
score_compare['PSI'] = score_compare['Index'].sum()
return score_compare
# 评分比较分布图
def plot_score_compare(df, plt_size=None):
fig = plt.figure(figsize=plt_size)
x = df.score_bin
y1 = df.建模户数占比
y2 = df.线上户数占比
width = 0.3
plt.title('评分分布对比图')
plt.xlabel('得分区间')
plt.ylabel('用户占比')
plt.xticks(np.arange(len(x)) + 0.15, x)
plt.bar(np.arange(len(y1)), y1, width=width, color='seagreen', label='建模样本')
plt.bar(np.arange(len(y2)) + width, y2, width=width, color='hotpink', label='上线样本')
plt.legend()
return plt.show()
# 变量稳定度分析
def var_stable(score_result, df, var, id_col, score_col, bins):
"""
score_result:评分卡的score明细表，包含区间，用户数，用户占比,得分
var：分析的变量名
df:上线样本变量的得分，包含用户id,变量的value，变量的score
id_col:df的用户id字段名
score_col:df的得分字段名
bins:变量划分的区间
return :变量的稳定性分析表
"""
model_var_group = score_result.loc[score_result.col == var, ['bin', 'total', 'totalrate', 'score']].reset_index(
drop=True).rename(columns={'total': '建模用户数',
'totalrate': '建模用户占比',
'score': '得分'})
df['bin'] = pd.cut(df[score_col], bins=bins)
online_var_group = df.groupby('bin', as_index=False)[id_col].count().assign(
pct=lambda x: x[id_col] / x[id_col].sum()).rename(columns={id_col: '线上用户数',
'pct': '线上用户占比'})
var_stable_df = pd.merge(model_var_group, online_var_group, on='bin', how='inner')
var_stable_df = var_stable_df.iloc[:, [0, 3, 1, 2, 4, 5]]
var_stable_df['得分'] = var_stable_df['得分'].astype('int64')
var_stable_df['建模样本权重'] = np.abs(var_stable_df['得分'] * var_stable_df['建模用户占比'])
var_stable_df['线上样本权重'] = np.abs(var_stable_df['得分'] * var_stable_df['线上用户占比'])
var_stable_df['权重差距'] = var_stable_df['线上样本权重'] - var_stable_df['建模样本权重']
return var_stable_df本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
