python导入本地dat数据 python导入本地库

1.安装mysql，本机使用的mysql安装包是 mysql-installer-community-5.7.32.0.msi 

2.下载mysql客户端，本机使用客户端是heidisql，连接到本地数据库打开，建表。（建表后要刷新数据库）

3.vscode编写python脚本 

import pandas as pd
import pymysql
import time
import datetime

config = {'host': '127.0.0.1','port': 3306,'user': 'root','passwd': '123456','db':'qxy','charset':'utf8mb4'}
conn = pymysql.connect(**config)
conn.autocommit(1)
cursor = conn.cursor()



#mysql导入数据

def into_mysql(sql,val):
    try:
        # 批量插入纪录
        cursor.execute(sql,val)
    except:
        import traceback
        traceback.print_exc()
        # 发生错误时会滚
        conn.rollback()

#修改数据
def update_mysql(sql,val):
    try:
        # 批量插入纪录
        cursor.execute(sql,val)
    except:
        import traceback
        traceback.print_exc()
        # 发生错误时会滚
        conn.rollback()



#查询数据库已有数据
def select_sql(query_data):
    cursor.execute(query_data)
    countAll = cursor.fetchall()
    data = pd.DataFrame(list(countAll))
    return data

#mysql导入excel数据

def in_mysql(val):
    try:
        # 批量插入纪录
        sql = "INSERT INTO order_3c_nj(ord_item_id,ord_id,sbmt_time,bill_tp_cd,gds_cd,str_cd,chnl_cd,vendor_cd,busi_chnl_cd,pay_amt,sal_qty,ord_src_cd,stl_cd,dlvr_plant_cd,dlvr_plant_nm,send_city_nm,gds_nm,ndtl_busi_chnl_cd,wl_box_code_qty,wl_price1,wl_price2,weight,volume,delivery_hour,creat_time) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
        cursor.execute(sql,val)
    except:
        import traceback
        traceback.print_exc()
        # 发生错误时会滚
        conn.rollback()

#导入订单数据
def excel_into_mysql(excel_data):
    len_excel_data = len(excel_data)
    creat_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())#创建时间
    for i in range(len_excel_data):
        excel_data_1 = list(excel_data.iloc[i].values)
        excel_data_1.extend([creat_time])
        excel_data_2 = tuple(excel_data_1)
        in_mysql(excel_data_1)
        print('进度: {:.2%}'.format(i/len_excel_data))


excel_data = pd.read_csv(r"F:\\0326.csv",encoding = "utf-8",delimiter="\t")


excel_into_mysql(excel_data)



#导入商品价值数据
def gds_price(sql,data):
    len_excel_data = len(data)
    creat_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())#创建时间
    for i in range(len_excel_data):
        excel_data_1 = list(data.iloc[i].values)
        excel_data_1.extend([creat_time])
        excel_data_2 = tuple(excel_data_1)
        into_mysql(sql,excel_data_2)
        print('进度: {:.2%}'.format(i/len_excel_data))
    

sql = "INSERT INTO goods_price(gds_cd,gds_price1,gds_price2,creat_time) VALUES (%s,%s,%s,%s)"
data = pd.read_excel(r"C:\Users\86181\Desktop\文件\15消费电子前置仓\02计算数据\晓雨\sku成本.xlsx")
i=1
gds_price(sql,data)


#时间日期、周转换
def data_week():
    data_time_sql = "select id,sbmt_time from order_3c_nj where week IS NULL"
    data_time = select_sql(data_time_sql)
    data_time_1 = data_time
    data_time_2 = pd.concat([data_time_1, data_time_1[1].str.split(' ', expand=True)], axis=1)
    data_time_2.columns=['id','date_time','date','time']
    data_time_3 = pd.concat([data_time_2, data_time_2["date"].str.split('-', expand=True)], axis=1)    
    data_time_len = len(data_time_3)
    for i in range(data_time_len):
        id = str(data_time_3.iloc[i,0])
        sbmt_time = data_time_3.iloc[i,1]#创建日期时间
        date = data_time_3.iloc[i,2]#创建日期
        time =  data_time_3.iloc[i,3]#创建时间
        year =  data_time_3.iloc[i,4]#创建年
        mo =  data_time_3.iloc[i,5]#创建月
        day =  data_time_3.iloc[i,6]#创建日
        week = datetime.date(int(year),int(mo),int(day)).isocalendar()[1]
        sql = "update order_3c_nj set date = %s,week = %s,time = %s WHERE id = '"+id+"'"
        val = (date,week,time)
        update_mysql(sql,val)
        print('进度: {:.2%}'.format(i/data_time_len))

data_week()




#重量、采购金额转换
def weight_price():
    weight_sql = "select id,gds_cd,sal_qty,weight from order_3c_nj where price IS NULL"
    weight = select_sql(weight_sql)
    price = pd.read_excel(r"C:\Users\86181\Desktop\文件\15消费电子前置仓\02计算数据\晓雨\sku成本 - 副本.xlsx")
    weight[2]= weight[2].astype('float')
    weight[3]= weight[3].astype('float')
    weight[1]= weight[1].astype('str')
    price["sku"]= price["sku"].astype('str')
    weight["单位重量"] = weight[3]/weight[2]
    weight_price = pd.merge(weight,price, how='left', left_on=[1], right_on=["sku"])
    weight_price["行金额"] = weight_price[2]*weight_price["中位数"]
    weight_price_1 = weight_price[[0,"单位重量","中位数","行金额"]]
    weight_price_1_len = len(weight_price_1)
    for i in range(weight_price_1_len):
        id = str(weight_price_1.iloc[i,0])
        sku_weight = weight_price_1.iloc[i,1]#创建日期时间
        sku_price = weight_price_1.iloc[i,2]#创建日期
        price =  weight_price_1.iloc[i,3]#创建时间
        sql = "update order_3c_nj set sku_weight = %s,sku_price = %s,price = %s WHERE id = '"+id+"'"
        val = (sku_weight,sku_price,price)
        update_mysql(sql,val)
        print('进度: {:.4%}'.format(i/weight_price_1_len))



#线上数据处理抽取
def online_order():
    order = pd.read_csv(r"C:\Users\86181\Desktop\文件\15消费电子前置仓\02计算数据\结果数据\非10渠道订单数据汇总.csv")
    order_x = order.sample(frac=0.16, replace=True, random_state=1)
    order_x.to_excel(r"C:\Users\86181\Desktop\文件\15消费电子前置仓\02计算数据\结果数据\非10渠道订单数据汇总1.xlsx")



#线上成本计算
def all_data():
    all_sql = "select * from order_3c_nj"
    all_data = select_sql(all_sql)
    target_data = pd.read_excel(r"C:\Users\86181\Desktop\文件\15消费电子前置仓\02计算数据\结果数据\非10渠道订单数据汇总1.xlsx")

    price["sku"]= price["sku"].astype('str')
    all_data[2] = all_data[2].astype('str')
    target_data["订单号"] = target_data["订单号"].astype('str')
    target_data_price = pd.merge(all_data,target_data, how='right', left_on=[2], right_on=["订单号"])

    target_data_price.to_excel(r"C:\Users\86181\Desktop\文件\15消费电子前置仓\02计算数据\结果数据\O2O订单数据汇总.xlsx")