(1)UDF的方式清理数据
import sys
reload(sys)
sys.setdefaultencoding('utf8')
import re
import json
from pyspark.sql import SparkSession
from pyspark.sql import Row
from pyspark.sql.functions import udf
from pyspark.sql.types import StringType,StructField,StructType
import copy
master_url = 'spark://sc-bd-10:7077'
spark = SparkSession.builder \
.master(master_url) \
.appName("saic_crawler_huangyu") \
.getOrCreate()
spark.conf.set("spark.driver.maxResultSize", "4g")
spark.conf.set("spark.sql.broadcastTimeout", 1200)
spark.conf.set("spark.sql.crossJoin.enabled", "true")
####重点从这看:
spark.sparkContext.addPyFile("clean_utils.py")
from clean_utils import name_clean, parse_date
parse_name_clean_udf = udf(name_clean, StringType()) #用udf的方式清洗数据(1),先要“注册”!
def load_basic_info():
basic_info_field_name_list = ["eid", "name", "reg_no", "zczj", "zczjbz", "type", "status", "clrq", "hzrq"]
basic_info_schema = StructType([StructField(field_name, StringType(), True) for field_name in basic_info_field_name_list])
# basic_info_schema = StructType([StructField(field_name, StringType(), True) for field_name in ["eid", "name", "province", "ent_type", "clrq", "hzrq", "zczj", "zczjbz", "status"]])
df_basic_info = spark.read.load("hdfs://sc-bd-10:9000/scdata/huangyu/pre_result/basic_info.csv", format="csv",schema=basic_info_schema, delimiter=',')
df_basic_info.createOrReplaceTempView("basic_info")
spark.sql("""select eid, name from basic_info """).createOrReplaceTempView("name_eid")
def clean_case_shixin():
shixin_schema = StructType([StructField(field_name, StringType(), True) for field_name in ["company_name", "sc_data_id", "publish_date"]])
df_shixin = spark.read.load("hdfs://sc-bd-10:9000/scdata/huangyu/case/shixin_company.csv", format="csv", schema=shixin_schema, delimiter=',') #不带表头,自动推断;
df_shixin = df_shixin.withColumn("company_name", parse_name_clean_udf(df_shixin['company_name'])) #用udf的方式清洗数据(2)
df_shixin = df_shixin.withColumn("publish_date", parse_date_udf(df_shixin['publish_date']))
df_shixin.createOrReplaceTempView("shixin")
df_shixin_diff = spark.sql("""
select max(company_name) as company_name, max(sc_data_id) as sc_data_id, max(publish_date) as publish_date
from shixin
GROUP BY company_name, sc_data_id
""")
df_shixin_diff.createOrReplaceTempView("shixin_diff")
方式2,直接导入函数进行清洗:
@staticmethod
def clean_row_name(row, raw_zhixing_info_field):
row_dict = dict([(k, row[k]) for k in raw_zhixing_info_field])
row_dict["company_name"] = name_clean(row_dict["company_name"])
return row_dict
def rdd_map_reduce_demo(self):
# 注意 rdd 的时候需要将map reduce中的self 去掉
clean_row_name = self.clean_row_name
raw_zhixing_info_field = self.raw_zhixing_info_field
self.df_zhixing_info_sample = self.df_zhixing_info_sample \
.rdd \
.map(lambda row: Row(**clean_row_name(row, raw_zhixing_info_field))) \
.filter(lambda row: len(row["company_name"]) >= 5) \
.toDF(self.zhixing_info_schema)
self.df_zhixing_info_sample.createOrReplaceTempView("zhixing_info_sample")
方式3,用spark.sql清洗;
python zip与zip*区别:
Python中的zip()与*zip()函数详解
这里写代码片
zip()函数的定义
从参数中的多个迭代器取元素组合成一个新的迭代器;
返回:
返回一个zip对象,其内部元素为元组;可以转化为列表或元组;
传入参数:
元组、列表、字典等迭代器。
zip()函数的用法
当zip()函数中只有一个参数时
zip(iterable)从iterable中依次取一个元组,组成一个元组。
示例:
## zip()函数单个参数
list1 = [1, 2, 3, 4]
tuple1 = zip(list1)
# 打印zip函数的返回类型
print("zip()函数的返回类型:\n", type(tuple1))
# 将zip对象转化为列表
print("zip对象转化为列表:\n", list(tuple1))
输出:
zip()函数的返回类型:
<class 'zip'>
zip对象转化为列表:
[(1,), (2,), (3,), (4,)]
当zip()函数有两个参数时
zip(a,b)zip()函数分别从a和b依次各取出一个元素组成元组,再将依次组成的元组组合成一个新的迭代器--新的zip类型数据。
注意:
要求a与b的维数相同,当两者具有相同的行数与列数时,正常组合对应位置元素即可;
当a与b的行数或列数不同时,取两者结构中最小的行数和列数,依照最小的行数和列数将对应位置的元素进行组合;这时相当于调用itertools.zip_longest(*iterables)函数。
举例:
m = [[1,2,3], [4,5,6], [7,8,9]]
n = [[2,2,2], [3,3,3], [4,4,4]]
p = [[2,2,2], [3,3,3,]
*zip函数:
*zip()函数是zip()函数的逆过程,将zip对象变成原先组合前的数据。
代码示例:
## *zip()函数
print('=*'*10 + "*zip()函数" + '=*'*10)
m = [[1, 2, 3], [4, 5, 6], [7, 8, 9]]
n = [[2, 2, 2], [3, 3, 3], [4, 4, 4]]
print("*zip(m, n)返回:\n", *zip(m, n))
m2, n2 = zip(*zip(m, n))
# 若相等,返回True;说明*zip为zip的逆过程
print(m == list(m2) and n == list(n2))
输出:
*zip(m, n)返回:
([1, 2, 3], [2, 2, 2]) ([4, 5, 6], [3, 3, 3]) ([7, 8, 9], [4, 4, 4])
True
(4)python中的*args和**kwargs
先来看一个例子:
复制代码
1 >>> def foo(*args, **kwargs):
2 print 'args =', args
3 print 'kwargs = ', kwargs
4 print '-----------------------'
5
6
7 >>> if __name__ == '__main__':
8 foo(1, 2, 3, 4)
9 foo(a=1, b=2, c=3)
10 foo(1,2,3,4, a=1, b=2, c=3)
11 foo('a', 1, None, a=1, b='2', c=3)
复制代码
其输出结果如下:
复制代码
1 args = (1, 2, 3, 4)
2 kwargs = {}
3 -----------------------
4 args = ()
5 kwargs = {'a': 1, 'c': 3, 'b': 2}
6 -----------------------
7 args = (1, 2, 3, 4)
8 kwargs = {'a': 1, 'c': 3, 'b': 2}
9 -----------------------
10 args = ('a', 1, None)
11 kwargs = {'a': 1, 'c': 3, 'b': '2'}
12 -----------------------
复制代码
从以上例子可以看出,这两个是python中的可变参数。*args表示任何多个无名参数,它是一个tuple;**kwargs表示关键字参数,它是一个 dict。并且同时使用*args和**kwargs时,*args参数列必须要在**kwargs前,像foo(a=1, b='2', c=3, a', 1, None, )这样调用的话,会提示语法错误“SyntaxError: non-keyword arg after keyword arg”。如同所示: