应用场景:实时仪表盘(即大屏),每个集团下有多个mall,每个mall下包含多家shop,需实时计算集团下各mall及其shop的实时销售分析(区域、业态、店铺TOP、总销售额等指标)并提供可视化展现,之前时候一直在Strom实现,现在改为Spark2.3.2实现。

 

1、数据源:首先数据源来自于MQ、Socket、Flume和DFS等,一般Kafka、RocketMQ等居多,此处示例代码用的是RocketMQ;

2、实时计算框架:Storm(实时计算,Spout发射Tuple到各个Bolt,来一条即处理一条,一百毫秒以内的延迟)、SparkStreaming(准实时计算,基于微批次的实时计算,即一定时间段内的micro batch,每个micro batch的结构为DStream,底层是RDD);

3、此处Spark Streaming准实时处理应用流程:1、RocketMQ --> 2、SparkStreaming --> 3、SparkSQL(Parquet) --> 4、Redis(大屏使用) && HDFS(Hive数仓ODS层,ODS->DW[DWD-DWS]-DM);

4、系统设计

spark ess 热数据 spark对数据进行实时展示_java

 

5、代码如下(4.3是Spark Streaming实现,复制粘贴即可运行):

  • 5.1、RocketMQConfig类(用于配置和构建MQ消费者)
  • 5.2、SparkStreaming自定义RocketMQ接收器(可靠的接收器)
  • 5.3、SparkStreaming销售分析实时计算
  • 5.4、账单实体类
  • 5.5、BCD即账单裁剪后的实体类(减少数据量传输即降低节点间的序列化和反序列化开销)
  • 5.6、SparkSQL销售分析业务实现类

5.1、RocketMQConfig类(用于配置和构建MQ消费者)

package com.mengyao.graph.etl.apps.commons.datasource.mq.receiver;

import org.apache.commons.lang.Validate;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.remoting.common.RemotingUtil;

import java.util.HashMap;
import java.util.UUID;

/**
 * RocketMQConfig for Consumer
 * @author mengyao
 * 
 */
public class RocketMQConfig {
    
    // ------- the following is for common usage -------
    /**
     * RocketMq name server address
     */
    public static final String NAME_SERVER_ADDR = "nameserver.addr"; // Required

    public static final String CLIENT_NAME = "client.name";

    public static final String CLIENT_IP = "client.ip";
    public static final String DEFAULT_CLIENT_IP = RemotingUtil.getLocalAddress();

    public static final String CLIENT_CALLBACK_EXECUTOR_THREADS = "client.callback.executor.threads";
    public static final int DEFAULT_CLIENT_CALLBACK_EXECUTOR_THREADS = Runtime.getRuntime().availableProcessors();;

    public static final String NAME_SERVER_POLL_INTERVAL = "nameserver.poll.interval";
    public static final int DEFAULT_NAME_SERVER_POLL_INTERVAL = 30000; // 30 seconds

    public static final String BROKER_HEART_BEAT_INTERVAL = "brokerserver.heartbeat.interval";
    public static final int DEFAULT_BROKER_HEART_BEAT_INTERVAL = 30000; // 30 seconds


    // ------- the following is for push consumer mode -------
    /**
     * RocketMq consumer group
      */
    public static final String CONSUMER_GROUP = "consumer.group"; // Required

    /**
     * RocketMq consumer topic
     */
    public static final String CONSUMER_TOPIC = "consumer.topic"; // Required

    public static final String CONSUMER_TAG = "consumer.tag";
    public static final String DEFAULT_TAG = "*";

    public static final String CONSUMER_OFFSET_RESET_TO = "consumer.offset.reset.to";
    public static final String CONSUMER_OFFSET_LATEST = "latest";
    public static final String CONSUMER_OFFSET_EARLIEST = "earliest";
    public static final String CONSUMER_OFFSET_TIMESTAMP = "timestamp";

    public static final String CONSUMER_MESSAGES_ORDERLY = "consumer.messages.orderly";

    public static final String CONSUMER_OFFSET_PERSIST_INTERVAL = "consumer.offset.persist.interval";
    public static final int DEFAULT_CONSUMER_OFFSET_PERSIST_INTERVAL = 5000; // 5 seconds

    public static final String CONSUMER_MIN_THREADS = "consumer.min.threads";
    public static final int DEFAULT_CONSUMER_MIN_THREADS = 20;

    public static final String CONSUMER_MAX_THREADS = "consumer.max.threads";
    public static final int DEFAULT_CONSUMER_MAX_THREADS = 64;


    // ------- the following is for reliable Receiver -------
    public static final String QUEUE_SIZE = "spout.queue.size";
    public static final int DEFAULT_QUEUE_SIZE = 500;

    public static final String MESSAGES_MAX_RETRY = "spout.messages.max.retry";
    public static final int DEFAULT_MESSAGES_MAX_RETRY = 3;

    public static final String MESSAGES_TTL = "spout.messages.ttl";
    public static final int DEFAULT_MESSAGES_TTL = 300000;  // 5min


    // ------- the following is for pull consumer mode -------

    /**
     * Maximum rate (number of records per second) at which data will be read from each RocketMq partition ,
     * and the default value is "-1", it means consumer can pull message from rocketmq as fast as the consumer can.
     * Other that, you also enables or disables Spark Streaming's internal backpressure mechanism by the config
     *  "spark.streaming.backpressure.enabled".
     */
    public static final String  MAX_PULL_SPEED_PER_PARTITION = "pull.max.speed.per.partition";

    /**
     * To pick up the consume speed, the consumer can pull a batch of messages at a time. And the default
     * value is "32"
     */
    public static final String PULL_MAX_BATCH_SIZE = "pull.max.batch.size";

    /**
     * pull timeout for the consumer, and the default time is "3000".
     */
    public static final String PULL_TIMEOUT_MS = "pull.timeout.ms";

    // the following configs for consumer cache
    public static final String PULL_CONSUMER_CACHE_INIT_CAPACITY = "pull.consumer.cache.initialCapacity";
    public static final String PULL_CONSUMER_CACHE_MAX_CAPACITY = "pull.consumer.cache.maxCapacity";
    public static final String PULL_CONSUMER_CACHE_LOAD_FACTOR = "pull.consumer.cache.loadFactor";


    public static void buildConsumerConfigs(HashMap<String, String> props, DefaultMQPushConsumer consumer) {
        buildCommonConfigs(props, consumer);

        String group = props.get(CONSUMER_GROUP);
        Validate.notEmpty(group);
        consumer.setConsumerGroup(group);

        consumer.setPersistConsumerOffsetInterval(getInteger(props,
                CONSUMER_OFFSET_PERSIST_INTERVAL, DEFAULT_CONSUMER_OFFSET_PERSIST_INTERVAL));
        consumer.setConsumeThreadMin(getInteger(props,
                CONSUMER_MIN_THREADS, DEFAULT_CONSUMER_MIN_THREADS));
        consumer.setConsumeThreadMax(getInteger(props,
                CONSUMER_MAX_THREADS, DEFAULT_CONSUMER_MAX_THREADS));

        String initOffset = props.get(CONSUMER_OFFSET_RESET_TO) != null ? props.get(CONSUMER_OFFSET_RESET_TO) : CONSUMER_OFFSET_LATEST;
        switch (initOffset) {
            case CONSUMER_OFFSET_EARLIEST:
                consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
                break;
            case CONSUMER_OFFSET_LATEST:
                consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
                break;
            case CONSUMER_OFFSET_TIMESTAMP:
                consumer.setConsumeTimestamp(initOffset);
                break;
            default:
                consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
        }

        String topic = props.get(CONSUMER_TOPIC);
        Validate.notEmpty(topic);
        try {
            consumer.subscribe(topic, props.get(CONSUMER_TAG) != null ? props.get(CONSUMER_TAG) : DEFAULT_TAG);
        } catch (MQClientException e) {
            throw new IllegalArgumentException(e);
        }
    }

    public static void buildCommonConfigs(HashMap<String, String> props, ClientConfig client) {
        String namesvr = props.get(NAME_SERVER_ADDR);
        Validate.notEmpty(namesvr);
        client.setNamesrvAddr(namesvr);

        client.setClientIP(props.get(CLIENT_IP) != null ? props.get(CLIENT_IP) : DEFAULT_CLIENT_IP);
        // use UUID for client name by default
        String defaultClientName = UUID.randomUUID().toString();
        client.setInstanceName(props.get(CLIENT_NAME) != null ? props.get(CLIENT_NAME) : defaultClientName);

        client.setClientCallbackExecutorThreads(getInteger(props,
                CLIENT_CALLBACK_EXECUTOR_THREADS, DEFAULT_CLIENT_CALLBACK_EXECUTOR_THREADS));
        client.setPollNameServerInterval(getInteger(props,
                NAME_SERVER_POLL_INTERVAL, DEFAULT_NAME_SERVER_POLL_INTERVAL));
        client.setHeartbeatBrokerInterval(getInteger(props,
                BROKER_HEART_BEAT_INTERVAL, DEFAULT_BROKER_HEART_BEAT_INTERVAL));
    }

    public static int getInteger(HashMap<String, String> props, String key, int defaultValue) {
        return Integer.parseInt(props.get(key) != null ? props.get(key) : String.valueOf(defaultValue));
    }

    public static boolean getBoolean(HashMap<String, String> props, String key, boolean defaultValue) {
        return Boolean.parseBoolean(props.get(key) != null ? props.get(key) : String.valueOf(defaultValue));
    }

}

4.2、SparkStreaming自定义RocketMQ接收器(可靠的接收器)

package com.mengyao.graph.etl.apps.commons.datasource.mq.receiver;

import org.apache.activemq.util.ByteArrayInputStream;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.Validate;
//import org.apache.commons.lang3.SerializationUtils;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.MQPushConsumer;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.common.message.MessageExt;
import org.apache.spark.storage.StorageLevel;
import org.apache.spark.streaming.receiver.Receiver;

import com.gooagoo.entity.mq.bill.MQBillMessage;
import com.mengyao.graph.etl.apps.commons.beans.ods.fmt.BillInfoFmt;
import com.mengyao.graph.etl.apps.commons.beans.ods.fmt.ConvertTool;
import com.mengyao.graph.etl.apps.dashboard.beans.BCD;

import java.io.IOException;
import java.io.ObjectInputStream;
import java.util.HashMap;
import java.util.List;

/**
 * RocketMQ Receiver
 * @author mengyao
 *
 */
public class RocketMQReceiver extends Receiver<BCD> {
    
    /**
     * 
     */
    private static final long serialVersionUID = 2274826339951693341L;
    private MQPushConsumer consumer;
    private boolean ordered;
    private HashMap<String, String> conf;
    

    public RocketMQReceiver(HashMap<String, String> conf, StorageLevel storageLevel) {
        super(storageLevel);
        this.conf = conf;
    }

    @Override
    public void onStart() {
        Validate.notEmpty(conf, "Consumer properties can not be empty");
        ordered = RocketMQConfig.getBoolean(conf, RocketMQConfig.CONSUMER_MESSAGES_ORDERLY, true);
        consumer = new DefaultMQPushConsumer();
        RocketMQConfig.buildConsumerConfigs(conf, (DefaultMQPushConsumer)consumer);
        if (ordered) {
            consumer.registerMessageListener(new MessageListenerOrderly() {
                @Override
                public ConsumeOrderlyStatus consumeMessage(List<MessageExt> msgs, ConsumeOrderlyContext context) {
                    if (process(msgs)) {
                        return ConsumeOrderlyStatus.SUCCESS;
                    } else {
                        return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT;
                    }
                }
            });
        } else {
            consumer.registerMessageListener(new MessageListenerConcurrently() {
                @Override
                public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs, ConsumeConcurrentlyContext context) {
                    if (process(msgs)) {
                        return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
                    } else {
                        return ConsumeConcurrentlyStatus.RECONSUME_LATER;
                    }
                }
            });
        }
        try {
            consumer.start();
            System.out.println("==== RocketMQReceiver start ====");
        } catch (MQClientException e) {
            e.printStackTrace();
            throw new RuntimeException(e);
        }
    }

    public boolean process(List<MessageExt> msgs) {
        if (msgs.isEmpty()) {
            System.out.println("==== Msgs is null! ====");
            return true;
        }
        System.out.println("==== receiver msgs: "+msgs.size()+" record. ====");
        try {
            for (MessageExt messageExt : msgs) {
                //MQBillMessage message = SerializationUtils.deserialize(messageExt.getBody());
                MQBillMessage message = deserialize(messageExt.getBody());
                if (null != message) {
                    BillInfoFmt billFmt = ConvertTool.convertGagBillToOdsBillFmt(message.getData());
                    if (validBillType(billFmt)) {
               /**
                * this.store(BCD) 简单的接收一条存储一条,缺点是没有确认机制,不具备容错保证,会出现数据丢失。优点则是效率更高。
                * this.store(Iterator<BCD>)阻塞调用,当接收到的记录都存储到Spark后才会确认成功,当接收方采用复制(默认存储级别为复制)则在复制完成后确认成功,但在缓冲中的数据不被保证,会被重新发送。具备容错保证,可确保数据0丢失。
                */
                        this.store(new BCD(billFmt.getId(), billFmt.getReceivableAmount(), billFmt.getSaleTime(), billFmt.getBillType(), billFmt.getShopId(), billFmt.getShopEntityId()));
                    }
                    billFmt=null;
                    message.setData(null);
                    message = null;
                } else {
                    System.out.println("==== receiver msg is:"+messageExt.getBody()+", deserialize faild. ====");
                }
            }
            return true;
        } catch (Exception e) {
            e.printStackTrace();
            return false;
        }
    }

    /**
     * 验证账单类型是否为1、3、6
     * @param bean
     * @return
     */
    static boolean validBillType(BillInfoFmt bean) {
        if (null == bean) {
            System.out.println("==== BillBean is null! ====");
            return false;
        }
        String billType = bean.getBillType();
        if (StringUtils.isEmpty(billType)) {
            System.out.println("==== BillBean.billType is null! ====");
            return false;
        }
        String billTypeTrim = billType.trim();
        return billTypeTrim.equals("1") || billType.equals("3") || billType.equals("6");
    }
    
    @Override
    public void onStop() {
        consumer.shutdown();
        System.out.println("==== RocketMQReceiver stop ====");
    }
    
    private static MQBillMessage deserialize(byte[] body) throws Exception {
        if (body == null) {
            throw new IllegalArgumentException("The byte array must not be null");
        }
        MQBillMessage message = null;
        ObjectInputStream in = null;
        ByteArrayInputStream bais = null;
        try {
            bais = new ByteArrayInputStream(body);
            in = new ObjectInputStream(bais);
            message = (MQBillMessage) in.readObject();
        } catch (final ClassNotFoundException ex) {
            ex.printStackTrace();
            throw ex;
        } catch (final IOException ex) {
            ex.printStackTrace();
            throw ex;
        } finally {
            try {
                if (bais != null) {
                    bais.close();
                }
                if (in != null) {
                    in.close();
                }
            } catch (final IOException ex) {
                // ignore close exception
            }
        }
        return message;
    }

}

 

4.3、SparkStreaming销售分析实时计

package com.mengyao.graph.etl.apps.commons.datasource.bill;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.Validate;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.StorageLevels;
import org.apache.spark.api.java.function.Function0;
import org.apache.spark.broadcast.Broadcast;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SaveMode;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.util.LongAccumulator;
import org.apache.spark.util.SizeEstimator;
import org.apache.spark.streaming.Time;

import com.mengyao.graph.etl.apps.commons.beans.dim.RuianMall;
import com.mengyao.graph.etl.apps.commons.datasource.mq.receiver.RocketMQConfig;
import com.mengyao.graph.etl.apps.commons.datasource.mq.receiver.RocketMQReceiver;
import com.mengyao.graph.etl.apps.dashboard.beans.BCD;
import com.mengyao.graph.etl.apps.dashboard.service.SaleAnalysisService;

/**
 * BillConsumer 大屏实时计算
 *         1、hdfs dfs -rm -r hdfs://bd001:8020/data/consumer/bill/checkpoint/*
 *         2、hdfs dfs -rm -r hdfs://bd001:8020/data/dashboard/ruian/sdt=curTime
 *         3、spark-submit --class com.mengyao.graph.etl.apps.commons.datasource.bill.BillConsumer --master yarn --deploy-mode cluster --driver-memory 4g --executor-cores 6 --executor-memory 5g --queue default --verbose data-graph-etl.jar
 * 
 * @author mengyao
 *
 */
public class BillConsumer {

    private static final ThreadLocal<SimpleDateFormat> FORMATTER_YMD = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyyMMdd"));
    private static final ThreadLocal<SimpleDateFormat> FORMATTER_YMDHMS = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyyMMddHHmmss"));
    private static final ThreadLocal<SimpleDateFormat> FORMATTER_YMDHMSS = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyyMMddHHmmssSSS"));
    private static final String BASE_PATH = "hdfs://bd001:8020/data/dashboard/ruian/";
    private static final String TMP_PATH = "/merge";
    private static String appName = "BillConsumer";
    private static String logLevel = "ERROR";

    
    public static void main(String[] args) {
        args = new String[] {"hdfs://bd001:8020/data/consumer/bill/checkpoint", "10", "base.mq.goo.com:9877", "Bill", "bill_dev_group_00013"};
        if (args.length < 5) {
            System.err.println("Usage: "+appName+" <checkpointDir> <milliseconds> <namesrvAddr> <groupId> <topic>");
            System.exit(1);
        }
        
        String checkPointDir = args[0];
        int second = Integer.parseInt(args[1]);
        String namesrv = args[2];
        Validate.notNull(namesrv, "RocketMQ namesrv not null!");
        String topic = args[3];
        Validate.notNull(topic, "RocketMQ topic not null!");
        String group = args[4];
        Validate.notNull(group, "RocketMQ group not null!");
        
        Function0<JavaStreamingContext> jscFunc = () -> createJavaSparkStreamingContext(checkPointDir, second, namesrv, topic, group);
        JavaStreamingContext jssc = JavaStreamingContext.getOrCreate(checkPointDir, jscFunc, new Configuration());
        jssc.sparkContext().setLogLevel(logLevel);
        
        try {
            jssc.start();
            jssc.awaitTermination();
        } catch (InterruptedException e) {
            e.printStackTrace();
        } finally {
            jssc.stop(false, true);//关闭StreamingContext但不关闭SparkContext,同时等待数据处理完成
        }
    }
    
    /**
     * 获取当前时间yyyyMMdd,匹配账单数据saleTime
     * @param curTime
     * @return
     */
    static String getCurrentDate(long curTime) {
        return FORMATTER_YMD.get().format(new Date(curTime));
    }
    
    /**
     * 打印bill RDD<BCD>的分区及占用空间大小,debug方法
     * @param rdd
     * @param time
     */
    static void printStream(JavaRDD<BCD> rdd, Time time) {rdd.id();
        System.out.println("==== time: "+FORMATTER_YMDHMSS.get().format(new Date(time.milliseconds()))+", rdd: partitions="+rdd.getNumPartitions()+", space="+SizeEstimator.estimate(rdd)/1048576+"mb");
    }

    /**
     * 合并parquet小文件
     * @param session
     * @param dfsDir
     */
    static void mergeSmallFiles(Dataset<Row> fulls, SparkSession session, String dfsDir) {
        fulls.coalesce(1).write().mode(SaveMode.Overwrite).parquet(BASE_PATH+TMP_PATH);
        session.read().parquet(BASE_PATH+TMP_PATH).coalesce(1).write().mode(SaveMode.Overwrite).parquet(dfsDir);
    }
    
    /**
     * 创建DFS Dir
     * @param session
     * @param dfsDir
     */
    static void mkDfsDir(SparkSession session, String dfsDir) {
        FileSystem fs = null;
        try {
            fs = FileSystem.get(session.sparkContext().hadoopConfiguration());
            Path path = new Path(dfsDir);
            if(!fs.exists(path)) {
                fs.mkdirs(path);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                if (null != fs) {fs.close();}
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
    
    /**
     * 容错Driver
     * @param checkPointDir
     * @param second
     * @param namesrv
     * @param topic
     * @param group
     * @return
     */
    static JavaStreamingContext createJavaSparkStreamingContext(String checkPointDir, int second, String namesrv, String topic, String group) {
        try {
            SparkConf conf = new SparkConf()
                    //==== Enable Back Pressure
                    .set("spark.streaming.backpressure.enabled", "true")//启用被压机制
                    .set("spark.streaming.backpressure.initialRate", "50000")//初始接收数据条数,如该值为空时使用spark.streaming.backpressure.initialRate为默认值
                    .set("spark.streaming.receiver.maxRate", "100")//每秒接收器可接收的最大记录数
                    //==== Enable Dynamic Resource Allocation
                    .set("spark.dynamicAllocation.enabled", "false")//禁用spark动态资源分配
                    .set("spark.streaming.dynamicAllocation.enabled", "true")//启用SparkStreaming动态资源分配,该配置和spark动态资源分配存在冲突,只能使用一个
                    .set("spark.streaming.dynamicAllocation.minExecutors", "2")//启用SparkStreaming动态资源分配后的给应用使用的最小executor数
                    .set("spark.streaming.dynamicAllocation.maxExecutors", "3")//启用SparkStreaming动态资源分配后的给应用使用的最大executor数
                    //==== Spark Streaming Parallelism and WAL
                    .set("spark.streaming.concurrentJobs", "1")//并行job数量,默认1
                    .set("spark.streaming.blockInterval", "5000")//SparkStreaming接收器接收数据后5000毫秒生成block,默认200毫秒
                    .set("spark.streaming.receiver.writeAheadLog.enable", "true")//开启SparkStreaming接收器的WAL来确保接收器实现至少一次的容错语义
                    .set("spark.streaming.driver.writeAheadLog.allowBatching", "true")//driver端WAL
                    .set("spark.streaming.driver.writeAheadLog.batchingTimeout", "15000")
                    //==== Hive on Spark
                    .set("hive.execution.engine", "spark")//设置hive引擎为spark,hdp-2.6.1.0默认支持tez、mr,可通过应用级别配置使用Hive on Spark
                    .set("hive.enable.spark.execution.engine", "true")//启用Hive on Spark
                    .set("spark.driver.extraJavaOptions", "-Dhdp.version=2.6.1.0-129")//hdp-2.6.1.0中要求Hive on Spark必须指定driver的jvm参数
                    .set("spark.yarn.am.extraJavaOptions", "-Dhdp.version=2.6.1.0-129")//hdp-2.6.1.0中要求Hive on Spark必须指定ApplicationMaster的jvm参数
                    //==== Hive Merge Small Files
                    .set("hive.metastore.uris", "thrift://bd001:9083")//hive ThriftServer
                    .set("hive.merge.sparkfiles", "true")//合并spark小文件
                    .set("hive.merge.mapfiles", "true")//在只有map任务的作业结束时合并小文件。
                    .set("hive.merge.mapredfiles", "true")//在mapreduce作业结束时合并小文件。
                    .set("hive.merge.size.per.task", "268435456")//作业结束时合并文件的大小。
                    .set("hive.merge.smallfiles.avgsize", "100000")//当作业的平均输出文件大小小于此数量时,Hive将启动另一个map-reduce作业,以将输出文件合并为更大的文件。如果hive.merge.mapfiles为true,则仅对仅map作业执行此操作;对于hive.merge.mapredfiles为true,仅对map-reduce作业执行此操作。
                    //==== Spark SQL Optimizer
                    .set("spark.sql.warehouse.dir", "hdfs://bd001:8020/apps/hive/warehouse")//SparkSQL依赖的hive仓库地址
                    .set("spark.sql.files.maxPartitionBytes", "134217728")//SparkSQL读取文件数据时打包到一个分区的最大字节数
                    .set("spark.sql.files.openCostInBytes", "134217728")//当SparkSQL读取的文件中有大量小文件时,小于该值的文件将被合并处理,默认4M,此处设置为128M
                    .set("spark.sql.shuffle.partitions", "600")//SparkSQL运行shuffle的并行度
                    .set("spark.sql.autoBroadcastJoinThreshold", "67108864")//设置为64M,执行join时自动广播小于该值的表,默认10M
                    //==== Spark Core Configure
                    .set("spark.rdd.compress","true")//开启rdd压缩以节省内存
                    .set("spark.default.parallelism", "600")//并行任务数
                    .set("spark.rpc.askTimeout", "300")//spark rpc超时时间
                    .set("spark.eventLog.enabled", "true")//开启eventLog
                    //==== Application Configure
                    .set("spark.app.name", appName)//Spark Application名称
                    .set("spark.master", "yarn")//运行模式为Spark on YARN
                    .set("spark.deploy.mode", "cluster")//部署模式为yarn-cluster
                    .set("spark.driver.memory", "4g")//driver内存4g
                    .set("spark.driver.cores", "1")//driver计算vcore数量为1
                    .set("spark.executor.memory", "5g")//executor内存为4g
                    .set("spark.executor.heartbeatInterval", "20000")//executor心跳间隔20秒,默认10秒
                    .set("spark.yarn.archive", "hdfs://bd001:8020/hdp/apps/2.6.1.0-129/spark2")//spark依赖jar存档到hdfs指定位置
                    .set("spark.executor.extraJavaOptions", "-XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+UseConcMarkSweepGC")//打印GC详情和耗时
                    .set("spark.jars", "/usr/hdp/2.6.1.0-129/sqoop/lib/mysql-connector-java.jar")//如果使用了数据库驱动,则通过此配置即可
                    //==== Serialized Configure
                    .set("spark.kryoserializer.buffer", "512k")//默认64k,设置为256k
                    .set("spark.kryoserializer.buffer.max", "256m")//默认64m,设置为256m
                    .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")//使用kryo序列化库
                    .registerKryoClasses(new Class[]{HashMap.class, BCD.class})
                    ;
            
            //构建MQ配置
            @SuppressWarnings("serial")
            HashMap<String, String> mqConf = new HashMap<String, String>() {{
                put(RocketMQConfig.NAME_SERVER_ADDR, namesrv);
                put(RocketMQConfig.CONSUMER_TOPIC, topic);
                put(RocketMQConfig.CONSUMER_GROUP, group);
            }};
            
            JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(second));
            jssc.checkpoint(checkPointDir);
            jssc.remember(Durations.minutes(1440));
            
            //接収RocketMQ账单
            JavaReceiverInputDStream<BCD> billListRDD = jssc.receiverStream(new RocketMQReceiver(mqConf, StorageLevels.MEMORY_AND_DISK_SER));
            
            billListRDD
                .foreachRDD((rdd, time) -> {
                    boolean isEmpty = rdd.partitions().isEmpty();
                    if (!isEmpty) {
                        printStream(rdd, time);
                        
                        long start = System.currentTimeMillis();
                        String curTime = getCurrentDate(start).intern();
                        String dfsDir = (BASE_PATH+"sdt="+curTime+"/").intern();
                        
                        //账单计数器
                        //LongAccumulator billAccumulator = BillAccumulator.getInstance(JavaSparkContext.fromSparkContext(rdd.context()), appName);
                        //billAccumulator.add(rdd.count());
                        
                        //初始化SparkSession
                        SparkSession session = HiveSession.getInstance(conf);
                        
                        //维表等广播
                        BroadcastDIM dim = BroadcastWrapper.getInstance(JavaSparkContext.fromSparkContext(rdd.context())).getValue();
                        SaleAnalysisService service = dim.getService();
                        Dataset<Row> shop = dim.getShop();
                        Dataset<Row> type = dim.getType();
                        Dataset<Row> ruian = dim.getRuian();
                        String mallIdStr = dim.getMallIdStr();
                        
                        Set<String> areaSet=dim.getAreaSet();
                        Map<String, Set<String>> areaMall=dim.getAreaMall();
                        Set<String> mallSet=dim.getMallSet();
                        Set<String> typeSet=dim.getTypeSet();
                        
                        
                        //如果时间为00:00:00时则认为是新的一天,更新广播数据
                        if ((curTime+"000000").equals(FORMATTER_YMDHMS.get().format(new Date(time.milliseconds())))) {
                            BroadcastWrapper.update(session, JavaSparkContext.fromSparkContext(rdd.context()));//更新dim表数据
                            mkDfsDir(session, dfsDir);//初始化dfsDir
                        }
                        
                        //先持久化本次接收到的账单(写入当日)
                        session.createDataFrame(rdd, BCD.class)
                            .filter("sdt = "+curTime)
                            .write()
                            .mode("append")
                            .parquet(dfsDir);
                        
                        //再读取全量账单(读取当日)
                        Dataset<Row> bills = session.read().parquet(dfsDir)
                            .filter("shopId in ("+mallIdStr+")")
                            .dropDuplicates("billId")
                            //.coalesce(1)
                            .cache();
                        
                        //计算大屏指标
                        System.out.println("==== 计算指标:时间条件:"+curTime+" ====");
                        service.totalSale(bills);
                        service.totalRefund(bills);
                        service.peakTime(bills);
                        service.areaSaleTrendForAll(bills, ruian,areaSet, curTime);
                        service.projectContrast(bills, ruian,areaMall);
                        service.saleForShopTop10(bills, shop, ruian);
                        service.projectTypeSaleContrast(bills, shop, type, ruian,areaSet,mallSet,typeSet);
                        long end = System.currentTimeMillis();
                        System.out.println("==== 计算指标:耗时:"+(end-start)+"/ms ====");
                        bills.unpersist();
                        //每天最多存6个文件
                        if (bills.inputFiles().length > 6) {
                            mergeSmallFiles(bills, session, dfsDir);
                        }
                    } else {//如果SparkStreaming接收的Batch为空,则不做处理
                        System.out.println("==== rdd is null! ");
                    }
                });
            return jssc;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

}

/**
 * 广播DIM相关数据
 * @author mengyao
 *
 */
class BroadcastWrapper {
    private static volatile Broadcast<BroadcastDIM> instance = null;
    
    public static Broadcast<BroadcastDIM> getInstance(JavaSparkContext jsc) {
        if (instance == null) {
            synchronized (BroadcastWrapper.class) {
                if (instance == null) {
                    SparkSession session = HiveSession.getInstance(jsc.getConf());
                    BroadcastDIM dim = new BroadcastDIM(session);
                    dim.assign();
                    instance = jsc.broadcast(dim);
                }
            }
        }
        return instance;
    }
    
    /**
     * 每日更新数据
     * @param batchTime    batch发生时间
     * @param dayES    每日开始时间
     */
    public static void update(SparkSession session, JavaSparkContext jsc) {
        BroadcastDIM dim = instance.getValue();
        if (null!=dim) {
            dim.assign();
            jsc.broadcast(dim);
        }
    }
}

class BroadcastDIM {
    private SparkSession session;
    private SaleAnalysisService service = new SaleAnalysisService();
    private Dataset<Row> shop;
    private Dataset<Row> type;
    private Dataset<Row> ruian;
    private String mallIdStr;
    private List<RuianMall> ruianList ;
    private Set<String>  areaSet;
    private Set<String>  typeSet;
    private Set<String> mallSet;
    private Map<String, Set<String>> areaMall;
    public BroadcastDIM(SparkSession session) {
        this.session = session;
    }
    public void assign() {
        ruian = session.sql("select id,item,name,area_en,area_cn,channel,mid,rmid from tbl_dim_ruian").cache();
        Row[] ruianRows = (Row[])ruian.collect();
        setRuianList(ruianRows);
        setAreaSet();
        setMallIdStr();
        setRuianMallAll();
        setMallSet();
        shop = session.sql("select id,shop_entity_id,shop_entity_name,shop_id,shop_entity_type_root,shop_entity_type_leaf,bill_match_mode,leasing_model,shop_entity_status,open_time,close_time,open_time_list,close_time_list,monite_begin_time_list,monite_end_time_list,marketing_copywriter,marketing_image,font_style,font_size,contract_area,province,city,area,billhint,is_del,brand,brand_code,classify,in_aera,storey,leasing_resource,shop_entity_source,create_time,c_time_stamp,shop_entity_img,business_area_id,source,status,logo,door_head_photo,business_license,certificate,coordinates,consume_per,brand_name,brand_log,alipay,process  "
                + "from tbl_ods_shop where shop_id in ("+mallIdStr+")").cache();
        type = session.sql("select * from tbl_ods_type").cache();
        setRuianType();
    }
    public void  setRuianList(Row[] rows) {
        if(rows.length>0){
            ruianList=new ArrayList<>();
            for(Row row:rows){
                RuianMall rm=new RuianMall();
                if(!row.isNullAt(0)){//id
                    rm.setId(row.getInt(0));
                }
                if(!row.isNullAt(1)){//item
                    rm.setItem(row.getString(1));
                }
                if(!row.isNullAt(2)){//name
                    rm.setName(row.getString(2));
                }
                if(!row.isNullAt(3)){//area_en
                    rm.setAreaEn(row.getString(3));
                }
                if(!row.isNullAt(4)){//area_cn
                    rm.setArenCn(row.getString(4));
                }
                if(!row.isNullAt(5)){//channel
                    rm.setChannel(row.getString(5));
                }
                if(!row.isNullAt(6)){//mid
                    rm.setMid(row.getInt(6));
                }
                if(!row.isNullAt(7)){//rmid
                    rm.setRmid(row.getString(7));
                }
                ruianList.add(rm);
            }
        }
    }
    /**
     * 提取rmid 拼接成字符串
     * @param rows
     */
    public void setMallIdStr() {
        if(ruianList.size()>0){
            StringBuilder sbStr=new StringBuilder();
            for(RuianMall row : ruianList){
                sbStr.append("'").append(row.getRmid()).append("',");
            }
            String tmpValue = sbStr.toString();
            mallIdStr=tmpValue.substring(0, tmpValue.length()-1);
        }
    }
    /**
     * 提取非空唯一中文区域名称 
     * @return
     */
    public void setAreaSet() {
        if(ruianList.size()>0){
            areaSet=new java.util.HashSet<>();
            for(RuianMall row:ruianList){
                areaSet.add(row.getArenCn());
            }
        }
    }
    /**
     * 提取瑞安mall 14个机构中文名
     * 
     * @return
     * Map<String,Set<String>>
     * key:areaCn value :  Set<mallName>
     */
    public void setRuianMallAll( ){
        areaMall=new HashMap<>();
        ruianList.forEach(rm->{
            if(areaMall.containsKey(rm.getArenCn())){//存在,更新mall的列表
                areaMall.get(rm.getArenCn()).add(rm.getName());
            }else{//新的区域,新的mall
                Set<String> mallSet=new HashSet<>();
                mallSet.add(rm.getName());
                areaMall.put(rm.getArenCn(),mallSet);
            }
        });
    }
    /**
     * 提取瑞安mall 14个机构中文名
     * 
     * @return
     */
    public void setMallSet(){
        mallSet=new java.util.HashSet<>();
        ruianList.forEach(rm->{
            if(!mallSet.contains(rm.getName())){//存在,更新mall的列表
                mallSet.add(rm.getName());
            }
        });
    }
    /**
     * 性能、性能、性能 考虑,先写死
     * 如果用账单、店铺、业态关联查询,效率会很慢
     * 
     * @return
     */
    public void setRuianType(){
        //TODO 目前写死,需要改
        typeSet=new HashSet<>();
        typeSet.add("服务");
        typeSet.add("零售");
        typeSet.add("娱乐");
        typeSet.add("主力店");
        typeSet.add("餐饮");
        typeSet.add("其它");
    }
    public SaleAnalysisService getService() {
        return service;
    }
    public void setService(SaleAnalysisService service) {
        this.service = service;
    }
    public Dataset<Row> getShop() {
        return shop;
    }
    public void setShop(Dataset<Row> shop) {
        this.shop = shop;
    }
    public Dataset<Row> getType() {
        return type;
    }
    public void setType(Dataset<Row> type) {
        this.type = type;
    }
    public Dataset<Row> getRuian() {
        return ruian;
    }
    public void setRuian(Dataset<Row> ruian) {
        this.ruian = ruian;
    }
    public String getMallIdStr() {
        return mallIdStr;
    }
    public void setMallIdStr(String mallIdStr) {
        this.mallIdStr = mallIdStr;
    }
    public List<RuianMall> getRuianList() {
        return ruianList;
    }
    public Set<String> getAreaSet() {
        return this.areaSet;
    }
    public Map<String, Set<String>> getAreaMall() {
        return this.areaMall;
    }
    public Set<String> getMallSet() {
        return this.mallSet;
    }
    public Set<String> getTypeSet() {
        return this.typeSet;
    }
}

/**
 * 账单累加器
 * @author mengyao
 *
 */
class BillAccumulator {
    private static volatile LongAccumulator instance = null;

    public static LongAccumulator getInstance(JavaSparkContext jsc, String name) {
        if (instance == null) {
            synchronized (BillAccumulator.class) {
                if (instance == null) {
                    instance = jsc.sc().longAccumulator(name);
                }
            }
        }
        return instance;
    }
}

/**
 * SparkSQL的Hive数据源
 * @author mengyao
 *
 */
class HiveSession {
    private static transient SparkSession instance = null;
    
    public static SparkSession getInstance(SparkConf conf) {
        if (instance == null) {
            instance = SparkSession.builder()
                    .config(conf)
                    .enableHiveSupport()
                    .getOrCreate();
        }
        return instance;
    }
}

 

4.4、账单实体类

package com.mengyao.graph.etl.apps.commons.beans.ods.fmt;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringUtils;

import com.mengyao.graph.etl.apps.commons.beans.ods.BillInfo;
import com.mengyao.graph.etl.apps.commons.beans.ods.DiscountDetailsInfo;
import com.mengyao.graph.etl.apps.commons.beans.ods.GoodsDetailInfo;
import com.mengyao.graph.etl.apps.commons.beans.ods.SettlementWayInfo;
import com.mengyao.utils.DateTimeUtils;

/**
 * 账单信息 gag_bill.bill_info
 * 
 * @author jmb
 * @update 2018-12-13 for mengyao
 */
public class BillInfoFmt implements Serializable {
    private static final long serialVersionUID = 1L;

    /**
     * 预结单
     */
    public static final String BILLTYPE_JUJIEDAN = "2";
    /**
     * 结账单
     */
    public static final String BILLTYPE_JIEZHANGDAN = "1";
    /**
     * 日结账单
     */
    public static final String BILLTYPE_RIJIEDAN = "3";
    /**
     * 点菜单
     */
    public static final String BILLTYPE_DIANCAIDAN = "7";

    /**
     * 账单编号(系统产生),UUID
     */
    private String id;

    /**
     * 账单编号(系统产生),HBase主键
     */
    private String rowKey;

    /**
     * 商家编号(系统产生)
     */
    private String shopId;

    /**
     * 商家名称(系统产生)
     */
    private String shopName;

    /**
     * 实体店编号(系统产生)
     */
    private String shopEntityId;

    /**
     * 实体店名称(系统产生)
     */
    private String shopEntityName;

    /**
     * 创建时间(系统产生)
     */
    private String createTime;

    /**
     * 最后一次修改时间(系统产生)
     */
    private String cTimeStamp;

    /**
     * 信息上传时间(以header中创建时间为准yyyyMMddHHmmss)
     */
    private String hcTime;

    /**
     * 流水号(header中的流水号)
     */
    private String hserial;

    /**
     * 修正账单数据的设备编号(如果没有修正,则与原始上传账单的采集终端编号相同)
     */
    private String fixTerminal;

    /**
     * 采集终端编号(截获)
     */
    private String terminalNumber;

    /**
     * 账单文件名称,唯一(截获),12位MAC地址+17位时间
     */
    private String billfileName;

    /**
     * 反扫支付时终端生成的终端流水号UUID(全球唯一)[先支付后打单必填]
     */
    private String tsuuid;

    /**
     * 历史账单文件名称,记录合并过程中历次账单文件名称,全量,mongodb为Array
     */
    private List<String> billfileNameHis;

    /**
     * 账单序号,不保证唯一(截获)
     */
    private String billNo;

    /**
     * 截获时间(截获)
     */
    private String interceptTime;

    /**
     * 店铺全名(截获)
     */
    private String shopEntityFullName;

    /**
     * 店铺地址(截获)
     */
    private String shopEntityAddress;

    /**
     * 电话(截获)
     */
    private String telephone;

    /**
     * 售货员(截获)
     */
    private String saler;

    /**
     * 收银台(截获)
     */
    private String checkstand;

    /**
     * 收银员(截获)
     */
    private String cashier;

    /**
     * 应收金额(截获)
     */
    private Double receivableAmount;

    /**
     * 原始应收金额(截获)
     */
    private Double defaultReceivableAmount;

    /**
     * 商品数量(截获)
     */
    private Double totalNum;

    /**
     * 原始商品数量(截获)
     */
    private Double defaultTotalNum;

    /**
     * 小票流水号(截获)
     */
    private String billSerialNumber;

    /**
     * 总金额(截获)
     */
    private Double totalFee;

    /**
     * 原始总金额(截获)
     */
    private Double defaultTotalFee;

    /**
     * 实收金额(截获)
     */
    private Double paidAmount;

    /**
     * 原始实收金额(截获)
     */
    private Double defaultPaidAmount;

    /**
     * 折扣金额(截获)
     */
    private Double discountAmount;

    /**
     * 原始折扣金额(截获)
     */
    private Double defaultDiscountAmount;

    /**
     * 优惠金额(截获)
     */
    private Double couponAmount;

    /**
     * 原始优惠金额(截获)
     */
    private Double defaultCouponAmount;

    /**
     * 找零金额(截获)
     */
    private Double changeAmount;

    /**
     * 原始找零金额(截获)
     */
    private Double defaultChangeAmount;

    /**
     * 结算方式(支持多项)(截获)例:[{"a":5.0,"p":"现金"},{"a":10.01,"p":"书券"}],"a":结算金额,小数[截获,必填],
     * "p":"结算方式[截获,必填]"
     */
    private List<String> settlementWay;

    /**
     * 销售时间(截获)
     */
    private String saleTime;

    /**
     * 会员卡号(截获)
     */
    private String memberCardNumber;

    /**
     * 累计消费(截获)
     */
    private Double totalConsumption;

    /**
     * 原始累计消费(截获)
     */
    private Double defaultTotalConsumption;

    /**
     * 网址(截获)
     */
    private String website;

    /**
     * 小票图片(截获),只存url
     */
    private String billImage;

    /**
     * 商品详情(支持多项)(截获)[{"name":"可乐","itemserial":"PUMU00123","price":5.01,"totalnum":5.0,"totalprice":25.05},{"name":"金枪鱼","itemserial":"FOOD02012","price":10.55,"totalnum":1.5,"totalprice":15.83}]
     * ,"name":"商品名称[截获,选填]","itemserial":"条形码[截获,选填]","price":单价,小数[截获,选填],"totalnum":总数,小数[截获,选填],"totalprice":总价,小数[截获,选填]
     */
    private List<String> goodsDetails;

    /**
     * 房间号(截获)(酒店特有)
     */
    private String roomNo;

    /**
     * 入住姓名(截获)(酒店特有)
     */
    private String checkinName;

    /**
     * 桌号(截获)(餐饮特有)
     */
    private String deskNo;

    /**
     * 消费人数(截获)(一般用于餐饮)
     */
    private Double consumeNum;

    /**
     * 原始消费人数(截获)(一般用于餐饮)
     */
    private Double defaultConsumeNum;

    /**
     * 原始账单所有文本信息(截获)
     */
    private String billText;

    /**
     * 入住时间(截获)(酒店特有)
     */
    private String inTime;

    /**
     * 离店时间(截获)(钟点房特有)
     */
    private String outTime;
    /**
     * 默认打印时间
     */
    private String defaultPrintDate;
    /**
     * 默认入住时间
     */
    private String defaultInTime;
    /**
     * 默认离店时间
     */
    private String defaultOutTime;

    /**
     * 上传类型 1:全单上传 2. 筛选账单上传
     */
    private String uploadType;

    /**
     * 除了上面截获外的自定义数据(截获),json串(Map<String, Object>),json串中的key,value由采集终端自定义。
     */
    private Map<String, String> customRecord;

    /**
     * 账单类型1:结账单2:预结单 3:日结单 4:处方 5:预付押金单 6:退货单 7:点菜单 8:发票单 [选填]
     */
    private String billType;

    /**
     * 账单修改方式 0:默认值 1:收银员重打 2:人工修改金额 3:自动重解析
     */
    private String modifyType = "0";

    /**
     * 账单来源 1:设备采集 2:人工补录3、解析服务 4.第三方
     */
    private String billSource = "1";

    /**
     * 服务端解析路径[选填,用于服务端解析分析问题]
     */
    private String analyzPath;

    /**
     * 刷卡,钱包等匹配账单的key,格式BILL_MAC_金额_截获时间
     */
    private String billMatchKey;

    /**
     * 数据匹配支付结果等成功后,此字段保存支付结果等的主键 [选填] 匹配成功后为必填,理论上应该保证此值为全局唯一
     */
    private String matchId;

    /**
     * 默认销售时间;
     */
    private String defaultSaleTime;

    /**
     * 客户名称[截获,选填]
     */
    private String customerName;
    /**
     * 会员编号[截获,选填]
     */
    private String membershipId;
    /**
     * 会员级别[截获,选填]
     */
    private String memberLevels;
    /**
     * 宠物名称[截获,选填]
     */
    private String petName;
    /**
     * 宠物编号[截获,选填]
     */
    private String petNumber;
    /**
     * 负责人(医生)[截获,选填]
     */
    private String principal;
    /**
     * 预交押金[截获,选填]
     */
    private String deposit;
    /**
     * 打印时间yyyyMMddHHmmss[截获,选填,如果截获位数不够,后面补0]
     */
    private String printDate;
    /**
     * 默认拦截时间
     */
    private String defaultInterceptTime;
    /**
     * 匹配模型 sk0001:先刷卡后打单,单次刷卡; dd0001:先打单后刷卡,单次刷卡
     */
    private String printMatchType;

    /**
     * 账单合并时使用,唯一
     */
    private String billMergeKey;
    /**
     * 存重打单的应收金额
     */
    private Double modifyAmount;
    /**
     * 存重打单的应收金额List
     */
    private List<Double> modifyAmountList;
    /**
     * 存重打单的应收金额的销售时间List
     */
    private List<String> modifyAmountSaleTimeList;
    /**
     * 存重打单的应收金额的截获时间List
     */
    private List<String> modifyAmountInterceptTimeList;
    /**
     * 唯一标识[选填](目前可用来做积分标识使用)
     */
    private String uniqueId;
    /**
     * 历史唯一标识,记录合并过程中历次唯一标识,全量,mongodb为Array(目前可用来做积分标识使用)
     */
    private List<String> uniqueIdHis;
    /**
     * 是否追加二维码 1:是 2:否 [必填]
     */
    private String ifqrcode;
    /**
     * 优惠券券码
     */
    private String couponNum;
    /**
     * ERP会员
     */
    private String erpMemberCard;
    /**
     * 凭证类型 11:水单(商户pos、ERP打印)12:收银凭证(大pos打印) 13:支付凭证(签购单)99:类型未知 [选填,默认写99,表示未知]
     */
    private String voucherType;
    /**
     * 小票二维码[选填]
     */
    private String qrCode;
    /**
     * 积分标识 0:本次积分字段没找到或没有配置 1:有本次积分字段[选填]
     */
    private String integralmark;
    /**
     * 本次积分[选填]
     */
    private String thisintegral;
    /**
     * 备注[选填]
     */
    private String remarks;
    /**
     * 账单子类型,庖丁用于配置文件分类
     */
    private String templateName;
    /**
     * 购货方名称[截获,选填]
     */
    private String custName;
    /**
     * 购货方税号[截获,选填]
     */
    private String custTaxNo;
    /**
     * 购货方地址、电话[截获,选填]
     */
    private String custAdress;
    /**
     * 购货方银行及账号[截获,选填]
     */
    private String custBankAccount;

    /**
     * 第三方订单号(第三方系统唯一)
     */
    private String thirdPartyOrderNo;
    /**
     * 充值卡消费金额[截获,选填]
     */
    private Double rechargeableCardConsumeAmount;
    /**
     * 原始充值卡消费金额[截获,选填]
     */
    private Double defaultRechargeableCardConsumeAmount;
    /**
     * 实付金额[截获,选填]
     */
    private Double outOfPocketAmount;
    /**
     * 原始实付金额[截获,选填]
     */
    private Double defaultOutOfPocketAmount;
    /**
     * 充值金额[截获,选填]
     */
    private Double rechargeAmount;
    /**
     * 原始充值金额[截获,选填]
     */
    private Double defaultRechargeAmount;
    /**
     * 会员价[截获,选填]
     */
    private Double memberPrice;
    /**
     * 原始会员价[截获,选填]
     */
    private Double defaultMemberPrice;
    /**
     * 会员折扣率[截获,选填]
     */
    private Double memberDiscountrate;
    /**
     * 原始会员折扣率[截获,选填]
     */
    private Double defaultMemberDiscountrate;
    /**
     * 会员累计消费[截获,选填]
     */
    private Double memberTotalConsumption;
    /**
     * 原始会员累计消费[截获,选填]
     */
    private Double defaultMemberTotalConsumption;
    /**
     * 外卖单 1:外卖单 2:非外卖单[截获,选填]
     */
    private String takeout;

    /**
     * 优惠商品详情(支持多项)(截获),{"name":"可乐","price":5.01}
     */
    private List<String> discountDetails;

    public String getThirdPartyOrderNo() {
        return thirdPartyOrderNo;
    }

    public void setThirdPartyOrderNo(String thirdPartyOrderNo) {
        this.thirdPartyOrderNo = thirdPartyOrderNo;
    }

    public String getRemarks() {
        return remarks;
    }

    public void setRemarks(String remarks) {
        this.remarks = remarks;
    }

    public String getTemplateName() {
        return templateName;
    }

    public void setTemplateName(String templateName) {
        this.templateName = templateName;
    }

    public String getDefaultPrintDate() {
        return this.defaultPrintDate;
    }

    public void setDefaultPrintDate(String defaultPrintDate) {
        this.defaultPrintDate = defaultPrintDate;
    }

    public String getDefaultInTime() {
        return this.defaultInTime;
    }

    public void setDefaultInTime(String defaultInTime) {
        this.defaultInTime = defaultInTime;
    }

    public String getDefaultOutTime() {
        return this.defaultOutTime;
    }

    public void setDefaultOutTime(String defaultOutTime) {
        this.defaultOutTime = defaultOutTime;
    }

    public String getCouponNum() {
        return this.couponNum;
    }

    public void setCouponNum(String couponNum) {
        this.couponNum = couponNum;
    }

    public String getErpMemberCard() {
        return this.erpMemberCard;
    }

    public void setErpMemberCard(String erpMemberCard) {
        this.erpMemberCard = erpMemberCard;
    }

    public String getVoucherType() {
        return this.voucherType;
    }

    public void setVoucherType(String voucherType) {
        this.voucherType = voucherType;
    }

    public String getAnalyzPath() {
        return this.analyzPath;
    }

    public void setAnalyzPath(String analyzPath) {
        this.analyzPath = analyzPath;
    }

    public List<Double> getModifyAmountList() {
        return this.modifyAmountList;
    }

    public String getUniqueId() {
        return this.uniqueId;
    }

    public void setUniqueId(String uniqueId) {
        this.uniqueId = uniqueId;
    }

    public String getIfqrcode() {
        return this.ifqrcode;
    }

    public void setIfqrcode(String ifqrcode) {
        this.ifqrcode = ifqrcode;
    }

    public void setModifyAmountList(List<Double> modifyAmountList) {
        this.modifyAmountList = modifyAmountList;
    }

    public Double getModifyAmount() {
        return this.modifyAmount;
    }

    public void setModifyAmount(Double modifyAmount) {
        this.modifyAmount = modifyAmount;
    }

    public String getDefaultSaleTime() {
        return this.defaultSaleTime;
    }

    public void setDefaultSaleTime(String defaultSaleTime) {
        this.defaultSaleTime = defaultSaleTime;
    }

    public String getBillMergeKey() {
        return this.billMergeKey;
    }

    public void setBillMergeKey(String billMergeKey) {
        this.billMergeKey = billMergeKey;
    }

    public String getCustName() {
        return custName;
    }

    public void setCustName(String custName) {
        this.custName = custName;
    }

    public String getCustTaxNo() {
        return custTaxNo;
    }

    public void setCustTaxNo(String custTaxNo) {
        this.custTaxNo = custTaxNo;
    }

    public String getCustAdress() {
        return custAdress;
    }

    public void setCustAdress(String custAdress) {
        this.custAdress = custAdress;
    }

    public String getCustBankAccount() {
        return custBankAccount;
    }

    public void setCustBankAccount(String custBankAccount) {
        this.custBankAccount = custBankAccount;
    }

    public List<String> getUniqueIdHis() {
        return uniqueIdHis;
    }

    public void setUniqueIdHis(List<String> uniqueIdHis) {
        this.uniqueIdHis = uniqueIdHis;
    }

    public Double getRechargeableCardConsumeAmount() {
        return rechargeableCardConsumeAmount;
    }

    public void setRechargeableCardConsumeAmount(Double rechargeableCardConsumeAmount) {
        this.rechargeableCardConsumeAmount = rechargeableCardConsumeAmount;
    }

    public Double getDefaultRechargeableCardConsumeAmount() {
        return defaultRechargeableCardConsumeAmount;
    }

    public void setDefaultRechargeableCardConsumeAmount(Double defaultRechargeableCardConsumeAmount) {
        this.defaultRechargeableCardConsumeAmount = defaultRechargeableCardConsumeAmount;
    }

    public Double getOutOfPocketAmount() {
        return outOfPocketAmount;
    }

    public void setOutOfPocketAmount(Double outOfPocketAmount) {
        this.outOfPocketAmount = outOfPocketAmount;
    }

    public Double getDefaultOutOfPocketAmount() {
        return defaultOutOfPocketAmount;
    }

    public void setDefaultOutOfPocketAmount(Double defaultOutOfPocketAmount) {
        this.defaultOutOfPocketAmount = defaultOutOfPocketAmount;
    }

    public Double getRechargeAmount() {
        return rechargeAmount;
    }

    public void setRechargeAmount(Double rechargeAmount) {
        this.rechargeAmount = rechargeAmount;
    }

    public Double getDefaultRechargeAmount() {
        return defaultRechargeAmount;
    }

    public void setDefaultRechargeAmount(Double defaultRechargeAmount) {
        this.defaultRechargeAmount = defaultRechargeAmount;
    }

    public Double getMemberPrice() {
        return memberPrice;
    }

    public void setMemberPrice(Double memberPrice) {
        this.memberPrice = memberPrice;
    }

    public Double getDefaultMemberPrice() {
        return defaultMemberPrice;
    }

    public void setDefaultMemberPrice(Double defaultMemberPrice) {
        this.defaultMemberPrice = defaultMemberPrice;
    }

    public Double getMemberDiscountrate() {
        return memberDiscountrate;
    }

    public void setMemberDiscountrate(Double memberDiscountrate) {
        this.memberDiscountrate = memberDiscountrate;
    }

    public Double getDefaultMemberDiscountrate() {
        return defaultMemberDiscountrate;
    }

    public void setDefaultMemberDiscountrate(Double defaultMemberDiscountrate) {
        this.defaultMemberDiscountrate = defaultMemberDiscountrate;
    }

    public Double getMemberTotalConsumption() {
        return memberTotalConsumption;
    }

    public void setMemberTotalConsumption(Double memberTotalConsumption) {
        this.memberTotalConsumption = memberTotalConsumption;
    }

    public Double getDefaultMemberTotalConsumption() {
        return defaultMemberTotalConsumption;
    }

    public void setDefaultMemberTotalConsumption(Double defaultMemberTotalConsumption) {
        this.defaultMemberTotalConsumption = defaultMemberTotalConsumption;
    }

    public String getTakeout() {
        return takeout;
    }

    public void setTakeout(String takeout) {
        this.takeout = takeout;
    }

    public List<String> getDiscountDetails() {
        return discountDetails;
    }

    public void setDiscountDetails(List<String> discountDetails) {
        this.discountDetails = discountDetails;
    }

    public String getId() {
        return this.id;
    }

    public void setId(String id) {
        this.id = id;
    }

    public String getShopId() {
        return this.shopId;
    }

    public void setShopId(String shopId) {
        this.shopId = shopId;
    }

    public String getShopName() {
        return this.shopName;
    }

    public void setShopName(String shopName) {
        this.shopName = shopName;
    }

    public String getShopEntityId() {
        return this.shopEntityId;
    }

    public void setShopEntityId(String shopEntityId) {
        this.shopEntityId = shopEntityId;
    }

    public String getShopEntityName() {
        return this.shopEntityName;
    }

    public void setShopEntityName(String shopEntityName) {
        this.shopEntityName = shopEntityName;
    }

    public String getCreateTime() {
        return this.createTime;
    }

    public void setCreateTime(String createTime) {
        this.createTime = createTime;
    }

    public String getCTimeStamp() {
        return this.cTimeStamp;
    }

    public void setCTimeStamp(String cTimeStamp) {
        this.cTimeStamp = cTimeStamp;
    }

    public String getHcTime() {
        return this.hcTime;
    }

    public void setHcTime(String hcTime) {
        this.hcTime = hcTime;
    }

    public String getHserial() {
        return this.hserial;
    }

    public void setHserial(String hserial) {
        this.hserial = hserial;
    }

    public String getFixTerminal() {
        return this.fixTerminal;
    }

    public void setFixTerminal(String fixTerminal) {
        this.fixTerminal = fixTerminal;
    }

    public String getTerminalNumber() {
        return this.terminalNumber;
    }

    public void setTerminalNumber(String terminalNumber) {
        this.terminalNumber = terminalNumber;
    }

    public String getBillfileName() {
        return this.billfileName;
    }

    public void setBillfileName(String billfileName) {
        this.billfileName = billfileName;
    }

    public List<String> getBillfileNameHis() {
        return this.billfileNameHis;
    }

    public void setBillfileNameHis(List<String> billfileNameHis) {
        this.billfileNameHis = billfileNameHis;
    }

    public String getBillNo() {
        return this.billNo;
    }

    public void setBillNo(String billNo) {
        this.billNo = billNo;
    }

    public String getInterceptTime() {
        return this.interceptTime;
    }

    public void setInterceptTime(String interceptTime) {
        this.interceptTime = interceptTime;
    }

    public String getShopEntityFullName() {
        return this.shopEntityFullName;
    }

    public void setShopEntityFullName(String shopEntityFullName) {
        this.shopEntityFullName = shopEntityFullName;
    }

    public String getShopEntityAddress() {
        return this.shopEntityAddress;
    }

    public void setShopEntityAddress(String shopEntityAddress) {
        this.shopEntityAddress = shopEntityAddress;
    }

    public String getTelephone() {
        return this.telephone;
    }

    public void setTelephone(String telephone) {
        this.telephone = telephone;
    }

    public String getSaler() {
        return this.saler;
    }

    public void setSaler(String saler) {
        this.saler = saler;
    }

    public String getCheckstand() {
        return this.checkstand;
    }

    public void setCheckstand(String checkstand) {
        this.checkstand = checkstand;
    }

    public String getCashier() {
        return this.cashier;
    }

    public void setCashier(String cashier) {
        this.cashier = cashier;
    }

    public Double getReceivableAmount() {
        return this.receivableAmount;
    }

    public void setReceivableAmount(Double receivableAmount) {
        this.receivableAmount = receivableAmount;
    }

    public Double getTotalNum() {
        return this.totalNum;
    }

    public void setTotalNum(Double totalNum) {
        this.totalNum = totalNum;
    }

    public String getBillSerialNumber() {
        return this.billSerialNumber;
    }

    public void setBillSerialNumber(String billSerialNumber) {
        this.billSerialNumber = billSerialNumber;
    }

    public Double getTotalFee() {
        return this.totalFee;
    }

    public void setTotalFee(Double totalFee) {
        this.totalFee = totalFee;
    }

    public Double getPaidAmount() {
        return this.paidAmount;
    }

    public void setPaidAmount(Double paidAmount) {
        this.paidAmount = paidAmount;
    }

    public Double getDiscountAmount() {
        return this.discountAmount;
    }

    public void setDiscountAmount(Double discountAmount) {
        this.discountAmount = discountAmount;
    }

    public Double getCouponAmount() {
        return this.couponAmount;
    }

    public void setCouponAmount(Double couponAmount) {
        this.couponAmount = couponAmount;
    }

    public Double getChangeAmount() {
        return this.changeAmount;
    }

    public void setChangeAmount(Double changeAmount) {
        this.changeAmount = changeAmount;
    }

    public List<String> getSettlementWay() {
        return this.settlementWay;
    }

    public void setSettlementWay(List<String> settlementWay) {
        this.settlementWay = settlementWay;
    }

    public String getSaleTime() {
        return saleTime;
    }

    public void setSaleTime(String saleTime) {
        this.saleTime = saleTime;
    }

    public String getMemberCardNumber() {
        return this.memberCardNumber;
    }

    public void setMemberCardNumber(String memberCardNumber) {
        this.memberCardNumber = memberCardNumber;
    }

    public Double getTotalConsumption() {
        return this.totalConsumption;
    }

    public void setTotalConsumption(Double totalConsumption) {
        this.totalConsumption = totalConsumption;
    }

    public String getWebsite() {
        return this.website;
    }

    public void setWebsite(String website) {
        this.website = website;
    }

    public String getBillImage() {
        return this.billImage;
    }

    public void setBillImage(String billImage) {
        this.billImage = billImage;
    }

    public List<String> getGoodsDetails() {
        return this.goodsDetails;
    }

    public void setGoodsDetails(List<String> goodsDetails) {
        this.goodsDetails = goodsDetails;
    }

    public String getRoomNo() {
        return this.roomNo;
    }

    public void setRoomNo(String roomNo) {
        this.roomNo = roomNo;
    }

    public String getCheckinName() {
        return this.checkinName;
    }

    public void setCheckinName(String checkinName) {
        this.checkinName = checkinName;
    }

    public String getDeskNo() {
        return this.deskNo;
    }

    public void setDeskNo(String deskNo) {
        this.deskNo = deskNo;
    }

    public Double getConsumeNum() {
        return this.consumeNum;
    }

    public void setConsumeNum(Double consumeNum) {
        this.consumeNum = consumeNum;
    }

    public String getBillText() {
        return this.billText;
    }

    public void setBillText(String billText) {
         this.billText = billText;
    }

    public Map<String, String> getCustomRecord() {
        return this.customRecord;
    }

    public void setCustomRecord(Map<String, String> customRecord) {
        this.customRecord = customRecord;
    }

    public String getBillType() {
        return this.billType;
    }

    public void setBillType(String billType) {
        this.billType = billType;
    }

    public String getInTime() {
        return this.inTime;
    }

    public String getOutTime() {
        return this.outTime;
    }

    public void setInTime(String inTime) {
        this.inTime = inTime;
    }

    public void setOutTime(String outTime) {
        this.outTime = outTime;
    }

    public String getBillMatchKey() {
        return this.billMatchKey;
    }

    public void setBillMatchKey(String billMatchKey) {
        this.billMatchKey = billMatchKey;
    }

    public String getMatchId() {
        return this.matchId;
    }

    public void setMatchId(String matchId) {
        this.matchId = matchId;
    }

    public String getPrintMatchType() {
        return this.printMatchType;
    }

    public void setPrintMatchType(String printMatchType) {
        this.printMatchType = printMatchType;
    }

    public String getTsuuid() {
        return this.tsuuid;
    }

    public void setTsuuid(String tsuuid) {
        this.tsuuid = tsuuid;
    }

    public String getUploadType() {
        return this.uploadType;
    }

    public void setUploadType(String uploadType) {
        this.uploadType = uploadType;
    }

    public String getCustomerName() {
        return this.customerName;
    }

    public void setCustomerName(String customerName) {
        this.customerName = customerName;
    }

    public String getMembershipId() {
        return this.membershipId;
    }

    public void setMembershipId(String membershipId) {
        this.membershipId = membershipId;
    }

    public String getMemberLevels() {
        return this.memberLevels;
    }

    public void setMemberLevels(String memberLevels) {
        this.memberLevels = memberLevels;
    }

    public String getPetName() {
        return this.petName;
    }

    public void setPetName(String petName) {
        this.petName = petName;
    }

    public String getPetNumber() {
        return this.petNumber;
    }

    public void setPetNumber(String petNumber) {
        this.petNumber = petNumber;
    }

    public String getPrincipal() {
        return this.principal;
    }

    public void setPrincipal(String principal) {
        this.principal = principal;
    }

    public String getDeposit() {
        return this.deposit;
    }

    public void setDeposit(String deposit) {
        this.deposit = deposit;
    }

    public String getPrintDate() {
        return this.printDate;
    }

    public void setPrintDate(String printDate) {
        this.printDate = printDate;
    }

    public String getDefaultInterceptTime() {
        return this.defaultInterceptTime;
    }

    public void setDefaultInterceptTime(String defaultInterceptTime) {
        this.defaultInterceptTime = defaultInterceptTime;
    }

    public String getModifyType() {
        return this.modifyType;
    }

    public void setModifyType(String modifyType) {
        this.modifyType = modifyType;
    }

    public String getBillSource() {
        return this.billSource;
    }

    public void setBillSource(String billSource) {
        this.billSource = billSource;
    }

    public String getQrCode() {
        return qrCode;
    }

    public void setQrCode(String qrCode) {
        this.qrCode = qrCode;
    }

    public String getIntegralmark() {
        return integralmark;
    }

    public void setIntegralmark(String integralmark) {
        this.integralmark = integralmark;
    }

    public String getThisintegral() {
        return thisintegral;
    }

    public void setThisintegral(String thisintegral) {
        this.thisintegral = thisintegral;
    }

    public List<String> getModifyAmountSaleTimeList() {
        return modifyAmountSaleTimeList;
    }

    public void setModifyAmountSaleTimeList(List<String> modifyAmountSaleTimeList) {
        this.modifyAmountSaleTimeList = modifyAmountSaleTimeList;
    }

    public List<String> getModifyAmountInterceptTimeList() {
        return modifyAmountInterceptTimeList;
    }

    public void setModifyAmountInterceptTimeList(List<String> modifyAmountInterceptTimeList) {
        this.modifyAmountInterceptTimeList = modifyAmountInterceptTimeList;
    }

    public Double getDefaultReceivableAmount() {
        return defaultReceivableAmount;
    }

    public void setDefaultReceivableAmount(Double defaultReceivableAmount) {
        this.defaultReceivableAmount = defaultReceivableAmount;
    }

    public Double getDefaultTotalNum() {
        return defaultTotalNum;
    }

    public void setDefaultTotalNum(Double defaultTotalNum) {
        this.defaultTotalNum = defaultTotalNum;
    }

    public Double getDefaultTotalFee() {
        return defaultTotalFee;
    }

    public void setDefaultTotalFee(Double defaultTotalFee) {
        this.defaultTotalFee = defaultTotalFee;
    }

    public Double getDefaultPaidAmount() {
        return defaultPaidAmount;
    }

    public void setDefaultPaidAmount(Double defaultPaidAmount) {
        this.defaultPaidAmount = defaultPaidAmount;
    }

    public Double getDefaultDiscountAmount() {
        return defaultDiscountAmount;
    }

    public void setDefaultDiscountAmount(Double defaultDiscountAmount) {
        this.defaultDiscountAmount = defaultDiscountAmount;
    }

    public Double getDefaultCouponAmount() {
        return defaultCouponAmount;
    }

    public void setDefaultCouponAmount(Double defaultCouponAmount) {
        this.defaultCouponAmount = defaultCouponAmount;
    }

    public Double getDefaultChangeAmount() {
        return defaultChangeAmount;
    }

    public void setDefaultChangeAmount(Double defaultChangeAmount) {
        this.defaultChangeAmount = defaultChangeAmount;
    }

    public Double getDefaultTotalConsumption() {
        return defaultTotalConsumption;
    }

    public void setDefaultTotalConsumption(Double defaultTotalConsumption) {
        this.defaultTotalConsumption = defaultTotalConsumption;
    }

    public Double getDefaultConsumeNum() {
        return defaultConsumeNum;
    }

    public void setDefaultConsumeNum(Double defaultConsumeNum) {
        this.defaultConsumeNum = defaultConsumeNum;
    }

    public String getRowKey() {
        return rowKey;
    }

    public void setRowKey(String rowKey) {
        this.rowKey = rowKey;
    }

    @Override
    public String toString() {
        return id + "\t" + rowKey + "\t" + shopId + "\t" + shopName + "\t" + shopEntityId + "\t" + shopEntityName + "\t"
                + createTime + "\t" + cTimeStamp + "\t" + hcTime + "\t" + hserial + "\t" + fixTerminal + "\t"
                + terminalNumber + "\t" + billfileName + "\t" + tsuuid + "\t" + billfileNameHis + "\t" + billNo + "\t"
                + interceptTime + "\t" + shopEntityFullName + "\t" + shopEntityAddress + "\t" + telephone + "\t" + saler
                + "\t" + checkstand + "\t" + cashier + "\t" + receivableAmount + "\t" + defaultReceivableAmount + "\t"
                + totalNum + "\t" + defaultTotalNum + "\t" + billSerialNumber + "\t" + totalFee + "\t" + defaultTotalFee
                + "\t" + paidAmount + "\t" + defaultPaidAmount + "\t" + discountAmount + "\t" + defaultDiscountAmount
                + "\t" + couponAmount + "\t" + defaultCouponAmount + "\t" + changeAmount + "\t" + defaultChangeAmount
                + "\t" + settlementWay + "\t" + saleTime + "\t" + memberCardNumber + "\t" + totalConsumption + "\t"
                + defaultTotalConsumption + "\t" + website + "\t" + billImage + "\t" + goodsDetails + "\t" + roomNo
                + "\t" + checkinName + "\t" + deskNo + "\t" + consumeNum + "\t" + defaultConsumeNum + "\t" + billText
                + "\t" + inTime + "\t" + outTime + "\t" + defaultPrintDate + "\t" + defaultInTime + "\t"
                + defaultOutTime + "\t" + uploadType + "\t" + customRecord + "\t" + billType + "\t" + modifyType + "\t"
                + billSource + "\t" + analyzPath + "\t" + billMatchKey + "\t" + matchId + "\t" + defaultSaleTime + "\t"
                + customerName + "\t" + membershipId + "\t" + memberLevels + "\t" + petName + "\t" + petNumber + "\t"
                + principal + "\t" + deposit + "\t" + printDate + "\t" + defaultInterceptTime + "\t" + printMatchType
                + "\t" + billMergeKey + "\t" + modifyAmount + "\t" + modifyAmountList + "\t" + modifyAmountSaleTimeList
                + "\t" + modifyAmountInterceptTimeList + "\t" + uniqueId + "\t" + uniqueIdHis + "\t" + ifqrcode + "\t"
                + couponNum + "\t" + erpMemberCard + "\t" + voucherType + "\t" + qrCode + "\t" + integralmark + "\t"
                + thisintegral + "\t" + remarks + "\t" + templateName + "\t" + custName + "\t" + custTaxNo + "\t"
                + custAdress + "\t" + custBankAccount + "\t" + thirdPartyOrderNo + "\t" + rechargeableCardConsumeAmount
                + "\t" + defaultRechargeableCardConsumeAmount + "\t" + outOfPocketAmount + "\t"
                + defaultOutOfPocketAmount + "\t" + rechargeAmount + "\t" + defaultRechargeAmount + "\t" + memberPrice
                + "\t" + defaultMemberPrice + "\t" + memberDiscountrate + "\t" + defaultMemberDiscountrate + "\t"
                + memberTotalConsumption + "\t" + defaultMemberTotalConsumption + "\t" + takeout + "\t"
                + discountDetails;
    }

    public static BillInfoFmt cloneBill(BillInfo fromBean) {
        if (null == fromBean) {
            return null;
        }
        BillInfoFmt toBean = new BillInfoFmt();
        toBean.setId(stringRemove(fromBean.getId()));
        toBean.setRowKey(stringRemove(fromBean.getRowKey()));
        toBean.setShopId(stringRemove(fromBean.getShopId()));
        toBean.setShopName(stringRemove(fromBean.getShopName()));
        toBean.setShopEntityId(stringRemove(fromBean.getShopEntityId()));
        toBean.setShopEntityName(stringRemove(fromBean.getShopEntityName()));
        toBean.setCreateTime(DateTimeUtils.getYmdhmsForNo(fromBean.getCreateTime()));
        toBean.setCTimeStamp(DateTimeUtils.getYmdhmsForNo(fromBean.getCTimeStamp()));
        toBean.setHcTime(stringRemove(fromBean.getHcTime()));
        toBean.setHserial(stringRemove(fromBean.getHserial()));
        toBean.setFixTerminal(stringRemove(fromBean.getFixTerminal()));
        toBean.setTerminalNumber(stringRemove(fromBean.getTerminalNumber()));
        toBean.setBillfileName(stringRemove(fromBean.getBillfileName()));
        toBean.setTsuuid(stringRemove(fromBean.getTsuuid()));
        toBean.setBillfileNameHis(fromBean.getBillfileNameHis());
        toBean.setBillNo(stringRemove(fromBean.getBillNo()));
        toBean.setInterceptTime(DateTimeUtils.getYmdhmsForNo(fromBean.getInterceptTime()));
        toBean.setShopEntityFullName(stringRemove(fromBean.getShopEntityFullName()));
        toBean.setShopEntityAddress(stringRemove(fromBean.getShopEntityAddress()));
        toBean.setTelephone(stringRemove(fromBean.getTelephone()));
        toBean.setSaler(stringRemove(fromBean.getSaler()));
        toBean.setCheckstand(stringRemove(fromBean.getCheckstand()));
        toBean.setCashier(stringRemove(fromBean.getCashier()));
        toBean.setReceivableAmount(fromBean.getReceivableAmount());
        toBean.setDefaultReceivableAmount(fromBean.getDefaultReceivableAmount());
        toBean.setTotalNum(fromBean.getTotalNum());
        toBean.setDefaultTotalNum(fromBean.getDefaultTotalNum());
        toBean.setBillSerialNumber(stringRemove(fromBean.getBillSerialNumber()));
        toBean.setTotalFee(fromBean.getTotalFee());
        toBean.setDefaultTotalFee(fromBean.getDefaultTotalFee());
        toBean.setPaidAmount(fromBean.getPaidAmount());
        toBean.setDefaultPaidAmount(fromBean.getDefaultPaidAmount());
        toBean.setDiscountAmount(fromBean.getDiscountAmount());
        toBean.setDefaultDiscountAmount(fromBean.getDefaultDiscountAmount());
        toBean.setCouponAmount(fromBean.getCouponAmount());
        toBean.setDefaultCouponAmount(fromBean.getDefaultCouponAmount());
        toBean.setChangeAmount(fromBean.getChangeAmount());
        toBean.setDefaultChangeAmount(fromBean.getDefaultChangeAmount());
        if (null != fromBean.getSettlementWay()) {
            List<SettlementWayInfo> rawList = fromBean.getSettlementWay();
            List<String> list = new ArrayList<>();
            for (SettlementWayInfo raw : rawList) {
                list.add(raw.toString());
            }
            toBean.setSettlementWay(list);
        }
        toBean.setSaleTime(fromBean.getSaleTime());
        toBean.setMemberCardNumber(stringRemove(fromBean.getMemberCardNumber()));
        toBean.setTotalConsumption(fromBean.getTotalConsumption());
        toBean.setDefaultTotalConsumption(fromBean.getDefaultTotalConsumption());
        toBean.setWebsite(stringRemove(fromBean.getWebsite()));
        toBean.setBillImage(stringRemove(fromBean.getBillImage()));
        if (null != fromBean.getGoodsDetails()) {
            List<GoodsDetailInfo> rawList = fromBean.getGoodsDetails();
            List<String> list = new ArrayList<>();
            for (GoodsDetailInfo raw : rawList) {
                list.add(raw.toString());
            }
            toBean.setGoodsDetails(list);
        }
        toBean.setRoomNo(stringRemove(fromBean.getRoomNo()));
        toBean.setCheckinName(stringRemove(fromBean.getCheckinName()));
        toBean.setDeskNo(stringRemove(fromBean.getDeskNo()));
        toBean.setConsumeNum(fromBean.getConsumeNum());
        toBean.setDefaultConsumeNum(fromBean.getDefaultConsumeNum());
        toBean.setBillText(stringRemove(fromBean.getBillText()));
        toBean.setInTime(DateTimeUtils.getYmdhmsForNo(fromBean.getInTime()));
        toBean.setOutTime(DateTimeUtils.getYmdhmsForNo(fromBean.getOutTime()));
        toBean.setDefaultPrintDate(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultPrintDate()));
        toBean.setDefaultInTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultInTime()));
        toBean.setDefaultOutTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultOutTime()));
        toBean.setUploadType(stringRemove(fromBean.getUploadType()));
        toBean.setCustomRecord(fromBean.getCustomRecord());
        toBean.setBillType(stringRemove(fromBean.getBillType()));
        toBean.setModifyType(stringRemove(fromBean.getModifyType()));
        toBean.setBillSource(stringRemove(fromBean.getBillSource()));
        toBean.setAnalyzPath(stringRemove(fromBean.getAnalyzPath()));
        toBean.setBillMatchKey(stringRemove(fromBean.getBillMatchKey()));
        toBean.setMatchId(stringRemove(fromBean.getMatchId()));
        toBean.setDefaultSaleTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultSaleTime()));
        toBean.setCustomerName(stringRemove(fromBean.getCustomerName()));
        toBean.setMembershipId(stringRemove(fromBean.getMembershipId()));
        toBean.setMemberLevels(stringRemove(fromBean.getMemberLevels()));
        toBean.setPetName(stringRemove(fromBean.getPetName()));
        toBean.setPetNumber(stringRemove(fromBean.getPetNumber()));
        toBean.setPrincipal(stringRemove(fromBean.getPrincipal()));
        toBean.setDeposit(stringRemove(fromBean.getDeposit()));
        toBean.setPrintDate(DateTimeUtils.getYmdhmsForNo(fromBean.getPrintDate()));
        toBean.setDefaultInterceptTime(DateTimeUtils.getYmdhmsForNo(fromBean.getDefaultInterceptTime()));
        toBean.setPrintMatchType(stringRemove(fromBean.getPrintMatchType()));
        toBean.setBillMergeKey(stringRemove(fromBean.getBillMergeKey()));
        toBean.setModifyAmount(fromBean.getModifyAmount());
        toBean.setModifyAmountList(fromBean.getModifyAmountList());
        toBean.setModifyAmountSaleTimeList(DateTimeUtils.getYmdhmsForNo(fromBean.getModifyAmountSaleTimeList()));
        toBean.setModifyAmountInterceptTimeList(
                DateTimeUtils.getYmdhmsForNo(fromBean.getModifyAmountInterceptTimeList()));
        toBean.setUniqueId(stringRemove(fromBean.getUniqueId()));
        toBean.setUniqueIdHis(fromBean.getUniqueIdHis());
        toBean.setIfqrcode(stringRemove(fromBean.getIfqrcode()));
        toBean.setCouponNum(stringRemove(fromBean.getCouponNum()));
        toBean.setErpMemberCard(stringRemove(fromBean.getErpMemberCard()));
        toBean.setVoucherType(stringRemove(fromBean.getVoucherType()));
        toBean.setQrCode(stringRemove(fromBean.getQrCode()));
        toBean.setIntegralmark(stringRemove(fromBean.getIntegralmark()));
        toBean.setThisintegral(stringRemove(fromBean.getThisintegral()));
        toBean.setRemarks(stringRemove(fromBean.getRemarks()));
        toBean.setTemplateName(stringRemove(fromBean.getTemplateName()));
        toBean.setCustName(stringRemove(fromBean.getCustName()));
        toBean.setCustName(stringRemove(fromBean.getCustTaxNo()));
        toBean.setCustAdress(stringRemove(fromBean.getCustAdress()));
        toBean.setCustBankAccount(stringRemove(fromBean.getCustBankAccount()));
        toBean.setThirdPartyOrderNo(stringRemove(fromBean.getThirdPartyOrderNo()));
        toBean.setRechargeableCardConsumeAmount(fromBean.getRechargeableCardConsumeAmount());
        toBean.setDefaultRechargeableCardConsumeAmount(fromBean.getDefaultRechargeableCardConsumeAmount());
        toBean.setOutOfPocketAmount(fromBean.getOutOfPocketAmount());
        toBean.setDefaultOutOfPocketAmount(fromBean.getDefaultOutOfPocketAmount());
        toBean.setRechargeAmount(fromBean.getRechargeAmount());
        toBean.setDefaultRechargeAmount(fromBean.getDefaultRechargeAmount());
        toBean.setMemberPrice(fromBean.getMemberPrice());
        toBean.setDefaultMemberPrice(fromBean.getDefaultMemberPrice());
        toBean.setMemberDiscountrate(fromBean.getMemberDiscountrate());
        toBean.setDefaultMemberDiscountrate(fromBean.getDefaultMemberDiscountrate());
        toBean.setMemberTotalConsumption(fromBean.getMemberTotalConsumption());
        toBean.setDefaultMemberTotalConsumption(fromBean.getDefaultMemberTotalConsumption());
        toBean.setTakeout(stringRemove(fromBean.getTakeout()));
        if (null != fromBean.getDiscountDetails()) {
            List<DiscountDetailsInfo> rawList = fromBean.getDiscountDetails();
            List<String> list = new ArrayList<>();
            for (DiscountDetailsInfo raw : rawList) {
                list.add(raw.toString());
            }
            toBean.setDiscountDetails(list);
        }
        return toBean;
    }

    /**
     * 原始数据格式化处理,待完善
     * 
     * @param jsonStr
     * @return
     */
    private static String stringRemove(String strVal) {
        if (StringUtils.isEmpty(strVal)) {
            return "";
        }
        strVal = strVal.replace("\t", "");
        strVal = strVal.replace("\n", "");
        strVal = strVal.replace("\r", "");
        return strVal;
    }
}

 

4.5、BCD裁剪实体类

package com.mengyao.graph.etl.apps.dashboard.beans;

import java.io.Serializable;

/**
 * Bill Consumer Dashboard Bean
 * @author mengyao
 *
 */
public class BCD implements Serializable {
    private static final long serialVersionUID = 1749406742944513387L;
    private String billId;
    private double receivableAmount;
    private String saleTime;
    private String sdt;//yyyyMMdd
    private int hour;
    private String billType;
    private String shopId;
    private String shopEntityId;
    public BCD() {
        super();
    }
    public BCD(String billId, double receivableAmount, String saleTime, String billType, String shopId, String shopEntityId) {
        super();
        this.billId = billId;
        this.receivableAmount = receivableAmount;
        this.saleTime = saleTime;
        setDateTime(saleTime);
        this.billType = billType;
        this.shopId = shopId;
        this.shopEntityId = shopEntityId;
    }
    public String getBillId() {
        return billId;
    }
    public void setBillId(String billId) {
        this.billId = billId;
    }
    public double getReceivableAmount() {
        return receivableAmount;
    }
    public void setReceivableAmount(double receivableAmount) {
        this.receivableAmount = receivableAmount;
    }
    public String getSaleTime() {
        return saleTime;
    }
    public void setSaleTime(String saleTime) {
        this.saleTime = saleTime;
        setDateTime(saleTime);
    }
    public String getSdt() {
        return sdt;
    }
    public void setSdt(String sdt) {
        this.sdt = sdt;
    }
    public int getHour() {
        return hour;
    }
    public void setHour(int hour) {
        this.hour = hour;
    }
    public String getBillType() {
        return billType;
    }
    public void setBillType(String billType) {
        this.billType = billType;
    }
    public String getShopId() {
        return shopId;
    }
    public void setShopId(String shopId) {
        this.shopId = shopId;
    }
    public String getShopEntityId() {
        return shopEntityId;
    }
    public void setShopEntityId(String shopEntityId) {
        this.shopEntityId = shopEntityId;
    }
    private void setDateTime(String saleTime) {
        if (null!=saleTime&&saleTime.length()==17) {
            this.sdt = saleTime.substring(0, 8);
            this.hour = Integer.parseInt(saleTime.substring(8, 10));
        }
    }
    @Override
    public String toString() {
        return  billId + "\t" + receivableAmount + "\t" + saleTime + "\t" + sdt + "\t" + billType + "\t" + shopId + "\t" + shopEntityId;
    }
    
}

 

4.6、销售分析业务实现类

package com.mengyao.graph.etl.apps.dashboard.service;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.spark.sql.Column;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.functions;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.mengyao.graph.etl.apps.dashboard.beans.AreaProjSale;
import com.mengyao.graph.etl.apps.dashboard.beans.AreaSaleTrendAll;
import com.mengyao.graph.etl.apps.dashboard.beans.PeakTime;
import com.mengyao.graph.etl.apps.dashboard.beans.ProjectTypeShopNumber;
import com.mengyao.graph.etl.apps.dashboard.beans.ShopSaleRank;
import com.mengyao.graph.etl.apps.dashboard.beans.TotalRefund;
import com.mengyao.graph.etl.apps.dashboard.beans.TotalSale;
import com.mengyao.graph.etl.apps.dashboard.beans.TotalSettlementBillNumber;
import com.mengyao.utils.RedisUtil;


/**
 * 大屏指标分析
 * @author mengyao
 *
 */
public class SaleAnalysisService implements Serializable {

    private static final long serialVersionUID = 8289368096001689148L;
    //解决区域名称hash重复问题
    private static final String SALT="aAb12";

    /**
     * 每日重置大屏指标值
     */
    @Deprecated
    public void reset() {
        RedisUtil.setObject("dtsbn_6", "{\"结账单数\":{\"val\":7270}}\"");
        RedisUtil.setObject("dpt_7", "{\"高峰时段\":{\"val\":13}}\"");
        RedisUtil.setObject("dtr_5", "{\"退款金额\":{\"val\":7301.8}}\"");
        RedisUtil.setObject("dts_4", "{\"总销售额\":{\"val\":1300523.8000000005}}\"");
        RedisUtil.setObject("curDay", "20190227\"");
        RedisUtil.setObject("dastfa_2_20190227", "{\"各区域销售额发展趋势\":{\"重庆天地\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,9.5,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],\"创智天地\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,43.3,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],\"上海瑞虹\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,16.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0],\"上海新天地\":[0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,553.75,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0]}}\"");
        RedisUtil.setObject("dasp_1", "{\"各区域销售额占比\":{\"重庆天地\":[{\"bn\":1,\"n\":\"重庆天地\",\"sa\":9.5}],\"创智天地\":[{\"bn\":3,\"n\":\"壹方\",\"sa\":43.3}],\"上海新天地\":[{\"bn\":4,\"n\":\"新天地时尚购物中心\",\"sa\":152.0},{\"bn\":3,\"n\":\"上海新天地南里北里\",\"sa\":74.0},{\"bn\":2,\"n\":\"湖滨道购物中心\",\"sa\":100.0},{\"bn\":10,\"n\":\"新天地广场\",\"sa\":227.75}],\"上海瑞虹\":[{\"bn\":1,\"n\":\"瑞虹天地星星堂\",\"sa\":16.0}]}}\"");
        RedisUtil.setObject("dpc_8", "{\"各项目销售额对比\":{\"重庆天地\":[{\"bn\":1,\"n\":\"重庆天地\",\"sa\":9.5}],\"创智天地\":[{\"bn\":3,\"n\":\"壹方\",\"sa\":43.3}],\"上海新天地\":[{\"bn\":4,\"n\":\"新天地时尚购物中心\",\"sa\":152.0},{\"bn\":3,\"n\":\"上海新天地南里北里\",\"sa\":74.0},{\"bn\":2,\"n\":\"湖滨道购物中心\",\"sa\":100.0},{\"bn\":10,\"n\":\"新天地广场\",\"sa\":227.75}],\"上海瑞虹\":[{\"bn\":1,\"n\":\"瑞虹天地星星堂\",\"sa\":16.0}]}}\"");
        RedisUtil.setObject("dpac_10", "{\"各项目单均消费\":{\"重庆天地\":[{\"bn\":1,\"n\":\"重庆天地\",\"sa\":9.5}],\"创智天地\":[{\"bn\":3,\"n\":\"壹方\",\"sa\":43.3}],\"上海新天地\":[{\"bn\":4,\"n\":\"新天地时尚购物中心\",\"sa\":152.0},{\"bn\":3,\"n\":\"上海新天地南里北里\",\"sa\":74.0},{\"bn\":2,\"n\":\"湖滨道购物中心\",\"sa\":100.0},{\"bn\":10,\"n\":\"新天地广场\",\"sa\":227.75}],\"上海瑞虹\":[{\"bn\":1,\"n\":\"瑞虹天地星星堂\",\"sa\":16.0}]}}\"");
        RedisUtil.setObject("dsfst_9", "{\"店铺销售排行\":[{\"pn\":\"新天地时尚购物中心\",\"sa\":152.0,\"sn\":\"GREYBOX COFFEE\"},{\"pn\":\"新天地广场\",\"sa\":114.75,\"sn\":\"Arabica\"},{\"pn\":\"湖滨道购物中心\",\"sa\":100.0,\"sn\":\"LOKAL\"},{\"pn\":\"上海新天地南里北里\",\"sa\":74.0,\"sn\":\"哈肯铺_手感烘焙\"},{\"pn\":\"壹方\",\"sa\":60.3,\"sn\":\"新一天便利\"},{\"pn\":\"新天地广场\",\"sa\":41.0,\"sn\":\"奈雪的茶\"},{\"pn\":\"新天地广场\",\"sa\":39.0,\"sn\":\"蒲石小点\"},{\"pn\":\"新天地广场\",\"sa\":18.0,\"sn\":\"Fresh_Every_Day\"},{\"pn\":\"瑞虹天地星星堂\",\"sa\":16.0,\"sn\":\"老盛昌\"},{\"pn\":\"新天地广场\",\"sa\":15.0,\"sn\":\"多几谷\"}]}\"");
        RedisUtil.setObject("dptsc_11", "{\"各项目业态销售额对比\":{\"上海新天地南里北里\":[{\"n\":\"餐饮\",\"sa\":74.0,\"sn\":3}],\"新天地广场\":[{\"n\":\"餐饮\",\"sa\":227.75,\"sn\":10}],\"重庆天地\":[{\"n\":\"其它\",\"sa\":9.5,\"sn\":0}],\"湖滨道购物中心\":[{\"n\":\"餐饮\",\"sa\":100.0,\"sn\":2}],\"壹方\":[{\"n\":\"餐饮\",\"sa\":43.3,\"sn\":3}],\"瑞虹天地星星堂\":[{\"n\":\"餐饮\",\"sa\":16.0,\"sn\":1}],\"新天地时尚购物中心\":[{\"n\":\"餐饮\",\"sa\":152.0,\"sn\":4}]}}\"");
        RedisUtil.setObject("dpsn_12", "{\"各项目店铺数量\":{\"上海新天地南里北里\":[{\"n\":\"餐饮\",\"sa\":74.0,\"sn\":3}],\"新天地广场\":[{\"n\":\"餐饮\",\"sa\":227.75,\"sn\":10}],\"重庆天地\":[{\"n\":\"其它\",\"sa\":9.5,\"sn\":0}],\"湖滨道购物中心\":[{\"n\":\"餐饮\",\"sa\":100.0,\"sn\":2}],\"壹方\":[{\"n\":\"餐饮\",\"sa\":43.3,\"sn\":3}],\"瑞虹天地星星堂\":[{\"n\":\"餐饮\",\"sa\":16.0,\"sn\":1}],\"新天地时尚购物中心\":[{\"n\":\"餐饮\",\"sa\":152.0,\"sn\":4}]}}\"");
    }
    
    /**
     * 总销售额
     * @param ds
     */
    public void totalSale(Dataset<Row> bill) {
        Row[] rows = (Row[])bill
                .filter("billType=1 and receivableAmount>=0")
                .agg(functions.sum(new Column("receivableAmount")).alias("totalSale"), functions.count("receivableAmount"))
                .head(1);
        Map<String, TotalSale> map = new HashMap<String, TotalSale>();
        double totalSale=0D;
        if(rows.length>0){
            Row row = rows[0];
            if (!row.isNullAt(0)) {
                //销售额
                totalSale=row.getDouble(0);
            }
            if (!row.isNullAt(1)) {
                //结账单数
                totalSettlementBillNumber(new TotalSettlementBillNumber(row.getLong(1)));
            }
        }
        map.put("总销售额", new TotalSale(totalSale));
        String str = JSONObject.toJSONString(map);
        System.out.println("====####--totalSale##" + str);
        // 将总销售额存入redis
        RedisUtil.setObject("dts_4", str);
    }
    
    /**
     * 退款金额
     * @param ds
     */
    public void totalRefund(Dataset<Row> bill) {
        Row[] rows = (Row[]) bill
                .filter("((billType=6) or (billType=1 and receivableAmount<0))")
                .agg(functions.sum(functions.abs(new Column("receivableAmount"))).alias("totalSale"))
                .head(1);
        Map<String, TotalRefund> map=new HashMap<>();
        map.put("退款金额", new TotalRefund(0));
        if(rows.length>0){
            Row row = rows[0];
            if (!row.isNullAt(0)) {
                map.put("退款金额", new TotalRefund(row.getDouble(0)));
            }
        }
        String str=JSONObject.toJSONString(map);
        System.out.println("====####--totalRefund##"+str);
        //将退款金额存入redis
        RedisUtil.setObject("dtr_5",str);
    }
    
    /**
     * 高峰时段
     *  1、时段:小时;
     *    2、高峰:当日每小时结账单数累计最大;
     *    3、高峰时段:所有mall累计每小时结账单数;
     * @param ds
     * @return {"高峰时段":{"val":12}}
     */
    public void peakTime(Dataset<Row> bill) {
        //账单表本身有mallid(shopId) 因此无需关联店铺表
        Map<String, PeakTime> map=new HashMap<>();
        map.put("高峰时段", new PeakTime(8));
        Row[] rows = (Row[])bill
                .filter("billType=1")
                .groupBy("hour")
                .agg(functions.sum("receivableAmount").alias("totalSale"))
                .orderBy(new Column("totalSale").desc())
                .limit(1)
                .head(1);
        if(rows.length>0){
            Row row = rows[0];
            if (!row.isNullAt(0)) {
                map.put("高峰时段", new PeakTime(row.getAs(0)));
            }
        }
        String str=JSONObject.toJSONString(map);
        System.out.println("====####--peakTime##"+str);
        //将高峰时段放入redis
        RedisUtil.setObject("dpt_7", str);
    }
    
    /**
     * 各区域销售发展趋势-多个区域(每个区域下有多个mall)当日0点~24点的累计销售额
     * @param bill
     * @param ruian
     * @param curDay
     */
    public void areaSaleTrendForAll(Dataset<Row> bill, Dataset<Row> ruian,Set<String> areaSet, String curDay) {
        Row[] rows = (Row[])bill
                .filter("billType=1 and receivableAmount>0")
                .join(ruian, bill.col("shopId").equalTo(ruian.col("rmid")), "leftouter")
                .groupBy("area_cn", "hour")
                .agg(functions.sum("receivableAmount").alias("areaDayHourSale"))
                .orderBy("areaDayHourSale")
                .select("area_cn","areaDayHourSale","hour")
                .collect();
//        Map<String, Map<String,Map<Integer, Double>>> map = new HashMap<>();
        Map<String, Map<String,Collection<Double>>> map = new HashMap<>();
        Map<String,AreaSaleTrendAll> maps=new HashMap<>();
        if(rows.length>0){
            for (Row row : rows) {
                String areaCn=null;
                double areaDayHourSale=0D;
                int saleHour=0;
                if(!row.isNullAt(0)){
                    areaCn=row.getString(0);
                }
                if(!row.isNullAt(1)){
                    areaDayHourSale=row.getDouble(1);
                }
                if(!row.isNullAt(2)){
                    saleHour=row.getInt(2);
                }
                if(maps.containsKey(areaCn+SALT)){
                    AreaSaleTrendAll ast=maps.get(areaCn+SALT);
                    ast.getVals().put(saleHour, areaDayHourSale);
                }else{
                    HashMap<Integer,Double> vals=new HashMap<Integer,Double>();
                    vals.put(saleHour, areaDayHourSale);
                    maps.put(areaCn+SALT, new AreaSaleTrendAll(areaCn, saleHour, areaDayHourSale));
                }
            }
        }
        //填充没有销售额的区域记录,使数据更加完整。
        areaSet.forEach(areaCn->{
            if(!maps.containsKey(areaCn+SALT)){
                maps.put(areaCn+SALT, new AreaSaleTrendAll(areaCn));
            }
        });

        System.out.println("==####========填充hou====begin=====================");
        for (String key:maps.keySet()) {
            System.out.println("Key:"+key);
        }
        System.out.println("==####========填充hou=====end====================");
        
//        Map<String,Map<Integer, Double>> rs=new HashMap<>();
//        for(AreaSaleTrendAll asta:maps.values()){
//            rs.put(asta.getAreaCn(), asta.getVals());
//        }
        Map<String,Collection<Double>> rs=new HashMap<>();
        for(AreaSaleTrendAll asta:maps.values()){
            rs.put(asta.getAreaCn(), asta.getVals().values());
        }
        map.put("各区域销售额发展趋势", rs);
        //转成json 字符串
        String str=JSONObject.toJSONString(map);
        System.out.println("====####--areaSaleTrendForAll##"+str);
        //放入redis
        RedisUtil.setObject("dastfa_2_"+curDay, str);
        //维护redis中最新日期
        if(!RedisUtil.existsObject("curDay")){//如果为空,说明是第一次运行,直接将当前日期设置到redis中
            RedisUtil.setObject("curDay", curDay);
        }else{
            String curDayRedis=(String) RedisUtil.getObject("curDay");
            if((curDayRedis.compareTo(curDay))<0){//说明当前日期大于redis中的日期,更新
                RedisUtil.setObject("curDay", curDay);
            }
        }
    }

    /**
     * 各项目销售额对比
     *     1、各项目:各个mall;
     *    2、单一项目销售额:mall的当日开始营业时间到当前时间累计销售额;
     */
    public void projectContrast(Dataset<Row> bill, Dataset<Row> ruian,Map<String, Set<String>> ruianMallAll) {
        Row[] rows = (Row[])bill
                .filter("billType=1 and receivableAmount>0")
                .join(ruian, bill.col("shopId").equalTo(ruian.col("rmid")), "leftouter")
                .groupBy("name", "area_cn")
                .agg(functions.sum("receivableAmount").alias("mallDaySale"), functions.count("receivableAmount").alias("mallDayBillNum"))
//                .orderBy("area_cn")
                .select("name","area_cn","mallDaySale","mallDayBillNum")
                .collect();
        Map<String,List<AreaProjSale>> rs=new HashMap<>();
        if(rows.length>0) {
            for (Row row : rows) {
                String mallName=null;
                String areaCn=null;
                double mallDaySale=0D;
                long mallDayBillNum=0L;
                AreaProjSale obj=new AreaProjSale();
                if(!row.isNullAt(0)){//项目、mall的名称
                    mallName=row.getString(0);
                    obj.setN(mallName);
                }
                if(!row.isNullAt(1)){//区域名称
                    areaCn=row.getString(1);
                }
                if(!row.isNullAt(2)){//mall的日销售额
                    mallDaySale=row.getDouble(2);
                    obj.setSa(mallDaySale);
                }
                if(!row.isNullAt(3)){//mall的日结账单数
                    mallDayBillNum=row.getLong(3);
                    obj.setBn(mallDayBillNum);
                }
                //判断是否有该区域
                if(rs.containsKey(areaCn)){
                    rs.get(areaCn).add(obj);
                }else{//不存在该区域
                    List<AreaProjSale> list=new ArrayList<>();
                    list.add(obj);
                    rs.put(areaCn, list);
                }
            }
        }
        //填充未产生账单的数据,默认0
        //获取全部瑞安的区域和mallName的集合
        Set<Entry<String,Set<String>>> entrySet = ruianMallAll.entrySet();
        for(Map.Entry<String, Set<String>> entry:entrySet){
            String areaCn=entry.getKey();//获取区域名称
            //获取每个区域的标准mall的集合
            Set<String> mallSet=entry.getValue();
            if(rs.containsKey(areaCn)){//实际数据中已经存在该区域相关数据
                //判断实际数据mall是否完整
                //用来存放实际数据中mallname的集合
                Set<String> mallSetCur=new HashSet<>();
                //用来存放没有账单的mall的集合
                Set<String> mallSetNew=new HashSet<>();
                //遍历该区域下实际数据集合 并填充set
                for(AreaProjSale obj:rs.get(areaCn)){
                    mallSetCur.add(obj.getN());
                }
                //求两个set的差集合  将标准数据放入
                mallSetNew.addAll(mallSet);
                //求差集合  标准数据-实际数据 得到差集
                mallSetNew.removeAll(mallSetCur);
                //遍历差集合 ,填充默认值
                mallSetNew.forEach(mn->{
                    rs.get(areaCn).add(new AreaProjSale(mn));
                });
            }else{//该区域不存在
                //便利该区域下标准mall集合,逐个放入
                List<AreaProjSale> list=new ArrayList<>();
                mallSet.forEach(mn->{
                    list.add(new AreaProjSale(mn));
                });
                rs.put(areaCn, list);
            }
        }
        //各区域销售额占比
        areaSaleProportion(rs);
        //各区域单均消费
        projectAvgConsumer(rs);
        //转成json
        Map<String, Map<String,List<AreaProjSale>>> pmap=new HashMap<>();
        pmap.put("各项目销售额对比", rs);
        //转成json
        String str=JSONObject.toJSONString(pmap);
        System.out.println("====####--projectContrast##"+str);
        //1 8各项目销售额对比dpc_8 
        RedisUtil.setObject("dpc_8", str);
    }
    
    /**
     * 所有mall中销售额最高的top10店铺
     * @param bill
     * @param shop
     * @param ruian
     */
    public void saleForShopTop10(Dataset<Row> bill, Dataset<Row> shop, Dataset<Row> ruian) {
        Row[] rows = (Row[])bill
                .filter("billType=1 and receivableAmount>0")
                .join(shop, bill.col("shopEntityId").equalTo(shop.col("shop_entity_id")), "leftouter")
                .join(ruian, bill.col("shopId").equalTo(ruian.col("rmid")), "leftouter")
                .groupBy("shop_entity_name", "name")
                .agg(functions.sum("receivableAmount").alias("shopDaySale"))
                .orderBy(new Column("shopDaySale").desc())
                .select("shop_entity_name","name","shopDaySale")
                .limit(10)
                .head(10);
        List<ShopSaleRank> ssrList=new LinkedList<>();
        if(rows.length>0) {
            for (Row row : rows) {
                ShopSaleRank ssr=new ShopSaleRank();
                if(!row.isNullAt(0)){//店铺名称
                    ssr.setSn(row.getString(0));
                }
                if(!row.isNullAt(1)){//项目/mall名称
                    ssr.setPn(row.getString(1));
                }
                if(!row.isNullAt(2)){//店铺日销售额
                    ssr.setSa(row.getDouble(2));
                }
                ssrList.add(ssr);
            }
        }
        //转成json
        Map<String, List<ShopSaleRank>> pmap=new HashMap<>();
        pmap.put("店铺销售排行", ssrList);
        String str=JSON.toJSONString(pmap);
        System.out.println("====####--saleForShopTop10##"+str);
        //将10个店铺日销售额放入redis
        RedisUtil.setObject("dsfst_9",str);
    }
    
    /**
     * 各项目业态销售额对比
     * 
     * @param session
     * @param beginYMDH
     * @param endYMDH
     */
    public void projectTypeSaleContrast(Dataset<Row> bill, Dataset<Row> shop, Dataset<Row> type, Dataset<Row> ruian,
            Set<String> areaSet,Set<String> mallSet,Set<String> ruianTypeAll) {
        Row[] rows = (Row[])bill
                .filter("billType=1 and receivableAmount>0")
                .join(shop, shop.col("shop_entity_id").equalTo(bill.col("shopEntityId")), "leftouter")
                .join(type, type.col("id").equalTo(shop.col("shop_entity_type_root")), "leftouter")
                .join(ruian, ruian.col("rmid").equalTo(bill.col("shopId")))
                .groupBy("name", "shop_type_name")
                .agg(functions.sum("receivableAmount").alias("shopTypeDaySale"), functions.countDistinct("shop_entity_id").alias("shopNum"))
                .select("name","shop_type_name","shopTypeDaySale","shopNum")
                .collect();
        Map<String,List<ProjectTypeShopNumber>> map=new HashMap<>();
        if(rows.length>0) {
            for (Row row : rows) {
                ProjectTypeShopNumber pac=new ProjectTypeShopNumber();
                String mallName=null;
                if(!row.isNullAt(0)){//项目、mall名称
                    mallName=row.getString(0);
                }
                if(!row.isNullAt(1)){//业态名称
                    pac.setN(row.getString(1));
                }else{
                    pac.setN("其它");
                }
                if(!row.isNullAt(2)){//mall的业态日销售额
                    pac.setSa(row.getDouble(2));
                }
                if(!row.isNullAt(3)){//mall的业态店铺数量
                    pac.setSn(row.getLong(3));
                }
                if(map.containsKey(mallName)){//更新map中的list列表
                    List<ProjectTypeShopNumber> pacList=map.get(mallName);
                    pacList.add(pac);
                }else{//新的mall 新建列表放入map
                    List<ProjectTypeShopNumber> pacList=new LinkedList<>();
                    pacList.add(pac);
                    map.put(mallName, pacList);
                }
            }
        }
        //为没有产生账单的业态或mall填充默认数据,是数据看起来完整
        mallSet.forEach(mallName->{
            if(!map.containsKey(mallName)) {//没有账单的mall
                List<ProjectTypeShopNumber> list=new ArrayList<>();
                //遍历全量业态,进行填充
                ruianTypeAll.forEach(type_->{
                    list.add(new ProjectTypeShopNumber(type_,0.0,0));
                });
                map.put(mallName,list);
            }else{//有账单的mall
                //用来获取实际账单数据中已存在的业态
                Set<String> ruianTypeCur = new HashSet<>();
                //用来存放没有账单的业态
                Set<String> ruianTypeNew = new HashSet<>();
                //遍历数据,提取实际数据中的业态
                for(ProjectTypeShopNumber obj:map.get(mallName)){
                    ruianTypeCur.add(obj.getN());
                }
                //将产生账单的业态和全量业态求差即
                ruianTypeNew.addAll(ruianTypeAll);
                ruianTypeNew.removeAll(ruianTypeCur);
                //将没有实际账单的业态数据填从(默认值填充)                            
                for(String type_:ruianTypeNew){
                    map.get(mallName).add(new ProjectTypeShopNumber(type_,0.0,0));
                }
            }
        });
        //为==各项目店铺数量==填充数据
        projectShopNumber(map);
        //转成json
        Map<String, Map<String,List<ProjectTypeShopNumber>>> pmap=new HashMap<>();
        pmap.put("各项目业态销售额对比", map);
        String str=JSON.toJSONString(pmap);
        System.out.println("====####--projectTypeSaleContrast##"+str);
        RedisUtil.setObject("dptsc_11",str);
    }
    
    //=============================================================================================================
    /**
     * 结账单数
     * 
     * @param tsb
     */
    public void totalSettlementBillNumber(TotalSettlementBillNumber tsb) {
         Map<String, TotalSettlementBillNumber> map=new HashMap<>();
         map.put("结账单数", tsb);
         String str=JSONObject.toJSONString(map);
         System.out.println("====####--totalSettlementBillNumber##"+str);
         //将结账单数放入redis
         RedisUtil.setObject("dtsbn_6", str);
    }
    
    /**
     * 各区域销售占比
     * 直接将数据封装成json,无需额外处理
     * 
     * @param rs 已经封装好的数据,请参考{@link com.mengyao.graph.etl.apps.dashboard.service.SaleAnalysisService.projectContrast()}
     */
    private void areaSaleProportion(Map<String, List<AreaProjSale>> rs) {
        Map<String, Map<String,List<AreaProjSale>>> pmap=new HashMap<>();
        pmap.put("各区域销售额占比 ", rs);
        //转成json
        String str=JSONObject.toJSONString(pmap);
        System.out.println("====####--areaSaleProportion##"+str);
        //各区域销售占比dasp_1
        RedisUtil.setObject("dasp_1", str);
    }
    
    /**
     *  各项目单均消费
     *  版本二 营业时间24小时制
     *    1、各项目:各个mall;
     *    2、单均消费:mall的当日开始营业时间到当前时间累计销售额,应收额累计/结账单累计;
     *  直接将数据封装成json,无需额外处理
     *  
     *  @param rs 已经封装好的数据,请参考{@link com.mengyao.graph.etl.apps.dashboard.service.SaleAnalysisService.projectContrast()}
     */
    private void projectAvgConsumer(Map<String, List<AreaProjSale>> rs) {
        Map<String, Map<String,List<AreaProjSale>>> pmap=new HashMap<>();
        pmap.put("各项目单均消费", rs);
        //转成json
        String str=JSONObject.toJSONString(pmap);
        System.out.println("====####--projectAvgConsumer##"+str);
        //10各项目单均消费dpac_10
        RedisUtil.setObject("dpac_10", str);        
    }
    
    /**
     * 各项目店铺数量
     * 数据填充自上面的方法
     * 
     * {@link projectTypeSaleContrast(SparkSession session,String beginYMDH,String endYMDH)}
     */
    private void projectShopNumber(Map<String,List<ProjectTypeShopNumber>> map) {
        //转成json
        Map<String, Map<String,List<ProjectTypeShopNumber>>> pmap=new HashMap<>();
        pmap.put("各项目店铺数量", map);
        String str=JSON.toJSONString(pmap);
        System.out.println("====####--projectShopNumber##"+str);
        RedisUtil.setObject("dpsn_12",str);
    }
    
    
}