mysql写入时用随机数字吗 mysql 写入10w 每秒

转载

mob6454cc68310b 2023-08-11 13:04:53

文章标签 mysql写入时用随机数字吗 mysql 数据库 java ci 文章分类 MySQL 数据库

作为一个Java后台，MySQL的使用是家常便饭，虽然平常不会遇到几十万甚至几百万的数据插入情况，但是耐不住我的好奇心，今天写几个方法用来测试一下

我们使用控制变量来对比出来最适合数据批量插入的条件
罗列一下条件（电脑硬件这种暂时改变不了）：

插入方法（循环插入、forEach插入、批处理）
数据库存储引擎（MyISAM、InnoDB）
主键（UUID、自增）

如果想到其他条件可以在评论区留言，我看到后进行测试

JDBC连接url中添加参数rewriteBatchedStatements=true可提高插入速度
先把测试结果放在这(单位：毫秒)

	10条	100条	1000条	10000条	1000000条
MyISAM（循环插入、自增）	29	211	1445	15322
MyISAM（forEach插入、自增）	5	15	127	1140	115120
MyISAM（批处理、自增）	5	9	60	510	74531

	10条	100条	1000条	10000条	1000000条
MyISAM（循环插入、UUID）	24	297	1331	13328
MyISAM（forEach插入、UUID）	8	24	111	1361	178578
MyISAM（批处理、UUID）	4	12	74	571	104161

	10条	100条	1000条	10000条	1000000条
InnoDB（循环插入、UUID）	24	179	2051	20724
InnoDB（forEach插入、UUID）	3	13	108	1068	267536
InnoDB（批处理、UUID）	9	36	262	2838	205075

像循环插入这种比较慢的方式就不进行百万条数据测试了，根据上面测试结果可以分析出，在数据量较大时

批处理、自增主键、MyISAM存储引擎是最优解，单线程插入1000000条数据需要70余秒

另外还有两个多线程方法，分别使用forEach和批处理两种方法，目的就是寻找插入一百万条数据最快的方法，这里根据电脑硬件不同会有差别

这里保持MyISAM存储引擎和自增主键的条件
控制变量为：插入方式和每次插入数量
50/20000意思为：循环50次，每次插入20000条数据

	50/20000	40/25000	25/40000	20/50000	10/100000
MyISAM（forEach插入、自增）	62494	71365	73018	66428	129074
MyISAM（批处理、自增）	42178	44786	44607	28964	55948

100万条数据最快需要30秒左右时间插入MySQL，这是目前我在本地测试的最短时间

测试使用数据表结构

/*
 Navicat Premium Data Transfer

 Source Server         : fitness
 Source Server Type    : MySQL
 Source Server Version : 80026
 Source Host           : 127.0.0.1:3306
 Source Schema         : fitness

 Target Server Type    : MySQL
 Target Server Version : 80026
 File Encoding         : 65001

 Date: 16/12/2021 15:46:55
*/

SET NAMES utf8mb4;
SET FOREIGN_KEY_CHECKS = 0;

-- ----------------------------
-- Table structure for user
-- ----------------------------
DROP TABLE IF EXISTS `user`;
CREATE TABLE `user`  (
  `id` varchar(50) CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL COMMENT '用户id',
  `user_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '用户昵称',
  `real_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '用户真实姓名',
  `sex` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '性别0保密1男2女',
  `phone` varchar(13) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '联系方式',
  `img` text CHARACTER SET utf8 COLLATE utf8_general_ci NULL COMMENT '头像',
  `birthday` date NULL DEFAULT NULL COMMENT '生日',
  `address` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '地址',
  `create_time` datetime(0) NULL DEFAULT NULL COMMENT '创建时间',
  `update_time` datetime(0) NULL DEFAULT NULL COMMENT '修改时间',
  PRIMARY KEY (`id`) USING BTREE,
  INDEX `id`(`id`) USING BTREE
) ENGINE = MyISAM CHARACTER SET = utf8 COLLATE = utf8_general_ci ROW_FORMAT = Dynamic;

SET FOREIGN_KEY_CHECKS = 1;

实体类结构

@Data
@EqualsAndHashCode(callSuper = false)
@Accessors(chain = true)
@ApiModel(value="User对象", description="")
@Builder
public class User implements Serializable {

    private static final long serialVersionUID = 1L;

    @ApiModelProperty(value = "用户id")
    @TableId(value = "id", type = IdType.UUID)
    private String id;

    @ApiModelProperty(value = "用户昵称")
    private String userName;

    @ApiModelProperty(value = "用户真实姓名")
    private String realName;

    @ApiModelProperty(value = "性别0保密1男2女")
    private String sex;

    @ApiModelProperty(value = "联系方式")
    private String phone;

    @ApiModelProperty(value = "头像")
    private String img;

    @ApiModelProperty(value = "生日")
    private Date birthday;

    @ApiModelProperty(value = "地址")
    private String address;

    @ApiModelProperty(value = "创建时间")
    @TableField(fill = FieldFill.INSERT)
    private Date createTime;

    @TableField(fill = FieldFill.INSERT_UPDATE)
    @ApiModelProperty(value = "修改时间")
    private Date updateTime;

}

service层方法

package com.lcp.fitness.service.Impl;

import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.lcp.fitness.dao.primary.UserMapper;
import com.lcp.fitness.entity.User;
import com.lcp.fitness.service.IUserService;
import lombok.extern.slf4j.Slf4j;
import org.apache.ibatis.session.ExecutorType;
import org.apache.ibatis.session.SqlSession;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionTemplate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;

/**
 * <p>
 *  服务实现类
 * </p>
 *
 * @author fitnessCode
 * @since 2021-12-15
 */
@Service
@Slf4j
public class UserServiceImpl extends ServiceImpl<UserMapper, User> implements IUserService {

    @Autowired
    private ThreadPoolExecutor executor;
    @Autowired
    private SqlSessionFactory sqlSessionFactory;
    @Autowired
    private SqlSessionTemplate sqlSessionTemplate;


    String[] names = {"刘备", "关羽", "张飞", "曹操", "诸葛亮", "庞统", "马超", "赵云", "黄忠", "司马懿"};
    String[] addrs = {"长坂坡", "北京", "江苏南京", "黑龙江哈尔滨", "山东青岛", "四川成都", "广东深圳", "山东济南", "福建福州", "河南郑州"};
    String[] imgs = {"https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fimg.jj20.com%2Fup%2Fallimg%2Ftp05%2F19100120461512E-0-lp.jpg&refer=http%3A%2F%2Fimg.jj20.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1642145607&t=d9fc7b5dceb96e2b31810e93a9f6723a",
    "https://gimg2.baidu.com/image_search/src=http%3A%2F%2Ffile02.16sucai.com%2Fd%2Ffile%2F2014%2F0829%2F372edfeb74c3119b666237bd4af92be5.jpg&refer=http%3A%2F%2Ffile02.16sucai.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1642145634&t=f901a85ce707cf4c776be406e5079849",
    "https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fimg.jj20.com%2Fup%2Fallimg%2Ftp01%2F1ZZQ20QJS6-0-lp.jpg&refer=http%3A%2F%2Fimg.jj20.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1642145650&t=2c2ff827abefebcd8eb1110848e52bcd",
    "https://img1.baidu.com/it/u=3108885758,3678256568&fm=26&fmt=auto",
    "https://img0.baidu.com/it/u=2211870254,3716611573&fm=26&fmt=auto",
    "https://img2.baidu.com/it/u=602572383,152242527&fm=26&fmt=auto",
    "https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fimg.daimg.com%2Fuploads%2Fallimg%2F111005%2F1-111005232354646.jpg&refer=http%3A%2F%2Fimg.daimg.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1642145758&t=eb28dfc9f364b46a4bbd6174862a660a",
    "https://img1.baidu.com/it/u=204866477,2500898168&fm=26&fmt=auto",
    "https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fimg.redocn.com%2Fsheying%2F20150316%2Fyueguangxiadebaishu_3994382.jpg&refer=http%3A%2F%2Fimg.redocn.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1642145769&t=2e08ec8a904994f768b33d9e58a09901",
    "https://gimg2.baidu.com/image_search/src=http%3A%2F%2Fimg.redocn.com%2Fsheying%2F20151026%2Fyiduidumuchuan_5181389.jpg&refer=http%3A%2F%2Fimg.redocn.com&app=2002&size=f9999,10000&q=a80&n=0&g=0n&fmt=jpeg?sec=1642145788&t=8cabdef305396193c5e7dcd7ac6b4bdc"};



    @Override
    public void insertListThreadForeach() {
        AtomicInteger integer = new AtomicInteger();
        log.info("本轮任务开始insertListThreadForeach");
        long startTime = System.currentTimeMillis();
        AtomicLong endTime = new AtomicLong();
        for (int i = 0; i < 10; i++) {
            Thread thread = new Thread(() ->{
                try {
                    this.getBaseMapper().insertList(getUserList(100000));
                    endTime.set(System.currentTimeMillis());
                    log.info("本轮任务耗时：" + (endTime.get() - startTime) + "——已执行" + integer.addAndGet(1));
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
            try {
                executor.execute(thread);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    @Override
    public void insertListThreadBatch() {
        AtomicInteger integer = new AtomicInteger();
        log.info("本轮任务开始insertListThreadBatch");
        long startTime = System.currentTimeMillis();
        AtomicLong endTime = new AtomicLong();
        for (int i = 0; i < 20; i++) {
            Thread thread = new Thread(() ->{
                try {
                    SqlSession sqlSession = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);
                    UserMapper mapper = sqlSession.getMapper(UserMapper.class);
                    List<User> userList = getUserList(50000);
                    userList.stream().forEach(user -> mapper.insert(user));
                    sqlSession.commit();
                    sqlSession.clearCache();
                    sqlSession.close();
                    endTime.set(System.currentTimeMillis());
                    log.info("本轮任务耗时：" + (endTime.get() - startTime) + "——已执行" + integer.addAndGet(1));
                } catch (Exception e) {
                    e.printStackTrace();
                }
            });
            try {
                executor.execute(thread);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    @Override
    public void insert(int count) {
        log.info("本轮任务开始insert");
        long startTime = System.currentTimeMillis();
        List<User> userList = getUserList(count);
        userList.stream().forEach(user -> this.getBaseMapper().insert(user));
        long endTime = System.currentTimeMillis();
        log.info("本轮任务结束insert，共插入" + count + "条数据---共耗时" + (endTime-startTime) + "毫秒");
    }

    @Override
    public void insertForeach(int count) {
        log.info("本轮任务开始insertForeach");
        long startTime = System.currentTimeMillis();
        List<User> userList = getUserList(count);
        for (int i = 0; i < count; i+=50000) {
            List<User> list;
            if (i + 50000 < count) {
                list = userList.subList(i, i + 50000);
            } else {
                list = userList.subList(i, count);
            }
            this.getBaseMapper().insertList(list);
        }
        long endTime = System.currentTimeMillis();
        log.info("本轮任务结束insertForeach，共插入" + count + "条数据---共耗时" + (endTime-startTime) + "毫秒");
    }

    @Override
    public void insertBatch(int count) {
        log.info("本轮任务开始insertBatch");
        long startTime = System.currentTimeMillis();
        List<User> userList = getUserList(count);
        /*SqlSession sqlSession = sqlSessionFactory.openSession(ExecutorType.BATCH, false);
        UserMapper mapper = sqlSession.getMapper(UserMapper.class);*/
        SqlSession sqlSession = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);
        UserMapper mapper = sqlSession.getMapper(UserMapper.class);
        for (int i = 0; i < count; i += 50000) {
            List<User> list;
            if (i + 50000 < count) {
                list = userList.subList(i, i + 50000);
            } else {
                list = userList.subList(i, count);
            }
            list.stream().forEach(user -> mapper.insert(user));
            sqlSession.commit();
            sqlSession.clearCache();
        }
        long endTime = System.currentTimeMillis();
        log.info("本轮任务结束insertBatch，共插入" + count + "条数据---共耗时" + (endTime-startTime) + "毫秒");
    }

    private List<User> getUserList(int count) {
        List<User> list = new ArrayList<>(count);
        for (int i = 0; i < count; i++) {
            list.add(getUser());
        }
        return list;
    }

    private User getUser() {
        Random random = new Random();
        User user = User.builder()
                .userName(names[random.nextInt(names.length)])
                .realName(names[random.nextInt(names.length)])
                .sex(String.valueOf(random.nextInt(2)))
                .phone(String.valueOf(18800000000L + random.nextInt(88888888)))
                .img(imgs[random.nextInt(imgs.length)])
                .birthday(new Date())
                .address(addrs[random.nextInt(addrs.length)]).build();
        return user;
    }

}

dao层

package com.lcp.fitness.dao.primary;

import com.lcp.fitness.entity.User;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Param;

import java.util.List;

/**
 * <p>
 *  Mapper 接口
 * </p>
 *
 * @author fitnessCode
 * @since 2021-12-15
 */
public interface UserMapper extends BaseMapper<User> {

    int insertList(List<User> users);

    int insert(User user);
}

xml文件

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.lcp.fitness.dao.primary.UserMapper">

    <insert id="insertList">
        insert into user(id, user_name, real_name, sex, phone, img, birthday, address, create_time, update_time)
        values
        <foreach collection="list" item="item" separator=",">
            (#{item.id}, #{item.userName}, #{item.realName}, #{item.sex}, #{item.phone}, #{item.img}, #{item.birthday}, #{item.address}, #{item.createTime}, now())
        </foreach>
    </insert>
    <insert id="insert">
        insert into user(id, user_name, real_name, sex, phone, img, birthday, address, create_time, update_time)
        values (#{id, jdbcType=INTEGER}, #{userName, jdbcType=VARCHAR}, #{realName, jdbcType=VARCHAR},
                #{sex, jdbcType=VARCHAR}, #{phone, jdbcType=VARCHAR}, #{img, jdbcType=VARCHAR},
                #{birthday,jdbcType=DATE}, #{address, jdbcType=VARCHAR}, #{createTime, jdbcType=DATE}, now())
    </insert>


</mapper>

多线程配置类

package com.lcp.fitness.config;

import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * @author : fitnessCode
 * @data : 2021/12/15 10:26
 * @description : 多线程配置
 */
@Configuration
public class ThreadPoolExecutorConfig {

    @Bean
    public ThreadPoolExecutor threadPoolExecutor() {
        ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(8, 12, 10, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100));
        threadPoolExecutor.allowCoreThreadTimeOut(true);
        return threadPoolExecutor;
    }
}

上面是测试代码，想要进行测试的话可以参考一下

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。