基于node的文件分片上传

需求背景

最近接了一些比较难搞的需求，主要任务是把单子关联的附件下载到本地，然后上传至文档库和ftp。这些文件原始出处有minio，有在集群之外的其他服务器。
上传过程要考虑不把系统打挂，控制好并发。
受制于网络环境，文件太大容易失败。

解决办法

• 采用node 快速开发。
• promise await 控制好并发
• 记录失败日志
• 和同事讨论后，采用分片上传，他给我shell脚本，我翻译成node

思路

1. 获取唯一标识
2. 获取文件md5值
3. 采用split-file分割文件
4. 循环分割文件上传

代码

// npm install crypto
// npm install request
// npm install split-file

const fs = require('fs');
const request = require('request');
const crypto = require('crypto');
const splitFile = require('split-file');

let repoId = 685;
let targetFilePath = "test/";
let artiName = "CentOS-7-x86_64-DVD-2009.iso";
// 文件路径
let fileName = "D:/vm/CentOS-7-x86_64-DVD-2009.iso";
let operationName = "石磊";
let artiType = 1;
let baseUrl = 'http://url'


// Split file size in MB
let fileSplitSize = 64;


function getChecksum(path) {
    return new Promise(function (resolve, reject) {
        // crypto.createHash('sha1');
        // crypto.createHash('sha256');
        const hash = crypto.createHash('md5');
        const input = fs.createReadStream(path);

        input.on('error', reject);

        input.on('data', function (chunk) {
            hash.update(chunk);
        });

        input.on('close', function () {
            resolve(hash.digest('hex'));
        });
    });
}

console.log("current file_split_size: " + fileSplitSize);

// Get upload identity
request.post({
    url: baseUrl + '/uuid',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ 'repoId': repoId })
}, function (error, response, body) {
    if (!error && response.statusCode == 200) {
        let identityResult = JSON.parse(body);
        let identity = identityResult.data;
        console.info('identity: ', identity);

        let splitFilePromise = splitFile.splitFileBySize(fileName, fileSplitSize * 1024 * 1024);
        let md5Promise = getChecksum(fileName);

        Promise.all([md5Promise, splitFilePromise])
            .then(async results => {
                let md5 = results[0];
                let names = results[1];
                for (let i = 0; i < names.length; i++) {
                    let tempPath = names[i];
                    console.log("tempPath: ", tempPath);
                    let formData = {
                        chunk: i,
                        chunks: names.length,
                        name: artiName,
                        md5: md5,
                        identity: identity,
                        repoId: repoId,
                        targetFilePath: targetFilePath,
                        artiName: artiName,
                        artiType: artiType,
                        operationName: operationName,
                        file: {
                            value: fs.createReadStream(tempPath),
                            options: {
                                filename: 'nameinpost',
                                contentType: 'multipart/form-data'
                            }
                        }
                    };

                    try {
                        await uploadFileChunk(formData);
                    } catch (err) {
                        console.error(err);
                    }


                }

            })
            .catch(err => {
                console.log(err);
            });

    }
});


async function uploadFileChunk(formData) {
    console.log("start -formData: ", formData);
    return new Promise((resolve, reject) => {
        request.post({
            url: baseUrl + '/api/uploadurl',
            formData: formData
        }, function (error, response, body) {
            if (!error && response.statusCode == 200) {
                let result = JSON.parse(body).data;
                console.log("Link: " + result);
                resolve(result);
            } else {
                console.log("Error from: ", formData);
                console.error("Error: ", error);
                reject(error);
            }
        });
    });
}

总结

放弃问题就没收获！