JAVA大文件保存分段存储与读取 java文件分段下载

转载

mob6454cc6f27a3 2024-06-13 13:01:52

文章标签 JAVA大文件保存分段存储与读取 java jvm 开发语言临时文件 文章分类 Java 后端开发

分析

例如：一个10字节的文件，一共开了3个线程，每个线程下载数：size = 10/3
0:0-2
1:3-5
2:6-9
开始位置：id * size
结束位置：(id + 1) * size - 1
最后一个线程的结束位置：length - 1

代码

package com.multidown;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.net.HttpURLConnection;
import java.net.URL;

/**
 * 例如：一个10字节的文件，一共开了3个线程，每个线程下载数：size = 10/3
 * 0:0-2
 * 1:3-5
 * 2:6-9
 * 开始位置：id * size
 * 结束位置：(id + 1) * size - 1
 * 最后一个线程的结束位置：length - 1 
 */
public class MultiDownload {
    // 启用线程总数
    static int threadCount = 3;
    // 已完成下载总数
    static int threadFinished = 0;

    // 要下载的文件URL地址
    static String urlFilePath = "http://localhost:8080/PdfJs/11111111.pdf";
    // 文件下载位置
    static String downloadFilePath = "E://11111111.pdf";
    
    public static void main(String[] args) {
        try {
            // 发送get请求，请求这个地址的资源
            URL url = new URL(urlFilePath);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setReadTimeout(5000);
            conn.setConnectTimeout(5000);
            if (conn.getResponseCode() == 200) {
                // 拿到所请求资源文件的长度
                int length = conn.getContentLength();
                // 主线程创建临时文件，子线程将各自下载的数据写入到此文件中
                File file = new File(downloadFilePath);
                // rwd模式，将下载的数据实时同步到硬盘，而不会存入到缓冲中
                RandomAccessFile raf = new RandomAccessFile(file, "rwd");
                raf.setLength(length);
                raf.close();
                // 计算出每个线程应该下载多少字节
                int size = length / threadCount;
                for (int i = 0; i < threadCount; i++) {
                    // 计算线程下载的开始位置和结束位置
                    int startIndex = i * size;
                    int endIndex = (i + 1) * size - 1;
                    if (i == threadCount - 1) {
                        endIndex = length - 1;        // 最后一个线程的结束位置
                    }
                    new DownLoadThread(i, startIndex, endIndex).start();
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

class DownLoadThread extends Thread {
    int threadId;
    int startIndex;
    int endIndex;

    public DownLoadThread(int threadId, int startIndex, int endIndex) {
        super();
        this.threadId = threadId;
        this.startIndex = startIndex;
        this.endIndex = endIndex;
    }

    @Override
    public void run() {
        // 再次发送http请求，下载原文件
        try {
            // 断点续传：记录用户下载进度
            File processfile = new File("E://" + threadId + ".txt");
            // 判断是否有断点续传文件
            if (processfile.exists()) {
                InputStream is = new FileInputStream(processfile);
                BufferedReader br = new BufferedReader(new InputStreamReader(is));
                // 从进度临时文件中读取出上一次下载的总进度，然后与原本的开始位置相加，得到新的开始位置
                startIndex += Integer.parseInt(br.readLine()); 
                is.close();
            }
            System.out.println("线程" + threadId + "的下载区间是:" + startIndex + "----" + endIndex);

            URL url = new URL(MultiDownload.urlFilePath);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setRequestMethod("GET");
            conn.setReadTimeout(5000);
            conn.setConnectTimeout(5000);
            // 设置本次http请求的数据区间
            conn.setRequestProperty("Range", "bytes=" + startIndex + "-" + endIndex);
            // 请求部分数据，响应码是206
            if (conn.getResponseCode() == 206) {
                InputStream is = conn.getInputStream();
                byte buf[] = new byte[1024]; // 1K
                File file = new File(MultiDownload.downloadFilePath);
                RandomAccessFile raf = new RandomAccessFile(file, "rwd");
                // 把文件的写入位置移动至startIndex
                raf.seek(startIndex);
                int len = 0;
                int total = 0;
                while ((len = is.read(buf)) != -1) {
                    // 文件写入
                    raf.write(buf, 0, len);
                    total += len;
                    System.out.println("线程" + threadId + "下载了:" + total);

                    // 生成一个专门用来记录下载进度的临时文件
                    RandomAccessFile processraf = new RandomAccessFile(processfile, "rwd");
                    processraf.write((total + "").getBytes());
                    processraf.close();
                }
                raf.close();
                System.out.println("线程" + threadId + "下载完毕---------------------");

                // 都下载完成，删除断点续传临时文件
                MultiDownload.threadFinished++;
                synchronized (MultiDownload.urlFilePath) {
                    if (MultiDownload.threadFinished == MultiDownload.threadCount) {
                        for (int i = 0; i < MultiDownload.threadCount; i++) {
                            File temp = new File("E://" + i + ".txt");
                            temp.delete();
                        }
                        MultiDownload.threadFinished = 0;
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

}