java多线程 下载 java多线程断点下载

基本原理：利用URLConnection获取要下载文件的长度、头部等相关信息，并设置响应的头部信息。并且通过URLConnection获取输入流，将文件分成指定的块，每一块单独开辟一个线程完成数据的读取、写入。通过输入流读取下载文件的信息，然后将读取的信息用RandomAccessFile随机写入到本地文件中。同时，每个线程写入的数据都文件指针也就是写入数据的长度，需要保存在一个临时文件中。这样当本次下载没有完成的时候，下次下载的时候就从这个文件中读取上一次下载的文件长度，然后继续接着上一次的位置开始下载。并且将本次下载的长度写入到这个文件中。

基本原理：利用URLConnection获取要下载文件的长度、头部等相关信息，并设置响应的头部信息。并且通过URLConnection获取输入流，将文件分成指定的块，每一块单独开辟一个线程完成数据的读取、写入。通过输入流读取下载文件的信息，然后将读取的信息用RandomAccessFile随机写入到本地文件中。同时，每个线程写入的数据都文件指针也就是写入数据的长度，需要保存在一个临时文件中。这样当本次下载没有完成的时候，下次下载的时候就从这个文件中读取上一次下载的文件长度，然后继续接着上一次的位置开始下载。并且将本次下载的长度写入到这个文件中。

一、下载文件信息类、实体

封装即将下载资源的信息

DownloadInfo.java

package com.hoo.entity;


/**

 * <b>function:</b> 下载文件信息类

 * @author hoojo

 * @createDate 2011-9-21 下午05:14:58

 * @file DownloadInfo.java

 * @package com.hoo.entity

 * @project MultiThreadDownLoad

 * @blog 

 * @email [e-mail]

 * @version 1.0

*/

public class DownloadInfo {

//下载文件url

    private String url;

//下载文件名称

    private String fileName;

//下载文件路径

    private String filePath;

//分成多少段下载， 每一段用一个线程完成下载

    private int splitter;


//下载文件默认保存路径

    private final static String FILE_PATH = "C:/temp";

//默认分块数、线程数

    private final static int SPLITTER_NUM = 5;


public DownloadInfo() {

super();

    }


/**

     * @param url 下载地址

*/

public DownloadInfo(String url) {

this(url, null, null, SPLITTER_NUM);

    }


/**

     * @param url 下载地址url

     * @param splitter 分成多少段或是多少个线程下载

*/

public DownloadInfo(String url, int splitter) {

this(url, null, null, splitter);

    }


/***

     * @param url 下载地址

     * @param fileName 文件名称

     * @param filePath 文件保存路径

     * @param splitter 分成多少段或是多少个线程下载

*/

public DownloadInfo(String url, String fileName, String filePath, int splitter) {

super();

if (url == null || "".equals(url)) {

throw new RuntimeException("url is not null!");

        }

this.url =  url;

this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName;

this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath;

this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter;

    }


/**

     * <b>function:</b> 通过url获得文件名称

     * @author hoojo

     * @createDate 2011-9-30 下午05:00:00

     * @param url

     * @return

*/

private String getFileName(String url) {

return url.substring(url.lastIndexOf("/") + 1, url.length());

    }


public String getUrl() {

return url;

    }


public void setUrl(String url) {

if (url == null || "".equals(url)) {

throw new RuntimeException("url is not null!");

        }

this.url = url;

    }


public String getFileName() {

return fileName;

    }


public void setFileName(String fileName) {

this.fileName = (fileName == null || "".equals(fileName)) ? getFileName(url) : fileName;

    }


public String getFilePath() {

return filePath;

    }


public void setFilePath(String filePath) {

this.filePath = (filePath == null || "".equals(filePath)) ? FILE_PATH : filePath;

    }


public int getSplitter() {

return splitter;

    }


public void setSplitter(int splitter) {

this.splitter = (splitter < 1) ? SPLITTER_NUM : splitter;

    }


    @Override

public String toString() {

return this.url + "#" + this.fileName + "#" + this.filePath + "#" + this.splitter;

    }

}

二、随机写入一段文件

这个类主要是完成向本地的指定文件指针出开始写入文件，并返回当前写入文件的长度（文件指针）。这个类将被线程调用，文件被分成对应的块后，将被线程调用。每个线程都将会调用这个类完成文件的随机写入。

SaveItemFile.java

package com.hoo.download;


import java.io.IOException;

import java.io.RandomAccessFile;


/**

 * <b>function:</b> 写入文件、保存文件

 * @author hoojo

 * @createDate 2011-9-21 下午05:44:02

 * @file SaveItemFile.java

 * @package com.hoo.download

 * @project MultiThreadDownLoad

 * @blog 

 * @email [e-mail]

 * @version 1.0

*/

public class SaveItemFile {

//存储文件

    private RandomAccessFile itemFile;


public SaveItemFile() throws IOException {

this("", 0);

    }


/**

     * @param name 文件路径、名称

     * @param pos 写入点位置 position

     * @throws IOException

*/

public SaveItemFile(String name, long pos) throws IOException {

        itemFile = new RandomAccessFile(name, "rw");

//在指定的pos位置开始写入数据

        itemFile.seek(pos);

    }


/**

     * <b>function:</b> 同步方法写入文件

     * @author hoojo

     * @createDate 2011-9-26 下午12:21:22

     * @param buff 缓冲数组

     * @param start 起始位置

     * @param length 长度

     * @return

*/

public synchronized int write(byte[] buff, int start, int length) {

int i = -1;

try {

            itemFile.write(buff, start, length);

            i = length;

        } catch (IOException e) {

            e.printStackTrace();

        }

return i;

    }


public void close() throws IOException {

if (itemFile != null) {

            itemFile.close();

        }

    }

}

 

三、单个线程下载文件

这个类主要是完成单个线程的文件下载，将通过URLConnection读取指定url的资源信息。然后用InputStream读取文件内容，然后调用调用SaveItemFile类，向本地写入当前要读取的块的内容。

DownloadFile.java

package com.hoo.download;


import java.io.IOException;

import java.io.InputStream;

import java.net.HttpURLConnection;

import java.net.MalformedURLException;

import java.net.URL;

import java.net.URLConnection;

import com.hoo.util.LogUtils;


/**

 * <b>function:</b> 单线程下载文件

 * @author hoojo

 * @createDate 2011-9-22 下午02:55:10

 * @file DownloadFile.java

 * @package com.hoo.download

 * @project MultiThreadDownLoad

 * @blog 

 * @email [e-mail]

 * @version 1.0

*/

public class DownloadFile extends Thread {


//下载文件url

    private String url;

//下载文件起始位置  

    private long startPos;

//下载文件结束位置

    private long endPos;

//线程id

    private int threadId;


//下载是否完成

    private boolean isDownloadOver = false;


private SaveItemFile itemFile;


private static final int BUFF_LENGTH = 1024 * 8;


/**

     * @param url 下载文件url

     * @param name 文件名称

     * @param startPos 下载文件起点

     * @param endPos 下载文件结束点

     * @param threadId 线程id

     * @throws IOException

*/

public DownloadFile(String url, String name, long startPos, long endPos, int threadId) throws IOException {

super();

this.url = url;

this.startPos = startPos;

this.endPos = endPos;

this.threadId = threadId;

//分块下载写入文件内容

        this.itemFile = new SaveItemFile(name, startPos);

    }



    @Override

public void run() {

while (endPos > startPos && !isDownloadOver) {

try {

                URL url = new URL(this.url);

                HttpURLConnection conn = (HttpURLConnection) url.openConnection();


// 设置连接超时时间为10000ms

                conn.setConnectTimeout(10000);

// 设置读取数据超时时间为10000ms

                conn.setReadTimeout(10000);


                setHeader(conn);


                String property = "bytes=" + startPos + "-";

                conn.setRequestProperty("RANGE", property);


//输出log信息

                LogUtils.log("开始 " + threadId + "：" + property + endPos);

//printHeader(conn);


//获取文件输入流，读取文件内容

                InputStream is = conn.getInputStream();


byte[] buff = new byte[BUFF_LENGTH];

int length = -1;

                LogUtils.log("#start#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos);

while ((length = is.read(buff)) > 0 && startPos < endPos && !isDownloadOver) {

//写入文件内容，返回最后写入的长度

                    startPos += itemFile.write(buff, 0, length);

                }

                LogUtils.log("#over#Thread: " + threadId + ", startPos: " + startPos + ", endPos: " + endPos);

                LogUtils.log("Thread " + threadId + " is execute over!");

this.isDownloadOver = true;

            } catch (MalformedURLException e) {

                e.printStackTrace();

            } catch (IOException e) {

                e.printStackTrace();

            } finally {

try {

if (itemFile != null) {

                        itemFile.close();

                    }

                } catch (IOException e) {

                    e.printStackTrace();

                }

            }

        }

if (endPos < startPos && !isDownloadOver) {

            LogUtils.log("Thread " + threadId  + " startPos > endPos, not need download file !");

this.isDownloadOver = true;

        }

if (endPos == startPos && !isDownloadOver) {

            LogUtils.log("Thread " + threadId  + " startPos = endPos, not need download file !");

this.isDownloadOver = true;

        }

    }


/**

     * <b>function:</b> 打印下载文件头部信息

     * @author hoojo

     * @createDate 2011-9-22 下午05:44:35

     * @param conn HttpURLConnection

*/

public static void printHeader(URLConnection conn) {

int i = 1;

while (true) {

            String header = conn.getHeaderFieldKey(i);

            i++;

if (header != null) {

                LogUtils.info(header + ":" + conn.getHeaderField(i));

            } else {

break;

            }

        }

    }


/**

     * <b>function:</b> 设置URLConnection的头部信息，伪装请求信息

     * @author hoojo

     * @createDate 2011-9-28 下午05:29:43

     * @param con

*/

public static void setHeader(URLConnection conn) {

        conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092510 Ubuntu/8.04 (hardy) Firefox/3.0.3");

        conn.setRequestProperty("Accept-Language", "en-us,en;q=0.7,zh-cn;q=0.3");

        conn.setRequestProperty("Accept-Encoding", "utf-8");

        conn.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");

        conn.setRequestProperty("Keep-Alive", "300");

        conn.setRequestProperty("connnection", "keep-alive");

        conn.setRequestProperty("If-Modified-Since", "Fri, 02 Jan 2009 17:00:05 GMT");

        conn.setRequestProperty("If-None-Match", "\"1261d8-4290-df64d224\"");

        conn.setRequestProperty("Cache-conntrol", "max-age=0");

        conn.setRequestProperty("Referer", "http://www.baidu.com");

    }


public boolean isDownloadOver() {

return isDownloadOver;

    }


public long getStartPos() {

return startPos;

    }


public long getEndPos() {

return endPos;

    }

}

 

四、分段多线程写入文件内容

这个类主要是完成读取指定url资源的内容，获取该资源的长度。然后将该资源分成指定的块数，将每块的起始下载位置、结束下载位置，分别保存在一个数组中。每块都单独开辟一个独立线程开始下载。在开始下载之前，需要创建一个临时文件，写入当前下载线程的开始下载指针位置和结束下载指针位置。

BatchDownloadFile.java

package com.hoo.download;


import java.io.DataInputStream;

import java.io.DataOutputStream;

import java.io.File;

import java.io.FileInputStream;

import java.io.FileOutputStream;

import java.io.IOException;

import java.net.HttpURLConnection;

import java.net.MalformedURLException;

import java.net.URL;

import com.hoo.entity.DownloadInfo;

import com.hoo.util.LogUtils;


/**

 * <b>function:</b> 分批量下载文件

 * @author hoojo

 * @createDate 2011-9-22 下午05:51:54

 * @file BatchDownloadFile.java

 * @package com.hoo.download

 * @project MultiThreadDownLoad

 * @blog

 * @email [e-mail]

 * @version 1.0

*/

public class BatchDownloadFile implements Runnable {

//下载文件信息 

    private DownloadInfo downloadInfo;

//一组开始下载位置

    private long[] startPos;

//一组结束下载位置

    private long[] endPos;

//休眠时间

    private static final int SLEEP_SECONDS = 500;

//子线程下载

    private DownloadFile[] fileItem;

//文件长度

    private int length;

//是否第一个文件

    private boolean first = true;

//是否停止下载

    private boolean stop = false;

//临时文件信息

    private File tempFile;


public BatchDownloadFile(DownloadInfo downloadInfo) {

this.downloadInfo = downloadInfo;

        String tempPath = this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName() + ".position";

        tempFile = new File(tempPath);

//如果存在读入点位置的文件

        if (tempFile.exists()) {

            first = false;

//就直接读取内容

            try {

                readPosInfo();

            } catch (IOException e) {

                e.printStackTrace();

            }

        } else {

//数组的长度就要分成多少段的数量

            startPos = new long[downloadInfo.getSplitter()];

            endPos = new long[downloadInfo.getSplitter()];

        }

    }


    @Override

public void run() {

//首次下载，获取下载文件长度

        if (first) {

            length = this.getFileSize();//获取文件长度

            if (length == -1) {

                LogUtils.log("file length is know!");

                stop = true;

            } else if (length == -2) {

                LogUtils.log("read file length is error!");

                stop = true;

            } else if (length > 0) {

/**

                 * eg 

                 * start: 1, 3, 5, 7, 9

                 * end: 3, 5, 7, 9, length

*/

for (int i = 0, len = startPos.length; i < len; i++) {

int size = i * (length / len);

                    startPos[i] = size;


//设置最后一个结束点的位置

                    if (i == len - 1) {

                        endPos[i] = length;

                    } else {

                        size = (i + 1) * (length / len);

                        endPos[i] = size;

                    }

                    LogUtils.log("start-end Position[" + i + "]: " + startPos[i] + "-" + endPos[i]);

                }

            } else {

                LogUtils.log("get file length is error, download is stop!");

                stop = true;

            }

        }


//子线程开始下载

        if (!stop) {

//创建单线程下载对象数组

            fileItem = new DownloadFile[startPos.length];//startPos.length = downloadInfo.getSplitter()

            for (int i = 0; i < startPos.length; i++) {

try {

//创建指定个数单线程下载对象，每个线程独立完成指定块内容的下载

                    fileItem[i] = new DownloadFile(

                        downloadInfo.getUrl(), 

this.downloadInfo.getFilePath() + File.separator + downloadInfo.getFileName(), 

                        startPos[i], endPos[i], i

                    );

                    fileItem[i].start();//启动线程，开始下载

                    LogUtils.log("Thread: " + i + ", startPos: " + startPos[i] + ", endPos: " + endPos[i]);

                } catch (IOException e) {

                    e.printStackTrace();

                }

            }


//循环写入下载文件长度信息

            while (!stop) {

try {

                    writePosInfo();

                    LogUtils.log("downloading……");

                    Thread.sleep(SLEEP_SECONDS);

                    stop = true;

                } catch (IOException e) {

                    e.printStackTrace();

                } catch (InterruptedException e) {

                    e.printStackTrace();

                }

for (int i = 0; i < startPos.length; i++) {

if (!fileItem[i].isDownloadOver()) {

                        stop = false;

break;

                    }

                }

            }

            LogUtils.info("Download task is finished!");

        }

    }


/**

     * 将写入点数据保存在临时文件中

     * @author hoojo

     * @createDate 2011-9-23 下午05:25:37

     * @throws IOException

*/

private void writePosInfo() throws IOException {

        DataOutputStream dos = new DataOutputStream(new FileOutputStream(tempFile));

        dos.writeInt(startPos.length);

for (int i = 0; i < startPos.length; i++) {

            dos.writeLong(fileItem[i].getStartPos());

            dos.writeLong(fileItem[i].getEndPos());

//LogUtils.info("[" + fileItem[i].getStartPos() + "#" + fileItem[i].getEndPos() + "]");

        }

        dos.close();

    }


/**

     * <b>function:</b>读取写入点的位置信息

     * @author hoojo

     * @createDate 2011-9-23 下午05:30:29

     * @throws IOException

*/

private void readPosInfo() throws IOException {

        DataInputStream dis = new DataInputStream(new FileInputStream(tempFile));

int startPosLength = dis.readInt();

        startPos = new long[startPosLength];

        endPos = new long[startPosLength];

for (int i = 0; i < startPosLength; i++) {

            startPos[i] = dis.readLong();

            endPos[i] = dis.readLong();

        }

        dis.close();

    }


/**

     * <b>function:</b> 获取下载文件的长度

     * @author hoojo

     * @createDate 2011-9-26 下午12:15:08

     * @return

*/

private int getFileSize() {

int fileLength = -1;

try {

            URL url = new URL(this.downloadInfo.getUrl());

            HttpURLConnection conn = (HttpURLConnection) url.openConnection();


            DownloadFile.setHeader(conn);


int stateCode = conn.getResponseCode();

//判断http status是否为HTTP/1.1 206 Partial Content或者200 OK

            if (stateCode != HttpURLConnection.HTTP_OK && stateCode != HttpURLConnection.HTTP_PARTIAL) {

                LogUtils.log("Error Code: " + stateCode);

return -2;

            } else if (stateCode >= 400) {

                LogUtils.log("Error Code: " + stateCode);

return -2;

            } else {

//获取长度

                fileLength = conn.getContentLength();

                LogUtils.log("FileLength: " + fileLength);

            }


//读取文件长度

            /*for (int i = 1; ; i++) {

                String header = conn.getHeaderFieldKey(i);

                if (header != null) {

                    if ("Content-Length".equals(header)) {

                        fileLength = Integer.parseInt(conn.getHeaderField(i));

                        break;

                    }

                } else {

                    break;

                }

            }

*/


            DownloadFile.printHeader(conn);

        } catch (MalformedURLException e) {

            e.printStackTrace();

        } catch (IOException e) {

            e.printStackTrace();

        }

return fileLength;

    }

}

 

五、工具类、测试类

日志工具类

LogUtils.java

package com.hoo.util;


/**

 * <b>function:</b> 日志工具类

 * @author hoojo

 * @createDate 2011-9-21 下午05:21:27

 * @file LogUtils.java

 * @package com.hoo.util

 * @project MultiThreadDownLoad

 * @blog 

 * @email [e-mail]

 * @version 1.0

*/

public abstract class LogUtils {


public static void log(Object message) {

        System.err.println(message);

    }


public static void log(String message) {

        System.err.println(message);

    }


public static void log(int message) {

        System.err.println(message);

    }


public static void info(Object message) {

        System.out.println(message);

    }


public static void info(String message) {

        System.out.println(message);

    }


public static void info(int message) {

        System.out.println(message);

    }

}

 

下载工具类

DownloadUtils.java

package com.hoo.util;


import com.hoo.download.BatchDownloadFile;

import com.hoo.entity.DownloadInfo;


/**

 * <b>function:</b> 分块多线程下载工具类

 * @author hoojo

 * @createDate 2011-9-28 下午05:22:18

 * @file DownloadUtils.java

 * @package com.hoo.util

 * @project MultiThreadDownLoad

 * @blog 

 * @email [e-mail]

 * @version 1.0

*/

public abstract class DownloadUtils {


public static void download(String url) {

        DownloadInfo bean = new DownloadInfo(url);

        LogUtils.info(bean);

        BatchDownloadFile down = new BatchDownloadFile(bean);

new Thread(down).start();

    }


public static void download(String url, int threadNum) {

        DownloadInfo bean = new DownloadInfo(url, threadNum);

        LogUtils.info(bean);

        BatchDownloadFile down = new BatchDownloadFile(bean);

new Thread(down).start();

    }


public static void download(String url, String fileName, String filePath, int threadNum) {

        DownloadInfo bean = new DownloadInfo(url, fileName, filePath, threadNum);

        LogUtils.info(bean);

        BatchDownloadFile down = new BatchDownloadFile(bean);

new Thread(down).start();

    }

}

 

下载测试类

TestDownloadMain.java

package com.hoo.test;


import com.hoo.util.DownloadUtils;


/**

 * <b>function:</b> 下载测试

 * @author hoojo

 * @createDate 2011-9-23 下午05:49:46

 * @file TestDownloadMain.java

 * @package com.hoo.download

 * @project MultiThreadDownLoad

 * @blog

 * @email [e-mail]

 * @version 1.0

*/

public class TestDownloadMain {


public static void main(String[] args) {

/*DownloadInfo bean = new DownloadInfo("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg");

        System.out.println(bean);

        BatchDownloadFile down = new BatchDownloadFile(bean);

        new Thread(down).start();*/


//DownloadUtils.download("http://i7.meishichina.com/Health/UploadFiles/201109/2011092116224363.jpg");

        DownloadUtils.download("http://mp3.baidu.com/j?j=2&url=http%3A%2F%2Fzhangmenshiting2.baidu.com%2Fdata%2Fmusic%2F1669425%2F%25E9%2599%25B7%25E5%2585%25A5%25E7%2588%25B1%25E9%2587%258C%25E9%259D%25A2.mp3%3Fxcode%3D2ff36fb70737c816553396c56deab3f1", "aa.mp3", "c:/temp", 5);

    }

}

多线程下载主要在第三部和第四部，其他的地方还是很好理解。源码中提供相应的注释了，便于理解。