JAVA使用HttpURLConnection进行多线程文件下载

遇到了一个下载文件的问题

在开发中，需要实现一个文件下载的方法，对下载时间有一点要求，对于小文件来说，问题不大，单线程下载既可;

• 单线程下载文件：

首先使用HttpURLConnection获取文件流；
创建RandomAccessFile文件对象，用于写入；
使用 randomAccessFile.write(buffer,0,size);将流转换字节写入文件
另外，可以启动一个单独的线程，记录下载进度；
在HttpURLConnection请求后记录总大小，
在写入文件时记录已下载大小；
使用NumberFormat记录输出百分比；

public boolean downloadFile(String url,String dest_path,int downloadTimeout,boolean downloadResume){

        //检测下载进度
        new Thread(new Runnable() {
            @Override
            public void run() {
                try {
                    NumberFormat numberFormat = NumberFormat.getPercentInstance();
                    numberFormat.setMinimumFractionDigits(2);
                    logger.debug("是否下载完成：{}",downloadFinish);
                    while (!downloadFinish){
                        Thread.sleep(10000);
                        if(totalSize > 0){
                            logger.info("已下载大小{},进度：{},如果长时间不更新,可能是进度汇报现场卡了，不影响下载",
                                    getDownloadSize(),
                                numberFormat.format(getDownloadSize()*1.0/getTotalSize()));
                        }
                    }
                }catch (Throwable e){
                    logger.error("记录下载文件进度出错，{}",e.getMessage(),e);
                    e.printStackTrace();
                }
            }
        }).start();

        logger.debug("开始下载,源文件地址：{},\n 目标地址：{},\n 设置的超时时间为：{}",url,dest_path,downloadTimeout);
        try {
            RandomAccessFile randomAccessFile = null;
            InputStream in = null;
            URL fileUrl = new URL(url);
            HttpURLConnection connection = (HttpURLConnection)fileUrl.openConnection();
            connection.setConnectTimeout(downloadTimeout);
            connection.setRequestMethod("GET");
            File tempFile = new File(dest_path+"_tmp");// 临时文件
            if(!tempFile.getParentFile().exists()){
                tempFile.getParentFile().mkdirs();
            }
            if(downloadResume){
                if(tempFile.exists() && tempFile.isFile()){
                    downloadSize = tempFile.length();
                    startIndex = downloadSize;
                }
                connection.setRequestProperty("Range", "bytes=" + startIndex + "-");
            }else{
                if(tempFile.exists() && tempFile.isFile()){
                    tempFile.delete();
                }
            }
            int staus = connection.getResponseCode();
            totalSize = downloadSize + connection.getContentLengthLong();
            logger.debug("请求状态码：{},文件总大小：{}[{}],本地需要下载的大小：{}",staus,totalSize, totalSize/1024/1024, totalSize - downloadSize);
            if(staus == 200 || staus == 206){
                randomAccessFile = new RandomAccessFile(tempFile,"rwd");
                randomAccessFile.seek(startIndex);
                in = connection.getInputStream();
                byte[] buffer = new byte[2048 * 10];
                int size = 0;
                while ((size = in.read(buffer))!= -1){
                    randomAccessFile.write(buffer,0,size);
                    downloadSize = downloadSize + size;
                }
                randomAccessFile.close();
            }

            in.close();
            File dest = new File(dest_path);
            this.setDownloadFinish(true);
            return  tempFile.renameTo(dest);
        }catch (Throwable e){
            logger.error("下载文件出错，{}",e.getMessage(),e);
        }finally {
            this.setDownloadFinish(true);
        }
        return false;
    }

但对于大文件（超5G）来说，下载时间就会很长，当然这个也需要考虑网络和硬件的关系；但是可以通过多线程的方式下载文件；

• 多线程下载文件:

  多线程分片下载文件， 获取文件总大小， 分成指定的份数，再启动指定的线程去下载自己的那一份；

  //获取总大小：
  private Long getRemoteFileSize(String remoteFileUrl) throws IOException {
        Long fileSize = 0L ;
        HttpURLConnection httpConnection = (HttpURLConnection) new URL(remoteFileUrl).openConnection();
        httpConnection.setRequestMethod("HEAD");
        Integer responseCode = 0;
        try {
            responseCode = httpConnection.getResponseCode();
        } catch (IOException e) {
            e.printStackTrace();
        }
        if (responseCode >= 400) {
            System.out.println("Web服务器响应错误!请稍后重试");
            return fileSize;
        }
        String sHeader;
        for (Integer i = 1; ; i++) {
            sHeader = httpConnection.getHeaderFieldKey(i);
            if ("Content-Length".equals(sHeader)) {
                fileSize = Long.parseLong(httpConnection.getHeaderField(sHeader));
                break;
            }
        }
        return fileSize;
    }

分片：

 CountDownLatch countDownLatch = new CountDownLatch(poolLength);
List<FileDownloadRunnable> downloadRunnables = new ArrayList<>();

        Long partSize = length / poolLength;
        for (int i = 1; i <= poolLength; i++) {
            //start = i * length / poolLength;
            //long end = (i + 1) * length / poolLength - 1;
            // 每一个线程下载的开始位置
            Long start = (i - 1) * partSize;
            // 每一个线程下载的开始位置
            Long end = start + partSize - 1;
            if (i == poolLength) {
                //最后一个线程下载的长度稍微长一点
                end = length;
            }
            //System.out.println(start+"---------------"+end);
            FileDownloadRunnable fileDownloadRunnable =
                    new FileDownloadRunnable(url, dest_path, start, end,countDownLatch);
            downloadRunnables.add(fileDownloadRunnable);
            Thread thread = new Thread(fileDownloadRunnable);
            thread.setName("下载线程"+i);
            thread.start();
        }
         countDownLatch.await(); //等待主线程完成
        for (FileDownloadRunnable downloadRunnable:downloadRunnables) {
            boolean downloadFinish = downloadRunnable.isDownloadFinish();
            logger.debug("多个线程是否都下载完成："+downloadRunnable.isDownloadFinish());
            if(!downloadFinish){
                throw  new RuntimeException("链接文件失败，无法落地");
            }
        }
        //FileDownloadRunnable.mergeFiles(dest_path,poolLength,false);
        long endTime = System.currentTimeMillis();
        logger.debug("文件下载完成，总耗时：{}s",(endTime - startTime) / 1000);
        return true;

下载：

public class FileDownloadRunnable implements Runnable{

    //下载路径
    private String urlLocation;
    //下载完存放路径
    private String filePath;
    //文件开始地址
    public Long start;
    //文件结束地址
    public Long end;

    private CountDownLatch countDownLatch;

    ThreadLocal<Long> startTime = new ThreadLocal<>();

    NumberFormat numberFormat = NumberFormat.getPercentInstance();
    Long allSize = 1L;

    private boolean downloadFinish = false;


    public FileDownloadRunnable(String urlLocation, String filePath, Long start, Long end, CountDownLatch countDownLatch) {
        this.urlLocation = urlLocation;
        this.filePath = filePath;
        this.start = start;
        this.end = end;
        this.countDownLatch = countDownLatch;
        numberFormat.setMinimumFractionDigits(2);
        allSize = end-start;
    }

    @Override
    public void run() {
        InputStream is = null;
        //FileOutputStream out = null;
        RandomAccessFile out = null;
        try {
            File file = new File(filePath);
            if(!file.getParentFile().exists()){
                file.getParentFile().mkdirs();
            }
            startTime.set(System.currentTimeMillis());
            System.out.println(start+"-"+end);
            //获取下载的部分
            URL url = new URL(urlLocation);
            HttpURLConnection conn = (HttpURLConnection) url.openConnection();
            conn.setReadTimeout(6000);
            conn.setRequestMethod("GET");
            conn.setRequestProperty("Range", "bytes=" + start + "-" + end);
            conn.setRequestProperty("Connection", "Keep-Alive");
            //conn.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1");
            //conn.connect();
            int statusCode = conn.getResponseCode();
            long length = conn.getContentLengthLong();
            System.out.println(Thread.currentThread().getName() +"：请求状态码："+statusCode+"，请求到的大小:"+length+"="+length/1024/1024+"M");
            is = conn.getInputStream();
            //System.out.println("getInputStream().available():"+is.available());
            //BufferedInputStream bis = new BufferedInputStream(is);
            //out = new FileOutputStream(filePath);
            out = new RandomAccessFile(file, "rw");
            out.seek(start);
            //System.out.println("out.seek:"+start);
            byte[] bytes = new byte[2048 * 10];
            int l = 0;
            //int totalSize = 0;
            while ((l = is.read(bytes))!=-1){
                //totalSize += l;
                out.write(bytes,0,l);
            }
            //bis.close();
            out.close();
            is.close();
            long time = System.currentTimeMillis()  - startTime.get();
            downloadFinish = true; //下载完成标识
            System.out.println(Thread.currentThread().getName()+"完成下载！共耗时(秒):" + time/1000);
        } catch (Throwable e) {
            e.printStackTrace();
            throw new RuntimeException(Thread.currentThread().getName()+"下载失败",e);
        } finally {
            countDownLatch.countDown();
        }

    }
 public boolean isDownloadFinish() {
        return downloadFinish;
    }

未解决的问题：

下载时遇到一个问题，挂载盘和本地盘下载结果不一致；不知道是不是挂载盘方式的问题？