Logo

菜单

kyyee
发布于 2024-08-07 / 71 阅读
0

一种文件分片上传的实现方法

#Spring

在现代Web应用中，文件上传是一个常见的功能需求，特别是处理大文件时，传统的单一文件上传方式可能会因为网络问题或服务器性能限制而导致上传失败。

文件分片上传技术通过将大文件分割成多个小片段（分片）分别上传，并在服务器端重新组装，有效解决了这些问题。

本文将介绍一种基于Spring框架的文件分片上传实现方法，其中前端在上传前计算文件的MD5值，而后递归对文件进行切片，同时基于切片计算上传进度，展示到页面，后端则通过MultipartFile接收分片，并使用RandomAccessFile进行随机写入，文件分片全部接收完毕后，通过MD5校验确保文件完整性。

分片上传展示

slice_upload.gif

序列图

sequenceDiagram actor Client as <<Brower>> <br/> Client participant frontend as <<Js>> <br/> frontend participant backend as <<Java>> <br/> backend autonumber Client->>+frontend: submit frontend->>frontend: _getFileMd5(file, chunkSize) rect rgb(191, 223, 255) loop repeat to upload sliced files frontend->>+backend: _sliceUploadFile <br/> `POST /api/kyyee/v2/sps/files/slice-upload` backend->>backend: sliceUpload(FileReqDto reqDto) backend->>backend: if (reqDto.getChunk() + 1) == reqDto.getChunks() <br/> return uri backend-->>-frontend: res frontend->>frontend: if (uri) end end frontend-->>-Client: res

前端

前端代码使用html5和原生javascript编写。

file.html

<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>File Upload</title>
  <link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.0/css/bootstrap.min.css" rel="stylesheet">
  <style>
    /* 自定义进度条样式 */
    .percent input[type=range] {
      -webkit-appearance: none;
      /*清除系统默认样式*/
      width: 7.8rem;
      /* background: -webkit-linear-gradient(#ddd, #ddd) no-repeat, #ddd; */
      /*设置左边颜色为#61bd12，右边颜色为#ddd*/
      background-size: 75% 100%;
      /*设置左右宽度比例*/
      height: 0.6rem;
      /*横条的高度*/
      border-radius: 0.4rem;
      border: 1px solid #ddd;
      box-shadow: 0 0 10px rgba(0,0,0,.125) inset ;
    }

    /*拖动块的样式*/
    .percent input[type=range]::-webkit-slider-thumb {
      -webkit-appearance: none;
      /*清除系统默认样式*/
      height: .9rem;
      /*拖动块高度*/
      width: .9rem;
      /*拖动块宽度*/
      background: #fff;
      /*拖动块背景*/
      border-radius: 50%;
      /*外观设置为圆形*/
      border: solid 1px #ddd;
      /*设置边框*/
    }

    a {
      height: 300px;
    }

  </style>
</head>
<body>
<a href="./index.html">返回首页</a>
<h1>文件上传</h1>
<form id="upload-form" enctype="multipart/form-data">
  <input id="uploadFile" type="file" name="file"/>
  <input id="uploadSubmitBtn" type="button" value="提交"/>
  <div>
    <a id="uploadFileHref" href="" target="_self">文件地址</a>
  </div>
</form>
<h1>大文件分片上传</h1>
<form id="slice-upload-form" enctype="multipart/form-data">
  <input id="sliceUploadFile" type="file" name="file"/>
  <input id="sliceUploadSubmitBtn" type="button" value="提交"/>
  <div class="percent">
    <input id="percentRange" type="range" value="0"/><span id="percentSpan">0%</span>
  </div>
  <div>
    <a id="sliceUploadFileHref" href="" target="_self">文件地址</a>
  </div>
</form>
<footer class="text-center panel-footer" role="contentinfo">
  <div class="container">
    <p class="">Designed by <a href="https://github.com/kyyee" target="_blank">@kyyee</a> with <a
      href="https://github.com/twbs/bootstrap" target="_blank">Bootstrap</a></p>
  </div>
</footer>
</body>
<script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap/5.2.0/js/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/spark-md5/3.0.2/spark-md5.js"></script>
<script src="file.js" type="module"></script>
</html>

file.js

'use strict';

import { Snowflake } from './snowflake.js'

class FileSlice {
    // 每个chunk的大小，设置为10兆
    static chunkSize = 10 * 1024 * 1024;
    constructor(configs) {
        this.configs = {
        };
        if (configs) {
            Object.assign(this.configs, configs);
        }
    };

    init() {
        // 检测 DOMContentLoaded 是否已完成
        document.readyState !== 'loading' ? this._doStart(this.configs) : document.addEventListener('DOMContentLoaded', () => this._doStart(this.configs));
    }

    _doStart(configs) {
        // 获取slice方法，做兼容处理
        // 提交
        const sliceUploadSubmitBtn = document.getElementById('sliceUploadSubmitBtn');
        sliceUploadSubmitBtn.addEventListener('click', async() => {
            let index = 0
            // 1.读取文件
            let fileDom = document.getElementById("sliceUploadFile");
            let files = fileDom.files;
            let file = files[0];
            if (!file) {
                alert('请选择文件');
                return;
            }
            // 2.设置分片参数属性、获取文件MD5值
            let chunkSize = FileSlice.chunkSize;
            sliceUploadSubmitBtn.disabled = true;
            fileDom.disabled = true;
            const md5 = await this._getFileMd5(file, chunkSize).then(md5 => {
                console.log(md5);
                sliceUploadSubmitBtn.disabled = false;
                fileDom.disabled = false;
                let id = Snowflake.getId();
                this._sliceUploadFile(file, index, chunkSize, id, md5);
            })
            // todo 暂停，断点续传
        });

        const uploadSubmitBtn = document.getElementById('uploadSubmitBtn');
        uploadSubmitBtn.addEventListener('click', async() => {
            let index = 0
            // 1.读取文件
            let fileDom = document.getElementById("uploadFile");
            let files = fileDom.files;
            let file = files[0];
            if (!file) {
                alert('请选择文件');
                return;
            }
            // 2.设置分片参数属性、获取文件MD5值
            let chunkSize = FileSlice.chunkSize;
            uploadSubmitBtn.disabled = true;
            fileDom.disabled = true;
            const md5 = await this._getFileMd5(file, chunkSize).then(md5 => {
                console.log(md5);
                uploadSubmitBtn.disabled = false;
                fileDom.disabled = false;
                let id = Snowflake.getId();
                this._uploadFile(file, id, md5);
            })
        });
    }

    // 对文件进行MD5加密(文件内容+文件标题形式)
    _getFileMd5(file, chunkSize) {
        return new Promise((resolve, reject) => {
            let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;
            let index = 0;
            let chunks = Math.ceil(file.size / chunkSize);
            let spark = new SparkMD5.ArrayBuffer();
            let fileReader = new FileReader();

            fileReader.onload = (e) => {
                spark.append(e.target.result); // Append array buffer
                index++;
                if (index < chunks) {
                    let start = index * chunkSize;
                    let end = Math.min(file.size, start + chunkSize);
                    fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
                } else {
                    let result = spark.end();
                    resolve(result);
                }
            };
            fileReader.onerror = (e) => {
                reject(e);
                alert('文件读取失败！');
            };
            // 文件分片
            let start = index * chunkSize;
            let end = Math.min(file.size, start + chunkSize);
            fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
        })
        .catch(err => { console.error(err); });
    }

    // 文件上传
    _sliceUploadFile(file, index, chunkSize, id, md5) {
        if (file) {
            let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;

            let chunks = Math.ceil(file.size / chunkSize); // 分片总数
            if (index > chunks - 1) {
                return;
            }
            let start = index * chunkSize;
            const end = Math.min(file.size, start + chunkSize);
            // 构建表单
            let form = new FormData();
            // blobSlice.call(file, start, end)方法是用于进行文件分片
            form.append('file', blobSlice.call(file, start, end));
            form.append('md5', md5);
            form.append('chunk', index);
            form.append('chunks', chunks);
            form.append('filename', file.name);
            form.append('size', chunkSize);
            form.append('id', id);
            // ajax提交 分片，此时 content-type 为 multipart/form-data
            fetch('/api/kyyee/v2/sps/files/slice-upload', {
                method: 'post',
//              // 不能设置 headers :{ 'Content-Type': 'multipart/form-data', }
                body: form,
                })
            .then(response => response.json())
            .catch(error => { console.error(error) })
            .then(async success => {
                console.debug(success);
                let isSuccess = success.code === '0000000000' && success.data;
                if (isSuccess) {
                    // 判断分片是否上传完成
                    let uri = success.data.uri;
                    if (uri) {
                        // 4.所有分片上传后，请求合并分片文件
                        this._setPercent(chunks, chunks); // 全部上传完成
                        let fileInfo = {
                            name: file.name,
                            size: file.size,
                            type: file.type,
                            uid: file.uid,
                            status: 'done',
                            uri: uri,
                            server_ip: success.data.server_ip,
                        };
                        const sliceUploadFileHref = document.getElementById('sliceUploadFileHref');
                        sliceUploadFileHref.href = window.location.protocol + '//' + window.location.host + uri;
                        sliceUploadFileHref.textContent = fileInfo.name;
                        console.log(fileInfo);
                    } else {
                        this._setPercent(index, chunks);
                        await this._sliceUploadFile(file, ++index, chunkSize, id, md5);
                    }
                } else {
                    alert("上传失败");
                    return;
                }
            });
        }
    }

    // 文件上传
    _uploadFile(file, id, md5) {
        if (file) {
            let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice;

            // 构建表单
            let form = new FormData();
            // blobSlice.call(file, start, end)方法是用于进行文件分片
            form.append('file', file);
            form.append('md5', md5);
            form.append('filename', file.name);
            form.append('size', file.size);
            form.append('id', id);
            // ajax提交 分片，此时 content-type 为 multipart/form-data
            fetch('/api/kyyee/v2/sps/files/upload', {
                method: 'post',
//              // 不能设置 headers :{ 'Content-Type': 'multipart/form-data', }
                body: form,
                })
            .then(response => response.json())
            .catch(error => { console.error(error) })
            .then(async success => {
                console.debug(success);
                let isSuccess = success.code === '0000000000' && success.data;
                if (isSuccess) {
                    // 判断分片是否上传完成
                    let uri = success.data.uri;
                    if (uri) {
                        let fileInfo = {
                            name: file.name,
                            size: file.size,
                            type: file.type,
                            uid: file.uid,
                            status: 'done',
                            uri: uri,
                            server_ip: success.data.server_ip,
                        };
                        const uploadFileHref = document.getElementById('uploadFileHref');
                        uploadFileHref.href = window.location.protocol + '//' + window.location.host + uri;
                        uploadFileHref.textContent = fileInfo.name;
                        console.log(fileInfo);
                    }
                } else {
                    alert("上传失败");
                    return;
                }
            });
        }
    }

    // 设置进度条
    _setPercent(index, chunks) {
        let percentValue = Math.ceil(index / chunks * 100);
        // 进度条
        const percentDom = document.getElementById('percentRange');
        percentDom.value = percentValue;
        percentDom.style.background = `-webkit-linear-gradient(top, #059CFA, #059CFA) 0% 0% / ${percentValue}% 100% no-repeat`;
        // 进度条值对应dom
        const percentSpan = document.getElementById('percentSpan');
        percentSpan.textContent = percentValue + '%';
     }
}

new FileSlice({}).init();

snowflake.js

'use strict';

class Snowflake {
    static dataCenterId = 0n;
    static workerId = 0n;
    static twepoch = 1661906860000n;
    static dataCenterIdBits = 5n;
    static workerIdBits = 5n;
    static maxDataCenterId = -1n ^ ( - 1n << Snowflake.dataCenterIdBits); // 值为：31
    static maxWorkerId = -1n ^ ( - 1n << Snowflake.workerIdBits); // 值为：31
    static sequenceBits = 12n;
    static workerIdShift = Snowflake.sequenceBits; // 值为：12
    static dataCenterIdShift = Snowflake.sequenceBits + Snowflake.workerIdBits; // 值为：17
    static timestampLeftShift = Snowflake.sequenceBits + Snowflake.workerIdBits + Snowflake.dataCenterIdBits; // 值为：22
    static sequenceMask = -1n ^ ( - 1n << Snowflake.sequenceBits); // 值为：4095

    static singleton;

    constructor(workerId, dataCenterId) {
        //设置默认值,从环境变量取
        this.dataCenterId = 1n;
        this.workerId = 1n;
        this.sequence = 0n;
        this.lastTimestamp = -1n;

        if (workerId > Snowflake.maxWorkerId || workerId < 0) {
            throw new Error('_worker Id can\'t be greater than maxWorkerId-[' + Snowflake.maxWorkerId + '] or less than 0');
        }
        if (dataCenterId > Snowflake.maxDataCenterId || dataCenterId < 0) {
            throw new Error('dataCenter Id  can\'t be greater than maxDataCenterId-[' + Snowflake.maxDataCenterId + '] or less than 0');
        }

        console.debug('worker starting. timestamp left shift ' + Snowflake.timestampLeftShift + ', datacenter id bits ' + Snowflake.dataCenterIdBits + ', worker id bits '
        + Snowflake.workerIdBits + ', sequence bits ' + Snowflake.sequenceBits + ', worker id ' + Snowflake.workerId);
        this.workerId = BigInt(workerId);
        this.dataCenterId = BigInt(dataCenterId);
    }

    static getId() {
        if (!Snowflake.singleton) {
            Snowflake.singleton = new Snowflake(Snowflake.dataCenterId, Snowflake.workerId);
        }
        return Snowflake.singleton._nextId();
    }

    _nextId = function() {
        let timestamp = this._timeGen();
        if (timestamp < this.lastTimestamp) {
            throw new Error('Clock moved backwards.  Refusing to generate id for ' + (this.lastTimestamp - timestamp) + 'milliseconds');
        }

        if (this.lastTimestamp === timestamp) {
            this.sequence = (this.sequence + 1n) & Snowflake.sequenceMask;
            if (this.sequence === 0n) {
                timestamp = this._tilNextMillis(this.lastTimestamp);
            }
        } else {
            this.sequence = 0n;
        }
        this.lastTimestamp = timestamp;
        //移位并通过或运算拼到一起组成64位的ID
        return ((timestamp - Snowflake.twepoch) << Snowflake.timestampLeftShift)
         | (this.dataCenterId << Snowflake.dataCenterIdShift)
         | (this.workerId << Snowflake.workerIdShift)
         | this.sequence;
    };

    _tilNextMillis(lastTimestamp) {
        let timestamp = this._timeGen();
        while (timestamp <= lastTimestamp) {
            timestamp = this._timeGen();
        }
        return BigInt(timestamp);
    };

    _timeGen() {
        return BigInt(Date.now());
    };
}

export { Snowflake };

这段代码中使用了spark-md5.js，需要引入该模块。

后端

后端代码采用Java代码编写，jdk版本要求21。

基于Spring框架，通过MultipartFile接收分片文件

FileController.java

package com.kyyee.sps.controller;

import com.kyyee.sps.common.component.validated.group.File;
import com.kyyee.sps.dto.request.FileReqDto;
import com.kyyee.sps.dto.response.FileResDto;
import com.kyyee.sps.service.FileService;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;

@RestController
@RequestMapping("${api-prefix}/files")
//@ApiVersion(1)
@Slf4j
@Tag(name = "文件管理")
@Validated
public class FileController {

    @Resource
    private FileService service;

    @PostMapping("/upload")
    public FileResDto upload(@Validated({File.Upload.class}) FileReqDto reqDto) {
        return service.upload(reqDto);
    }

    @PostMapping("/slice-upload")
    public FileResDto sliceUpload(@Validated({File.SliceUpload.class}) FileReqDto resDto) {
        return service.sliceUpload(resDto);
    }

    @DeleteMapping("{filename}")
    public void delete(@PathVariable(name = "filename", required = false) String filename) {
        service.delete(filename);
    }
}

FileServiceImpl.java

package com.kyyee.sps.service.impl;

import com.kyyee.framework.common.exception.BaseErrorCode;
import com.kyyee.framework.common.exception.BaseException;
import com.kyyee.sps.dto.request.FileReqDto;
import com.kyyee.sps.dto.response.FileResDto;
import com.kyyee.sps.service.FileService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Locale;

@Service
@Slf4j
public class FileServiceImpl implements FileService {
    // todo 区分操作系统语言
    Locale locale = Locale.getDefault();
    /**
     * No space left on device
     */
    public static final String NO_SPACE_LEFT_EN = "[\\s\\S]*No space left on device[\\s\\S]*";

    /**
     * 磁盘空间不足
     */
    public static final String NO_SPACE_LEFT_CN = "[\\s\\S]*磁盘空间不足[\\s\\S]*";

    @Value("${api-prefix:NA}")
    private String apiPrefix;

    @Value("${kyyee.file.save-dir:/home/file/}")
    private String saveDir;

    @Override
    public FileResDto upload(FileReqDto reqDto) {
        // 此处可以对数据库中的file md5进行比对，如存在则直接返回file uri
        MultipartFile multipartFile = reqDto.getFile();
        // 创建文件保存目录和文件压缩目录
        File file = copyMultipartFileToFile(multipartFile);
        verifyFileMd5(file, reqDto.getMd5());

        return FileResDto.builder().filename(multipartFile.getOriginalFilename())
            .uri(apiPrefix + "/fs/" + multipartFile.getOriginalFilename())
            .build();
    }

    /**
     * 创建文件目录
     *
     * @param dirPath 目录
     */
    public static void mkdir(String dirPath) {
        if (dirPath == null) {
            return;
        }

        File directory = new File(dirPath);
        if (directory.exists()) {
            return;
        }
        try {
            FileUtils.forceMkdir(directory);
        } catch (IOException e) {
            // ignored exception
            log.error("mkdir failed, message:{}:", e.getMessage());
            if (directory.getUsableSpace() <= 0) {
                throw BaseException.of(BaseErrorCode.FILE_CREATE_ERROR, "剩余空间不足，导出失败，请清理磁盘空间后重试");
            }
            if (e.getMessage().matches(NO_SPACE_LEFT_EN)) {
                throw BaseException.of(BaseErrorCode.FILE_CREATE_ERROR, "剩余空间不足，导出失败，请清理磁盘空间后重试");
            }
        }
    }

    /**
     * 将输入流转换为文件
     *
     * @param multipartFile
     * @return
     */
    public File copyMultipartFileToFile(MultipartFile multipartFile) {
        mkdir(saveDir);

        final String fileName = multipartFile.getOriginalFilename();
        if (StringUtils.isEmpty(fileName)) {
            throw BaseException.of(BaseErrorCode.FILE_READ_ERROR.of(), "文件转换异常");
        }
        File file = new File(saveDir.concat(multipartFile.getOriginalFilename()));
        try {
            multipartFile.transferTo(file);
        } catch (IOException e) {
            log.error("文件存储到本地失败，{}", e.getMessage());
            throw BaseException.of(BaseErrorCode.FILE_UPLOAD_ERROR.of(), "文件转换异常");
        }
        return file;
    }

    @Override
    public FileResDto sliceUpload(FileReqDto reqDto) {
        // 此处可以对数据库中的file md5进行比对，如存在则直接返回file uri
        mkdir(saveDir);

        File file = new File(saveDir, reqDto.getFilename());

        final String filename = file.getName();
        String baseName = FilenameUtils.getBaseName(filename);
        String fileType = FilenameUtils.getExtension(filename);

        MappedByteBuffer buffer = null;
        try (final RandomAccessFile tmpRaf = new RandomAccessFile(file, "rw");
             final FileChannel channel = tmpRaf.getChannel()) {
            final long offset = (reqDto.getSize()) * reqDto.getChunk();

            final byte[] bytes = reqDto.getFile().getBytes();
            buffer = channel.map(FileChannel.MapMode.READ_WRITE, offset, bytes.length);
            buffer.put(bytes);
        } catch (IOException e) {
            log.warn("upload slice file failed, message:{}", e.getMessage(), e);
            throw BaseException.of(BaseErrorCode.FILE_UPLOAD_ERROR.of(), "文件转换异常");
        } finally {
            unmap(buffer);
        }

        // chunk 从0开始
        if ((reqDto.getChunk() + 1) == reqDto.getChunks()) {
            verifyFileMd5(file, reqDto.getMd5());

            return FileResDto.builder().filename(filename)
                .uri(apiPrefix + "/fs/" + filename)
                .build();

        }

        return FileResDto.builder().filename(filename)
            .build();
    }

    private void unmap(MappedByteBuffer buffer) {
        if (buffer != null) {
            buffer.force();
        }
    }

    private void verifyFileMd5(File destFile, String md5) {
        //判断文件是否被更改
        try (FileInputStream fileInputStream = new FileInputStream(destFile)) {
            String realMd5 = org.springframework.util.DigestUtils.md5DigestAsHex(fileInputStream);
            if (!md5.equals(realMd5)) {
                throw BaseException.of(BaseErrorCode.FILE_UPLOAD_ERROR.of(), "文件md5不一致");
            }
        } catch (IOException e) {
            throw BaseException.of(BaseErrorCode.FILE_UPLOAD_ERROR.of(), "校验文件md5异常");
        }
    }

    /**
     * v1暂无此功能
     */
    public void delete(String filepath) {
        // 1.删除zip文件
        try {
            File file = new File(saveDir);
            if (!file.exists()) {
                return;
            }
        
            File directory = new File(saveDir + filepath);
            if (directory.exists()) {
                log.info("delete file:{}", directory.getName());
                if (directory.isDirectory()) {
                    FileUtils.deleteDirectory(directory);
                }
                if (directory.isFile()) {
                    FileUtils.deleteQuietly(directory);
                }
            }
        } catch (IOException e) {
            log.warn("delete filename:{} failed. message:{}", filepath, e.getMessage(), e);
        }
    }

}

这些代码实现了文件的分片上传，文件的md5计算和比对来确保文件的可用性，基于文件分片实现了上传进度。

后端通过RandomAccessFile随机读写文件提升了IO效率，减少了文件的读写频率。

基于上述的代码片段还能扩展出文件断点续传/文件妙传等高级功能，这些功能需要将文件进度持久化到数据库中。

部分类和代码有省略，更多内容可参考GitHub项目：springboot-project-seed。

一种超大规模数据导出Excel的实现方法
通过noVNC和websockif...