Web大文件分段上传

前端

test.jsx

import { useState } from 'react';
import { Button, Space, Upload } from 'antd';
import { InboxOutlined } from '@ant-design/icons';
import { uploadBigFile } from '@/services/system';

const { Dragger } = Upload;

export default () => {
  const [fileList, setFileList] = useState([]);

  const props = {
    fileList,
    name: 'file',
    multiple: true,
    beforeUpload: () => false,
    onChange(info) {
      setFileList(info.fileList);
    },
    onDrop(info) {
      setFileList(info.fileList);
    }
  };

  return (
    <Space>
      <Dragger {...props}>
        <p className="ant-upload-drag-icon">
          <InboxOutlined />
        </p>
        <p className="ant-upload-text">Click or drag file to this area to upload</p>
        <p className="ant-upload-hint">Support for a single or bulk upload. Strictly prohibit from uploading company data or other band files</p>
      </Dragger>

      <Button
        type="primary"
        style={{ marginTop: 8 }}
        onClick={async () => {
          const promises = fileList.map((item) =>
            uploadBigFile(item, (props) => {
              // 修改对应文件的进度
              setFileList((fileList) => {
                return fileList.map((file) => {
                  if (file.uid === item.uid) {
                    return { ...file, ...props };
                  }
                  return file;
                });
              });
            })
          );

          await Promise.all(promises);
        }}
      >
        Submit
      </Button>
    </Space>
  );
};

tool

// 大文件上传
export async function uploadBigFile(file, callback = undefined) {
  const { originFileObj, size: fileSize, name: fileName, uid: fileId } = file;
  const chunkSize = 1024 * 1024 * 10; // 每次上传5M
  const totalChunks = Math.ceil(fileSize / chunkSize);

  for (let i = 0; i < totalChunks; i++) {
    // 计算当前块的起始位置和结束位置并切片
    const start = i * chunkSize;
    const end = start + chunkSize;
    const chunk = originFileObj.slice(start, end);

    console.log('上传块', i);

    // 上传块
    try {
      await instance.post(`/v4/system/upload/${i}`, chunk, {
        params: { fileId, fileName, totalChunks },
        headers: { 'Content-Type': 'application/octet-stream' }
      });

      // 计算当前进度
      const progress = parseInt(((i + 1) / totalChunks) * 100);
      callback &&
        callback({
          status: progress !== 100 ? 'uploading' : 'done',
          percent: progress
        });
    } catch (error) {
      console.log(error);
      message.error('上传失败');
      return;
    }
  }
}

后端

存储在本地

@api.route('/upload/<string:chunk_number>', methods=['POST'])
def upload(chunk_number):
    """处理大文件上传"""
    TEMP_DIR = 'temp'

    chunk_number = int(chunk_number)
    first = chunk_number == 0
    file_id = request.args.get('fileId')
    file_name = request.args.get('fileName')
    total_chunks = int(request.args.get('totalChunks'))
    chunk = request.data

    # 保存文件快
    if not os.path.exists(os.path.join(TEMP_DIR, file_id)):
        os.makedirs(os.path.join(TEMP_DIR, file_id))

    with open(os.path.join(TEMP_DIR, file_id, str(chunk_number)), 'wb') as f:
        f.write(chunk)

    # 如果是最后一个文件快，合并文件
    if chunk_number == total_chunks - 1:
        file_path = os.path.join(TEMP_DIR, file_id, file_name)
        with open(file_path, 'wb') as f:
            for i in range(0, total_chunks):
                chunk_path = os.path.join(TEMP_DIR, file_id, str(i))
                with open(chunk_path, 'rb') as chunk_file:
                    f.write(chunk_file.read())

    return Success()

存储在华为云OBS

@api.route('/upload/<string:chunk_number>', methods=['POST'])
def upload(chunk_number):
    """处理大文件上传"""
    chunk_number = int(chunk_number) + 1
    first = chunk_number == 1
    file_id = request.args.get('fileId')
    file_name = request.args.get('fileName')
    obs_upload_id = request.args.get('uploadId')
    total_chunks = int(request.args.get('totalChunks'))
    chunk = request.data

    try:
        # 如果是第一块，创建分段上传任务
        if first:
            resp = obs.initiateMultipartUpload(
                bucketName=current_app.config['HUAWEI_OBS_BUCKET'],
                objectKey=file_name
            )
            current_app.logger.info('InitiateMultipartUpload resp', resp)
            obs_upload_id = resp.body.uploadId

        # 保存文件快，上传块
        resp = obs.uploadPart(
            bucketName=current_app.config['HUAWEI_OBS_BUCKET'],
            objectKey=file_name,
            partNumber=chunk_number,
            uploadId=obs_upload_id,
            object=io.BytesIO(chunk)
        )
        current_app.logger.info('UploadPart resp', resp)

        # 合并块
        if chunk_number == total_chunks:
            resp = obs.listParts(
                bucketName=current_app.config['HUAWEI_OBS_BUCKET'],
                objectKey=file_name,
                uploadId=obs_upload_id
            )
            current_app.logger.info('ListParts resp', resp)

            etags = []
            for part in resp.body.parts:
                etags.append(CompletePart(partNum=part.partNumber, etag=part.etag))

            # 完成分段上传
            resp = obs.completeMultipartUpload(
                bucketName=current_app.config['HUAWEI_OBS_BUCKET'],
                objectKey=file_name,
                uploadId=obs_upload_id,
                completeMultipartUploadRequest=CompleteMultipartUploadRequest(parts=etags)
            )
            current_app.logger.info('CompleteMultipartUpload resp', resp)

        if first:
            return jsonify({'uploadId': obs_upload_id})
        else:
            return Success()

    except Exception as e:
        current_app.logger.error(e)
        raise ServerError(msg='华为云OBS错误，文件上传失败')

// 大文件上传
export async function uploadBigFile2(file, callback = undefined) {
  const { originFileObj, size: fileSize, name: fileName, uid: fileId } = file;
  const chunkSize = 1024 * 1024 * 10; // 每次上传5M
  const totalChunks = Math.ceil(fileSize / chunkSize);
  let uploadId = '';

  for (let i = 0; i < totalChunks; i++) {
    // 计算当前块的起始位置和结束位置并切片
    const start = i * chunkSize;
    const end = start + chunkSize;
    const chunk = originFileObj.slice(start, end);

    console.log('上传块', i);

    // 上传块
    try {
      const resp = await instance.post(`/v4/system/upload/${i}`, chunk, {
        params: { fileId, fileName, totalChunks, uploadId },
        headers: { 'Content-Type': 'application/octet-stream' }
      });

      // 初始化obs上传任务
      if (resp?.uploadId) {
        uploadId = resp.uploadId;
      }

      // 计算当前进度
      const progress = parseInt(((i + 1) / totalChunks) * 100);
      callback &&
        callback({
          status: progress !== 100 ? 'uploading' : 'done',
          percent: progress
        });
    } catch (error) {
      console.log(error);
      callback({ status: 'error', percent: 0 });

      return;
    }
  }
}