大文件上传（含node代码）

为什么会使用到大文件上传？

在项目中有大文件上传的需求，在同一个请求，要上传大量数据，导致接口请求的时间很漫长，或许会造成接口超时的后果，且上传过程中如果出现网络异常，那么重新上传还是从零开始上传，大文件上传可以完美解决了以上的弊端，且支持暂停和继续的功能。

怎样实现大文件上传？

1.文件切片

我们可以将选中的文件通过读取文件将文件读取成ArrayBuffer或者DataURL,通过file中的slice方法根据我们规定上传的份数进行切割

2.前端生成文件名

这一步需要在前端生成文件名之后发送给服务器端，这样服务器端就会根据文件名的不分内容生成一个文件夹，每次前端请求接口上传切片的时候服务端都会校验一下文件名，找到对应的文件夹再进行插入

3.浏览器问题

由于我们将文件拆分成了n个，那就意味着要进行n次请求，如果进行一次性请求的话，谷歌浏览器最多一次性处理六个请求，当文件过大时会造成浏览器的卡顿，那么我们需要使用发布订阅的模式来控制并发请求问题

4.断点续传

当我们上传的过程中出现了网络问题造成强制中断，那么我们要实现当下次上传的时候需要校验一下上传到了哪一步，然后继续上传。这里前端可以将已经上传的标志存储到lcalstorage中,但是换一个浏览器的话会获取不到该内容。所以我们将这段逻辑在服务端完成，前端需要向服务端发送一个请求获取当前已经上传的内容，获取之后需要判断一下是否含有这个文件，如果存在的话我们不需要再进行该切片的上传，这样就实现了断点续传

5.上传进度和暂停

前端设置一个暂停的按钮，由于上面我们使用的发布订阅的模式，将每一个切片生成一个函数，再将每个函数放入到队列中，我们使用一个变量来记录上传了多少个，当用户点击暂停时，直接终止，当点击继续时，我们根据上面设置的变量就可以知道在事件池中拿到对应的方法，再通知依次执行

6.合并

当所有切片都上传成功之后，前端需要调用服务端一个合并的接口，并将文件的名字和切片的数量发送给服务端，服务端进行切片合并并将视频发送给前端。

前端源码

<template>
  <div id="app">
    <el-upload
      drag
      action
      :auto-upload="false"
      :show-file-list="false"
      :on-change="changeFile"
    >
      <i class="el-icon-upload"></i>
      <div class="el-upload__text">
        将文件拖到此处，或
        <em>点击上传</em>
      </div>
    </el-upload>

    <!-- PROGRESS -->
    <div class="progress">
      <span>上传进度：{{ totalNum }}%</span>
      <el-link
        v-if="total > 0 && total < 30"
        type="primary"
        @click="handleBtn"
        >{{ btn | btnText }}</el-link
      >
    </div>

    <!-- VIDEO -->
    <div class="uploadImg" v-if="video">
      <video :src="video" controls />
    </div>
  </div>
</template>

<script>
import SparkMD5 from "spark-md5";
export default {
  name: "App",
  data() {
    return {
      total: 0,
      video: null,
      btn: false,
      HASH: "",
      chunks: [],
      already: [],
      requestList: [],
      count: 0,
      alreadyUploadIndex: 0,
    };
  },
  filters: {
    btnText(btn) {
      return btn ? "继续" : "暂停";
    },
  },
  computed: {
    totalNum() {
      let isNum = (num) => num != null && !isNaN(num);
      let n =
        isNum(this.count) && isNum(this.total)
          ? ((this.total / this.count) * 100).toFixed(0)
          : 0;
      return isNaN(n) ? 0 : n;
    },
  },
  methods: {
    async changeFile(file) {
      if (!file) return;
      file = file.raw;
    //   读取文件成buffer类型
      let buffer = await this.fileParse(file, "buffer");
      let suffix = this.createName(buffer, file);
    //   校验已经上传的文件切片
      let data = await this.axios.get("/upload_already", {
        params: {
          HASH: this.HASH,
        },
      });
      this.already = data.fileList;
      this.total = data.fileList.length;
    //   拆分切片
      this.splitBuffer(file, suffix);
    //   将切片存储到事件池中
      this.saveEventPool()
    //   向服务端发送切片 并且校验是否已经传输完毕 如果传输完毕则调用服务端的合并接口
      this.sendRequest();
    },
    //  将文件读取成buffer类型
    fileParse(file, type) {
      return new Promise((resolve) => {
        let fileRead = new FileReader();
        fileRead.readAsArrayBuffer(file);
        fileRead.onload = (ev) => {
          resolve(ev.target.result);
        };
      });
    },
    // 前端生成文件名
    createName(buffer, file) {
      let spark = new SparkMD5.ArrayBuffer();
      spark.append(buffer);
      this.HASH = spark.end();
      let suffix = /\.([0-9a-zA-Z]+)$/i.exec(file.name)[1];
      return suffix;
    },
    // 根据buffer的大小进行合理的拆分
    splitBuffer(file, suffix) {
      let max = 1024 * 100,
        index = 0;
      this.count = Math.ceil(file.size / max);
      if (this.count > 30) {
        max = file.size / 30;
        this.count = 30;
      }
      while (index < this.count) {
        this.chunks.push({
          file: file.slice(index * max, (index + 1) * max),
          filename: `${this.HASH}_${index + 1}.${suffix}`,
        });
        index++;
      }
    },
    //使用发布订阅模式先将切片方法存储到事件池中
    saveEventPool() {
      this.chunks.forEach((chunk, index) => {
        if (this.already.length > 0 && this.already.includes(chunk.filename)) {
          return;
        }
        let fn = () => {
          let fm = new FormData();
          fm.append("file", chunk.file);
          fm.append("filename", chunk.filename);
          return this.axios.post("/upload_chunk", fm).then((data) => {
            this.total++;
          });
        };
        this.requestList.push(fn);
      });
    },
   
    async complete() {
      if (this.total < 30) return;
      let res = await this.axios.post(
        "/upload_merge",
        {
          HASH: this.HASH,
          count: this.count,
        },
        {
          headers: {
            "Content-Type": "application/x-www-form-urlencoded",
          },
        }
      );
      this.video = res.servicePath;
      alert("成功");
    },

    async sendRequest() {
      let send = async () => {
        // 已经中断则不再上传
        if (this.abort) return;
        if (this.alreadyUploadIndex >= this.requestList.length) {
          // 都传完了
          this.complete();
          return;
        }
        await this.requestList[this.alreadyUploadIndex]();
        this.alreadyUploadIndex++;
        send();
      };
      send();
    },
    // 暂停/继续
    handleBtn() {
      if (this.btn) {
        //断点续传
        this.abort = false;
        this.btn = false;
        this.sendRequest();
        return;
      }
      //暂停上传
      this.btn = true;
      this.abort = true;
    },
  },
};
</script>

服务端

const express = require('express'),
    fs = require('fs'),
    bodyParser = require('body-parser'),
    multiparty = require('multiparty'),
    SparkMD5 = require('spark-md5');

/*-CREATE SERVER-*/
const app = express(),
    PORT = 8888,
    HOST = 'http://127.0.0.1',
    HOSTNAME = `${HOST}:${PORT}`;
app.listen(PORT, () => {
    console.log(`THE WEB SERVICE IS CREATED SUCCESSFULLY AND IS LISTENING TO THE PORT：${PORT}，YOU CAN VISIT：${HOSTNAME}`);
});

/*-中间件-*/
app.use((req, res, next) => {
    res.header("Access-Control-Allow-Origin", "*");
    req.method === 'OPTIONS' ? res.send('CURRENT SERVICES SUPPORT CROSS DOMAIN REQUESTS!') : next();
});
app.use(bodyParser.urlencoded({
    extended: false,
    limit: '1024mb'
}));

/*-API-*/

// 检测文件是否存在
const exists = function exists(path) {
    return new Promise(resolve => {
        fs.access(path, fs.constants.F_OK, err => {
            if (err) {
                resolve(false);
                return;
            }
            resolve(true);
        });
    });
};

// 创建文件并写入到指定的目录 & 返回客户端结果
const writeFile = function writeFile(res, path, file, filename, stream) {
    return new Promise((resolve, reject) => {
        if (stream) {
            try {
                let readStream = fs.createReadStream(file.path),
                    writeStream = fs.createWriteStream(path);
                readStream.pipe(writeStream);
                readStream.on('end', () => {
                    resolve();
                    fs.unlinkSync(file.path);
                    res.send({
                        code: 0,
                        codeText: 'upload success',
                        originalFilename: filename,
                        servicePath: path.replace(__dirname, HOSTNAME)
                    });
                });
            } catch (err) {
                reject(err);
                res.send({
                    code: 1,
                    codeText: err
                });
            }
            return;
        }
        fs.writeFile(path, file, err => {
            if (err) {
                reject(err);
                res.send({
                    code: 1,
                    codeText: err
                });
                return;
            }
            resolve();
            res.send({
                code: 0,
                codeText: 'upload success',
                originalFilename: filename,
                servicePath: path.replace(__dirname, HOSTNAME)
            });
        });
    });
};

// 大文件切片上传 & 合并切片
const merge = function merge(HASH, count) {
    return new Promise(async (resolve, reject) => {
        let path = `${uploadDir}/${HASH}`,
            fileList = [],
            suffix,
            isExists;
        isExists = await exists(path);
        if (!isExists) {
            reject('HASH path is not found!');
            return;
        }
        fileList = fs.readdirSync(path);
        if (fileList.length < count) {
            reject('the slice has not been uploaded!');
            return;
        }
        fileList.sort((a, b) => {
            let reg = /_(\d+)/;
            return reg.exec(a)[1] - reg.exec(b)[1];
        }).forEach(item => {
            !suffix ? suffix = /\.([0-9a-zA-Z]+)$/.exec(item)[1] : null;
            fs.appendFileSync(`${uploadDir}/${HASH}.${suffix}`, fs.readFileSync(`${path}/${item}`));
            fs.unlinkSync(`${path}/${item}`);
        });
        fs.rmdirSync(path);
        resolve({
            path: `${uploadDir}/${HASH}.${suffix}`,
            filename: `${HASH}.${suffix}`
        });
    });
};
app.post('/upload_chunk', async (req, res) => {
    try {
        let {
            fields,
            files
        } = await multiparty_upload(req);
        let file = (files.file && files.file[0]) || {},
            filename = (fields.filename && fields.filename[0]) || "",
            path = '',
            isExists = false;
        // 创建存放切片的临时目录
        let [, HASH] = /^([^_]+)_(\d+)/.exec(filename);
        path = `${uploadDir}/${HASH}`;
        !fs.existsSync(path) ? fs.mkdirSync(path) : null;
        // 把切片存储到临时目录中
        path = `${uploadDir}/${HASH}/${filename}`;
        isExists = await exists(path);
        if (isExists) {
            res.send({
                code: 0,
                codeText: 'file is exists',
                originalFilename: filename,
                servicePath: path.replace(__dirname, HOSTNAME)
            });
            return;
        }
        writeFile(res, path, file, filename, true);
    } catch (err) {
        res.send({
            code: 1,
            codeText: err
        });
    }
});
app.post('/upload_merge', async (req, res) => {
    let {
        HASH,
        count
    } = req.body;
    try {
        let {
            filename,
            path
        } = await merge(HASH, count);
        res.send({
            code: 0,
            codeText: 'merge success',
            originalFilename: filename,
            servicePath: path.replace(__dirname, HOSTNAME)
        });
    } catch (err) {
        res.send({
            code: 1,
            codeText: err
        });
    }
});
app.get('/upload_already', async (req, res) => {
    let {
        HASH
    } = req.query;
    let path = `${uploadDir}/${HASH}`,
        fileList = [];
    try {
        fileList = fs.readdirSync(path);
        fileList = fileList.sort((a, b) => {
            let reg = /_(\d+)/;
            return reg.exec(a)[1] - reg.exec(b)[1];
        });
        res.send({
            code: 0,
            codeText: '',
            fileList: fileList
        });
    } catch (err) {
        res.send({
            code: 0,
            codeText: '',
            fileList: fileList
        });
    }
});

app.use(express.static('./'));
app.use((req, res) => {
    res.status(404);
    res.send('NOT FOUND!');
});

完整版

node与前端代码github地址：https://github.com/mengyuhang...