日常如果是上传一些小文件，在程序实现中，我们都是直接上传，一般都没什么问题。如果针对大文件上传的业务中，就会面临着：
1、网速问题，导致文件上传超时，而导致失败。
2、效率问题，上传大文件等待时间过长，如果是需要上传多个，就会更慢。
3、体验问题，用户无法预知上传还需花费的时间，系统没有及时反馈，用户无法判断文件是否还在上传，还是断开。
这时候就需要采用分布式文件上传系统。
项目简介
这是一个基于.Net Core构建的简单、跨平台分布式文件上传系统，支持分块上传、多个项目同时上传、接口权限控制采用JWT机制。
技术架构
1、跨平台：这是基于.Net Core开发的系统，可以部署在Docker, Windows, Linux, Mac。
2、.Net 2.1 + Jwt + simple-uploader
项目结构
项目分为分块上传与一般上传Demo，Web、控制台上传Demo。ufs项目是分布式文件上传的统一接口，ufs会根据配置把上传的文件发到ufs.node节点，ufs.node会把上传成功路径返回给ufs并存储，用户访问的时候，ufs会访问对应节点返回资源。
UploadServer为一般文件上传接口，UploadServer.FrontEndDemo为Web上传文件Demo。
**使用
**
1、配置
配置允许上传域名、服务接口地址、允许的文件格式、文件大小、存储路径等信息。
{ "AllowedHosts": "*", "urls": "http://localhost:6001", "uploadServer": { "rootUrl": "http://localhost:6001", "entryPoint1": "/upload", "entryPoint2": "/chunkUpload", "virtualPath": "", "physicalPath": "/Users/loogn/Desktop/uploader", "appendMimes": ".htm3:text/html;", "responseCache": 604800, "jwtSecret": "1234561234", "limitSize": "20mb", "allowExts": ".txt;.jpg;.jpeg;.png;.doc;.docx;.xls;.xlsx;.ppt;.pptx;.pdf", "apps": { "default": { "allowOrigins": "", "enableThumbnail": true, "limitExts": ".exe;", "thumbnailExts": ".jpg;.jpeg;.png;" }, "app1": { "allowOrigins": "*" } } }}
2、前端
一般上传代码
$("#file1").change(function () {
$.ajaxFileUpload({
fileElementId: 'file1',
url: 'http://localhost:6001/upload',
dataType: 'text',
//
success: function (data) {
console.log("上传成功：", data);
},
data: {
"jwt": jwt
}
});
});分块上传
var uploader = new Uploader({
target: 'http://localhost:6001/chunkupload',
headers: {jwt: jwt}
});
uploader.assignBrowse(document.getElementById('browseButton'));
//uploader.assignBrowse(document.getElementById('folderButton'), true);
//
// 文件添加 单个文件
uploader.on('fileAdded', function (file, event) {
console.log("fileAdded:", file, event)
});
// 单个文件上传成功
uploader.on('fileSuccess', function (rootFile, file, message) {
console.log("fileSuccess：", rootFile, file, message)
});
// 根下的单个文件（文件夹）上传完成
uploader.on('fileComplete', function (rootFile) {
console.log("fileComplete:", rootFile)
});
// 某个文件上传失败了
uploader.on('fileError', function (rootFile, file, message) {
console.log("fileError:", rootFile, file, message)
});3、后端
一般上传
public async Task InvokeAsync(HttpContext context, RequestDelegate next)
{
context.Response.Headers.Add("Access-Control-Allow-Origin", "*");
context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin");
if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase))
{
context.Response.StatusCode = (int) HttpStatusCode.OK;
}
else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase))
{
//验证jwt
string token = null;
if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt))
{
token = jwt.ToString();
}
else if (context.Request.Form.TryGetValue("jwt", out jwt))
{
token = jwt.ToString();
}
else
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "No JWT in the header and form"
}.toJson());
return;
}
try
{
var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature()
.Decode<JwtPayload>(token);
var msg = payload.validate();
if (msg != null)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = msg
}.toJson());
return;
}
//特定的配置
var appConfig = _config.GetAppConfig(payload.app);
//跨域
context.Request.Headers.TryGetValue("Origin", out var origins);
var origin = origins.ToString();
if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin))
{
context.Response.Headers.Add("Access-Control-Allow-Origin", origin);
}
//获取上传的文件
var file = context.Request.Form.Files.FirstOrDefault();
if (file == null || file.Length == 0)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "There is no file data"
}.toJson());
return;
}
//大小验证
if (file.Length > (payload.GetByteSize() ?? _config.GetByteSize()))
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "The file is too big"
}.toJson());
return;
}
//后缀验证
var ext = Path.GetExtension(file.FileName);
if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase)
|| appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase))
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "File extension is not allowed"
}.toJson());
return;
}
//上传逻辑
var now = DateTime.Now;
var yy = now.ToString("yyyy");
var mm = now.ToString("MM");
var dd = now.ToString("dd");
var fileName = Guid.NewGuid().ToString("n") + ext;
var folder = Path.Combine(_config.PhysicalPath, payload.app, yy, mm, dd);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
var filePath = Path.Combine(folder, fileName);
using (var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write))
{
file.CopyTo(fileStream);
fileStream.Flush(true);
}
var fileUrl = _config.RootUrl + "/" + payload.app + "/" + yy + "/" + mm +
"/" +
dd +
"/" + fileName;
await context.Response.WriteAsync(new UploadResult()
{
ok = true,
url = fileUrl
}.toJson());
}
catch (TokenExpiredException)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "Token has expired"
}.toJson());
}
catch (SignatureVerificationException)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "Token has invalid signature"
}.toJson());
}
}
else
{
await context.Response.WriteAsync(new UploadResult()
{
msg = $"Request method '{context.Request.Method}' is not supported"
}.toJson());
}
}分块上传
public async Task InvokeAsync(HttpContext context, RequestDelegate next)
{
context.Response.Headers.Add("Access-Control-Allow-Origin", "*");
context.Response.Headers.Add("Access-Control-Allow-Headers", "content-type,jwt,origin");
if (context.Request.Method.Equals(HttpMethods.Options, StringComparison.OrdinalIgnoreCase))
{
context.Response.StatusCode = (int)HttpStatusCode.OK;
}
else if (context.Request.Method.Equals(HttpMethods.Get, StringComparison.OrdinalIgnoreCase))
{
//简单实现
context.Request.Query.TryGetValue("chunkNumber", out var chunkNumbers);
int.TryParse(chunkNumbers.ToString(), out var chunkNumber);
context.Request.Query.TryGetValue("identifier", out var identifiers);
if (chunkNumber == 0 || string.IsNullOrEmpty(identifiers))
{
context.Response.StatusCode = 204;
}
else
{
var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifiers);
if (File.Exists(chunkFilename))
{
await context.Response.WriteAsync("found");
}
else
{
context.Response.StatusCode = 204;
}
}
}
else if (context.Request.Method.Equals(HttpMethods.Post, StringComparison.OrdinalIgnoreCase))
{
//验证jwt
string token = null;
if (context.Request.Headers.TryGetValue("jwt", out StringValues jwt))
{
token = jwt.ToString();
}
else if (context.Request.Form.TryGetValue("jwt", out jwt))
{
token = jwt.ToString();
}
else
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "No JWT in the header and form"
}.toJson());
return;
}
try
{
var payload = new JwtBuilder().WithSecret(_config.JWTSecret).MustVerifySignature()
.Decode<JwtPayload>(token);
var msg = payload.validate();
if (msg != null)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = msg
}.toJson());
return;
}
//特定的配置
var appConfig = _config.GetAppConfig(payload.app);
//跨域
context.Request.Headers.TryGetValue("Origin", out var origins);
var origin = origins.ToString();
if (!string.IsNullOrEmpty(origin) && appConfig.IsAllowOrigin(origin))
{
context.Response.Headers.Add("Access-Control-Allow-Origin", origin);
}
//获取上传的文件分片
var file = context.Request.Form.Files.FirstOrDefault();
if (file == null || file.Length == 0)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "There is no file data"
}.toJson());
return;
}
//后缀验证
var ext = Path.GetExtension(file.FileName);
if (!(payload.exts + _config.AllowExts).Contains(ext, StringComparison.OrdinalIgnoreCase)
|| appConfig.LimitExts.Contains(ext, StringComparison.OrdinalIgnoreCase))
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "File extension is not allowed"
}.toJson());
return;
}
//获取参数
getParams(context, out var chunkNumber, out var chunkSize, out var totalSize, out string identifier,
out string filename, out int totalChunks);
//验证参数
var validMsg = validateRequest(chunkNumber, chunkSize, totalSize, identifier, filename, file.Length, totalChunks, payload.GetByteSize() ?? _config.GetByteSize());
if (validMsg != null)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = validMsg
}.toJson());
return;
}
else
{
var chunkFilename = getChunkFilename(_config.PhysicalPath, chunkNumber, identifier);
try
{
using (var fileStream = File.OpenWrite(chunkFilename))
{
var stream = file.OpenReadStream();
stream.CopyTo(fileStream);
fileStream.Flush(true);
countDict.AddOrUpdate(identifier, 1, (key, oldValue) => oldValue + 1);
}
if (chunkNumber == totalChunks)
{
//验证块的完整性
while (true)
{
if (countDict.GetValueOrDefault(identifier) < totalChunks)
{
await Task.Delay(TimeSpan.FromMilliseconds(500));
}
else
{
countDict.Remove(identifier, out _);
break;
}
}
//merge file;
string[] chunkFiles = Directory.GetFiles(
Path.Combine(_config.PhysicalPath, temporaryFolder),
"uploader-" + identifier + ".*",
SearchOption.TopDirectoryOnly);
var fileUrl = await MergeChunkFiles(payload, ext, chunkFiles);
await context.Response.WriteAsync(new UploadResult()
{
ok = true,
url = fileUrl
}.toJson());
}
else
{
await context.Response.WriteAsync("partly_done");
return;
}
}
catch (Exception exp)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = exp.Message
}.toJson());
return;
}
}
}
catch (TokenExpiredException)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "Token has expired"
}.toJson());
}
catch (SignatureVerificationException)
{
await context.Response.WriteAsync(new UploadResult()
{
msg = "Token has invalid signature"
}.toJson());
}
}
else
{
context.Response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
await context.Response.WriteAsync($"Request method '{context.Request.Method}' is not supported");
}
}4、上传结果
上传成功
{"ok":true,"msg":null,"url":"http://localhost:6001/test/2019/06/17/abcd.jpg"}上传失败
{"ok":false,"msg":"The file is too big","url":null}
Gitee： https://gitee.com/loogn/UploadServer
- End -
专注分享编程知识、热门有用有趣的开源项目
