Studio/aliyunpan
5
0
Fork 0
mirror of https://gitee.com/zhangsken/aliyunpan.git synced 2026-10-16 10:29:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix webdav download file issue

Browse Source
This commit is contained in:
tickstep
2021-12-31 19:55:00 +08:00
parent f70a5b2fa0
commit b7b157947a
3 changed files with 218 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/webdav/pan_client_proxy.go
+177 -60
View File
@@ -7,30 +7,57 @@ import (
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/library-go/expires"
"github.com/tickstep/library-go/expires/cachemap"
"github.com/tickstep/library-go/logger"
"github.com/tickstep/library-go/requester"
"net/http"
"os"
"path"
"strconv"
"strings"
"time"
)
type FileDownloadStream struct {
fileUrl string
readOffset int64
resp *http.Response
timestamp int64
}
type PanClientProxy struct {
PanUser *config.PanUser
PanDriveId string
// 网盘文件路径到网盘文件信息实体映射缓存
filePathCacheMap cachemap.CacheOpMap
// 网盘文件夹路径到文件夹下面所有子文件映射缓存
fileDirectoryListCacheMap cachemap.CacheOpMap
// 网盘文件ID到文件下载链接映射缓存
fileIdDownloadUrlCacheMap cachemap.CacheOpMap
// 网盘文件ID到文件下载数据流映射缓存
fileIdDownloadStreamCacheMap cachemap.CacheOpMap
}
// CACHE_EXPIRED_MINUTE 缓存过期分钟
const CACHE_EXPIRED_MINUTE = 60
// FILE_DOWNLOAD_URL_EXPIRED_MINUTE 文件下载URL过期分钟,
const FILE_DOWNLOAD_URL_EXPIRED_SECONDS = 14400
func formatPathStyle(pathStr string) string {
pathStr = strings.ReplaceAll(pathStr, "\\", "/")
pathStr = strings.TrimSuffix(pathStr, "/")
return pathStr
}
// DeleteCache 删除含有 dirs 的缓存
func (p *PanClientProxy) deleteFilesDirectoriesListCache(dirs []string) {
cache := p.fileDirectoryListCacheMap.LazyInitCachePoolOp(p.PanDriveId)
for _, v := range dirs {
key := strings.TrimSuffix(v, "/")
key := formatPathStyle(v)
_, ok := cache.Load(key)
if ok {
cache.Delete(key)
@@ -40,14 +67,14 @@ func (p *PanClientProxy) deleteFilesDirectoriesListCache(dirs []string) {
// DeleteOneCache 删除缓存
func (p *PanClientProxy) deleteOneFilesDirectoriesListCache(dirPath string) {
dirPath = strings.TrimSuffix(dirPath, "/")
dirPath = formatPathStyle(dirPath)
ps := []string{dirPath}
p.deleteFilesDirectoriesListCache(ps)
}
// cacheFilesDirectoriesList 缓存文件夹下面的所有文件列表
func (p *PanClientProxy) cacheFilesDirectoriesList(pathStr string) (fdl aliyunpan.FileList, apiError *apierror.ApiError) {
pathStr = strings.TrimSuffix(pathStr, "/")
pathStr = formatPathStyle(pathStr)
data := p.fileDirectoryListCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
fi, er := p.cacheFilePath(pathStr)
if er != nil {
@@ -84,7 +111,7 @@ func (p *PanClientProxy) cacheFilesDirectoriesList(pathStr string) (fdl aliyunpa
// cacheFilePath 缓存文件绝对路径到网盘文件信息
func (p *PanClientProxy) cacheFilePath(pathStr string) (fe *aliyunpan.FileEntity, apiError *apierror.ApiError) {
pathStr = strings.TrimSuffix(pathStr, "/")
pathStr = formatPathStyle(pathStr)
data := p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
var fi *aliyunpan.FileEntity
fi, apiError = p.PanUser.PanClient().FileInfoByPath(p.PanDriveId, pathStr)
@@ -103,7 +130,7 @@ func (p *PanClientProxy) cacheFilePath(pathStr string) (fe *aliyunpan.FileEntity
}
func (p *PanClientProxy) cacheFilePathEntity(fe *aliyunpan.FileEntity) {
pathStr := strings.TrimSuffix(fe.Path, "/")
pathStr := formatPathStyle(fe.Path)
p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
return expires.NewDataExpires(fe, CACHE_EXPIRED_MINUTE*time.Minute)
})
@@ -111,7 +138,7 @@ func (p *PanClientProxy) cacheFilePathEntity(fe *aliyunpan.FileEntity) {
func (p *PanClientProxy) cacheFilePathEntityList(fdl aliyunpan.FileList) {
for _,entity := range fdl {
pathStr := strings.TrimSuffix(entity.Path, "/")
pathStr := formatPathStyle(entity.Path)
p.filePathCacheMap.CacheOperation(p.PanDriveId, pathStr, func() expires.DataExpires {
return expires.NewDataExpires(entity, CACHE_EXPIRED_MINUTE*time.Minute)
})
@@ -119,6 +146,101 @@ func (p *PanClientProxy) cacheFilePathEntityList(fdl aliyunpan.FileList) {
}
// cacheFileDownloadStream 缓存文件下载路径
func (p *PanClientProxy) cacheFileDownloadUrl(sessionId, fileId string) (urlResult *aliyunpan.GetFileDownloadUrlResult, apiError *apierror.ApiError) {
k := sessionId + "-" + fileId
data := p.fileIdDownloadUrlCacheMap.CacheOperation(p.PanDriveId, k, func() expires.DataExpires {
urlResult, err1 := p.PanUser.PanClient().GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: p.PanDriveId,
FileId: fileId,
ExpireSec: FILE_DOWNLOAD_URL_EXPIRED_SECONDS,
})
if err1 != nil {
return nil
}
return expires.NewDataExpires(urlResult, (FILE_DOWNLOAD_URL_EXPIRED_SECONDS-60)*time.Second)
})
if data == nil {
return nil, nil
}
return data.Data().(*aliyunpan.GetFileDownloadUrlResult), nil
}
// DeleteOneCache 删除缓存
func (p *PanClientProxy) deleteOneFileDownloadStreamCache(sessionId, fileId string) {
key := sessionId + "-" + fileId
cache := p.fileIdDownloadStreamCacheMap.LazyInitCachePoolOp(p.PanDriveId)
_, ok := cache.Load(key)
if ok {
cache.Delete(key)
}
}
// cacheFileDownloadStream 缓存文件下载流
func (p *PanClientProxy) cacheFileDownloadStream(sessionId, fileId string, offset int64) (fds *FileDownloadStream, apiError *apierror.ApiError) {
k := sessionId + "-" + fileId
data := p.fileIdDownloadStreamCacheMap.CacheOperation(p.PanDriveId, k, func() expires.DataExpires {
urlResult, err1 := p.cacheFileDownloadUrl(sessionId, fileId)
if err1 != nil {
return nil
}
var resp *http.Response
var err error
var client = requester.NewHTTPClient()
// set to no timeout
client.Timeout = 0
apierr := p.PanUser.PanClient().DownloadFileData(
urlResult.Url,
aliyunpan.FileDownloadRange{
Offset: offset,
End: 0,
},
func(httpMethod, fullUrl string, headers map[string]string) (*http.Response, error) {
resp, err = client.Req(httpMethod, fullUrl, nil, headers)
if err != nil {
return nil, err
}
return resp, err
})
if apierr != nil {
return nil
}
switch resp.StatusCode {
case 200, 206:
// do nothing, continue
break
case 416: //Requested Range Not Satisfiable
fallthrough
case 403: // Forbidden
fallthrough
case 406: // Not Acceptable
return nil
case 404:
return nil
case 429, 509: // Too Many Requests
return nil
default:
return nil
}
logger.Verboseln(sessionId + " create new cache for offset = " + strconv.Itoa(int(offset)))
return expires.NewDataExpires(&FileDownloadStream{
readOffset: offset,
resp: resp,
timestamp: time.Now().Unix(),
}, CACHE_EXPIRED_MINUTE*time.Minute)
})
if data == nil {
return nil, nil
}
return data.Data().(*FileDownloadStream), nil
}
func (p *PanClientProxy) FileInfoByPath(pathStr string) (fileInfo *aliyunpan.FileEntity, error *apierror.ApiError) {
return p.cacheFilePath(pathStr)
}
@@ -131,7 +253,7 @@ func (p *PanClientProxy) Mkdir(pathStr string, perm os.FileMode) error {
if pathStr == "" {
return fmt.Errorf("unknown error")
}
pathStr = strings.ReplaceAll(pathStr, "\\", "/")
pathStr = formatPathStyle(pathStr)
r,er := p.PanUser.PanClient().MkdirByFullPath(p.PanDriveId, pathStr)
if er != nil {
return er
@@ -151,6 +273,9 @@ func (p *PanClientProxy) Mkdir(pathStr string, perm os.FileMode) error {
}
func (p *PanClientProxy) Rename(oldpath, newpath string) error {
oldpath = formatPathStyle(oldpath)
newpath = formatPathStyle(newpath)
oldFile, er := p.cacheFilePath(oldpath)
if er != nil {
return os.ErrNotExist
@@ -173,6 +298,9 @@ func (p *PanClientProxy) Rename(oldpath, newpath string) error {
func (p *PanClientProxy) Move(oldpath, newpath string) error {
oldpath = formatPathStyle(oldpath)
newpath = formatPathStyle(newpath)
oldFile, er := p.cacheFilePath(oldpath)
if er != nil {
return os.ErrNotExist
@@ -203,64 +331,53 @@ func (p *PanClientProxy) Move(oldpath, newpath string) error {
return nil
}
func (p *PanClientProxy) DownloadFilePart(fileId string, offset int64, buffer []byte) (int, error) {
urlResult, err1 := p.PanUser.PanClient().GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
DriveId: p.PanDriveId,
FileId: fileId,
})
func (p *PanClientProxy) DownloadFilePart(sessionId, fileId string, offset int64, buffer []byte) (int, error) {
fds, err1 := p.cacheFileDownloadStream(sessionId, fileId, offset)
if err1 != nil {
return 0, err1
}
var resp *http.Response
var err error
var client = requester.NewHTTPClient()
apierr := p.PanUser.PanClient().DownloadFileData(
urlResult.Url,
aliyunpan.FileDownloadRange{
Offset: offset,
End: offset + int64(len(buffer)),
},
func(httpMethod, fullUrl string, headers map[string]string) (*http.Response, error) {
resp, err = client.Req(httpMethod, fullUrl, nil, headers)
if err != nil {
return nil, err
if fds.readOffset != offset {
// delete old one
if fds.resp != nil {
fds.resp.Body.Close()
}
p.deleteOneFileDownloadStreamCache(sessionId, fileId)
logger.Verboseln(sessionId + " offset mismatch offset = " + strconv.Itoa(int(offset)) + " cache offset = " + strconv.Itoa(int(fds.readOffset)))
// create new one
fds, err1 = p.cacheFileDownloadStream(sessionId, fileId, offset)
if err1 != nil {
return 0, err1
}
}
if fds.resp.Close {
// delete old one
p.deleteOneFileDownloadStreamCache(sessionId, fileId)
logger.Verboseln(sessionId + "remote data stream close, stream offset = " + strconv.Itoa(int(fds.readOffset)))
// create new one
fds, err1 = p.cacheFileDownloadStream(sessionId, fileId, offset)
if err1 != nil {
return 0, err1
}
}
readByteCount, readErr := fds.resp.Body.Read(buffer)
if readErr != nil {
if readErr.Error() == "EOF" {
logger.Verboseln(sessionId + " read EOF last offset = " + strconv.Itoa(int(offset)))
// end of file
if fds.resp != nil {
fds.resp.Body.Close()
}
return resp, err
})
if apierr != nil {
return 0, apierr
}
// close socket defer
if resp != nil {
defer func() {
resp.Body.Close()
}()
}
switch resp.StatusCode {
case 200, 206:
// do nothing, continue
break
case 416: //Requested Range Not Satisfiable
fallthrough
case 403: // Forbidden
fallthrough
case 406: // Not Acceptable
return 0, apierror.NewFailedApiError("")
case 404:
return 0, apierror.NewFailedApiError("")
case 429, 509: // Too Many Requests
return 0, apierror.NewFailedApiError("")
default:
return 0, apierror.NewApiErrorWithError(fmt.Errorf("unexpected http status code, %d, %s", resp.StatusCode, resp.Status))
}
readByteCount, readErr := resp.Body.Read(buffer)
if readErr != nil && readErr.Error() != "EOF"{
return 0, readErr
p.deleteOneFileDownloadStreamCache(sessionId, fileId)
} else {
// TODO: handler other error
return 0, readErr
}
}
fds.readOffset += int64(readByteCount)
return readByteCount, nil
}
internal/webdav/webdav.go
+1 -2
View File
@@ -142,10 +142,9 @@ func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
}
// Runs the WebDAV.
//u.Handler.LockSystem = webdav.NewMemLS()
u.Handler.ServeHTTP(w, r)
u.Handler.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), "sessionId", r.RemoteAddr)))
}
// responseWriterNoBody is a wrapper used to suprress the body of the response
internal/webdav/webdav_file.go
+40 -15
View File
@@ -2,6 +2,7 @@ package webdav
import (
"context"
"errors"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/library-go/logger"
"io"
@@ -9,6 +10,7 @@ import (
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
@@ -56,6 +58,14 @@ func (d WebDavDir) formatAbsoluteName(pathStr string) string {
return pathStr
}
func (d WebDavDir) getSessionId(ctx context.Context) string {
v := ctx.Value("sessionId")
if v != nil{
return v.(string)
}
return ""
}
func (d WebDavDir) resolve(name string) string {
// This implementation is based on Dir.Open's code in the standard net/http package.
if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 ||
@@ -78,7 +88,7 @@ func (d WebDavDir) Mkdir(ctx context.Context, name string, perm os.FileMode) err
func (d WebDavDir) OpenFile(ctx context.Context, name string, flag int, perm os.FileMode) (webdav.File, error) {
if name == "" {
return WebDavFile{
return &WebDavFile{
panClientProxy: d.panClientProxy,
nameSnapshot: d.fileInfo,
childrenSnapshot: nil,
@@ -95,13 +105,14 @@ func (d WebDavDir) OpenFile(ctx context.Context, name string, flag int, perm os.
}
wdfi := NewWebDavFileInfo(fileItem)
wdfi.fullPath = d.formatAbsoluteName(name)
return WebDavFile{
return &WebDavFile{
panClientProxy: d.panClientProxy,
nameSnapshot: wdfi,
childrenSnapshot: nil,
listPos: 0,
readPos: 0,
writePos: 0,
sessionId: d.getSessionId(ctx),
}, nil
}
@@ -164,16 +175,18 @@ type WebDavFile struct {
listPos int
readPos int64
writePos int64
sessionId string
}
func (f WebDavFile) Close() error {
func (f *WebDavFile) Close() error {
f.readPos = 0
f.writePos = 0
return nil
}
func (f WebDavFile) Read(p []byte) (int, error) {
count, err := f.panClientProxy.DownloadFilePart(f.nameSnapshot.fileId, f.readPos, p)
func (f *WebDavFile) Read(p []byte) (int, error) {
count, err := f.panClientProxy.DownloadFilePart(f.sessionId, f.nameSnapshot.fileId, f.readPos, p)
if err != nil {
return 0, err
}
@@ -182,7 +195,7 @@ func (f WebDavFile) Read(p []byte) (int, error) {
}
// Readdir 获取文件目录
func (f WebDavFile) Readdir(count int) (fis []os.FileInfo, err error) {
func (f *WebDavFile) Readdir(count int) (fis []os.FileInfo, err error) {
if f.childrenSnapshot == nil || len(f.childrenSnapshot) == 0 {
fileList, e := f.panClientProxy.FileListGetAll(f.nameSnapshot.fullPath)
if e != nil {
@@ -212,21 +225,33 @@ func (f WebDavFile) Readdir(count int) (fis []os.FileInfo, err error) {
return fis, nil
}
func (f WebDavFile) Seek(off int64, whence int) (int64, error) {
if whence == io.SeekEnd {
return f.nameSnapshot.size - f.readPos, nil
} else if whence == io.SeekStart{
f.readPos += off
return f.readPos, nil
func (f *WebDavFile) Seek(offset int64, whence int) (int64, error) {
var abs int64
switch whence {
case io.SeekStart:
abs = offset
logger.Verboseln(f.sessionId + " SeekStart offset = " + strconv.Itoa(int(offset)) + " return offset = " + strconv.Itoa(int(abs)))
case io.SeekCurrent:
abs = f.readPos + offset
logger.Verboseln(f.sessionId + " SeekCurrent offset = " + strconv.Itoa(int(offset)) + " return offset = " + strconv.Itoa(int(abs)))
case io.SeekEnd:
abs = f.nameSnapshot.size + offset
logger.Verboseln(f.sessionId + " SeekEnd offset = " + strconv.Itoa(int(offset)) + " return offset = " + strconv.Itoa(int(abs)))
default:
return 0, errors.New("invalid whence")
}
return 0, nil
if abs < 0 {
return 0, os.ErrInvalid
}
f.readPos = abs
return f.readPos, nil
}
func (f WebDavFile) Stat() (os.FileInfo, error) {
func (f *WebDavFile) Stat() (os.FileInfo, error) {
return &f.nameSnapshot, nil
}
func (f WebDavFile) Write(p []byte) (int, error) {
func (f *WebDavFile) Write(p []byte) (int, error) {
return 0, nil
}