mirror of
https://gitee.com/zhangsken/aliyunpan.git
synced 2026-10-16 10:29:42 +08:00
not support upload & download using InternalUrl
This commit is contained in:
@@ -652,7 +652,6 @@ func RunAlbumDownloadFile(albumNames []string, options *DownloadOptions) {
|
|||||||
MaxRate: config.Config.MaxDownloadRate,
|
MaxRate: config.Config.MaxDownloadRate,
|
||||||
InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON,
|
InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON,
|
||||||
ShowProgress: options.ShowProgress,
|
ShowProgress: options.ShowProgress,
|
||||||
UseInternalUrl: config.Config.TransferUrlType == 2,
|
|
||||||
ExcludeNames: options.ExcludeNames,
|
ExcludeNames: options.ExcludeNames,
|
||||||
}
|
}
|
||||||
if cfg.CacheSize == 0 {
|
if cfg.CacheSize == 0 {
|
||||||
|
|||||||
@@ -24,14 +24,7 @@ import (
|
|||||||
"github.com/urfave/cli"
|
"github.com/urfave/cli"
|
||||||
)
|
)
|
||||||
|
|
||||||
type (
|
type ()
|
||||||
// 秒传数据项
|
|
||||||
RapidUploadItem struct {
|
|
||||||
FileSha1 string
|
|
||||||
FileSize int64
|
|
||||||
FilePath string // 绝对路径,包含文件名
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
const (
|
||||||
cryptoDescription = `
|
cryptoDescription = `
|
||||||
@@ -128,10 +121,6 @@ func CmdConfig() cli.Command {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if c.IsSet("transfer_url_type") {
|
|
||||||
config.Config.TransferUrlType = c.Int("transfer_url_type")
|
|
||||||
}
|
|
||||||
|
|
||||||
if c.IsSet("savedir") {
|
if c.IsSet("savedir") {
|
||||||
config.Config.SaveDir = c.String("savedir")
|
config.Config.SaveDir = c.String("savedir")
|
||||||
}
|
}
|
||||||
@@ -183,11 +172,6 @@ func CmdConfig() cli.Command {
|
|||||||
Name: "max_upload_rate",
|
Name: "max_upload_rate",
|
||||||
Usage: "限制最大上传速度, 0代表不限制",
|
Usage: "限制最大上传速度, 0代表不限制",
|
||||||
},
|
},
|
||||||
cli.IntFlag{
|
|
||||||
Name: "transfer_url_type",
|
|
||||||
Usage: "上传下载URL类别,1-默认,2-阿里云ECS",
|
|
||||||
Value: 1,
|
|
||||||
},
|
|
||||||
cli.StringFlag{
|
cli.StringFlag{
|
||||||
Name: "savedir",
|
Name: "savedir",
|
||||||
Usage: "下载文件的储存目录",
|
Usage: "下载文件的储存目录",
|
||||||
|
|||||||
@@ -46,7 +46,6 @@ type (
|
|||||||
NoCheck bool
|
NoCheck bool
|
||||||
ShowProgress bool
|
ShowProgress bool
|
||||||
DriveId string
|
DriveId string
|
||||||
UseInternalUrl bool // 是否使用内置链接
|
|
||||||
ExcludeNames []string // 排除的文件名,包括文件夹和文件。即这些文件/文件夹不进行下载,支持正则表达式
|
ExcludeNames []string // 排除的文件名,包括文件夹和文件。即这些文件/文件夹不进行下载,支持正则表达式
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -256,7 +255,6 @@ func RunDownload(paths []string, options *DownloadOptions) {
|
|||||||
MaxRate: config.Config.MaxDownloadRate,
|
MaxRate: config.Config.MaxDownloadRate,
|
||||||
InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON,
|
InstanceStateStorageFormat: downloader.InstanceStateStorageFormatJSON,
|
||||||
ShowProgress: options.ShowProgress,
|
ShowProgress: options.ShowProgress,
|
||||||
UseInternalUrl: config.Config.TransferUrlType == 2,
|
|
||||||
ExcludeNames: options.ExcludeNames,
|
ExcludeNames: options.ExcludeNames,
|
||||||
}
|
}
|
||||||
if cfg.CacheSize == 0 {
|
if cfg.CacheSize == 0 {
|
||||||
|
|||||||
@@ -297,7 +297,6 @@ priority - 优先级,只对双向同步备份模式有效。选项支持三种
|
|||||||
|
|
||||||
func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadParallel int, downloadBlockSize, uploadBlockSize int64,
|
func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadParallel int, downloadBlockSize, uploadBlockSize int64,
|
||||||
flag syncdrive.SyncPriorityOption, localDelayTime int, taskStep syncdrive.TaskStep) {
|
flag syncdrive.SyncPriorityOption, localDelayTime int, taskStep syncdrive.TaskStep) {
|
||||||
useInternalUrl := config.Config.TransferUrlType == 2
|
|
||||||
maxDownloadRate := config.Config.MaxDownloadRate
|
maxDownloadRate := config.Config.MaxDownloadRate
|
||||||
maxUploadRate := config.Config.MaxUploadRate
|
maxUploadRate := config.Config.MaxUploadRate
|
||||||
activeUser := GetActiveUser()
|
activeUser := GetActiveUser()
|
||||||
@@ -335,10 +334,6 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
|
|||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("启动同步备份进程")
|
fmt.Println("启动同步备份进程")
|
||||||
typeUrlStr := "默认链接"
|
|
||||||
if useInternalUrl {
|
|
||||||
typeUrlStr = "阿里ECS内部链接"
|
|
||||||
}
|
|
||||||
|
|
||||||
// 文件同步记录器
|
// 文件同步记录器
|
||||||
fileRecorder := log.NewFileRecorder(config.GetLogDir() + "/sync_file_records.csv")
|
fileRecorder := log.NewFileRecorder(config.GetLogDir() + "/sync_file_records.csv")
|
||||||
@@ -348,7 +343,6 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
|
|||||||
FileUploadParallel: fileUploadParallel,
|
FileUploadParallel: fileUploadParallel,
|
||||||
FileDownloadBlockSize: downloadBlockSize,
|
FileDownloadBlockSize: downloadBlockSize,
|
||||||
FileUploadBlockSize: uploadBlockSize,
|
FileUploadBlockSize: uploadBlockSize,
|
||||||
UseInternalUrl: useInternalUrl,
|
|
||||||
MaxDownloadRate: maxDownloadRate,
|
MaxDownloadRate: maxDownloadRate,
|
||||||
MaxUploadRate: maxUploadRate,
|
MaxUploadRate: maxUploadRate,
|
||||||
SyncPriority: flag,
|
SyncPriority: flag,
|
||||||
@@ -360,8 +354,8 @@ func RunSync(defaultTask *syncdrive.SyncTask, fileDownloadParallel, fileUploadPa
|
|||||||
if tasks != nil {
|
if tasks != nil {
|
||||||
syncConfigFile = "(使用命令行配置)"
|
syncConfigFile = "(使用命令行配置)"
|
||||||
}
|
}
|
||||||
fmt.Printf("备份配置文件:%s\n链接类型:%s\n下载并发:%d\n上传并发:%d\n下载分片大小:%s\n上传分片大小:%s\n",
|
fmt.Printf("备份配置文件:%s\n下载并发:%d\n上传并发:%d\n下载分片大小:%s\n上传分片大小:%s\n",
|
||||||
syncConfigFile, typeUrlStr, fileDownloadParallel, fileUploadParallel, converter.ConvertFileSize(downloadBlockSize, 2),
|
syncConfigFile, fileDownloadParallel, fileUploadParallel, converter.ConvertFileSize(downloadBlockSize, 2),
|
||||||
converter.ConvertFileSize(uploadBlockSize, 2))
|
converter.ConvertFileSize(uploadBlockSize, 2))
|
||||||
if _, e := syncMgr.Start(tasks, taskStep); e != nil {
|
if _, e := syncMgr.Start(tasks, taskStep); e != nil {
|
||||||
fmt.Println("启动任务失败:", e)
|
fmt.Println("启动任务失败:", e)
|
||||||
|
|||||||
@@ -65,7 +65,6 @@ type (
|
|||||||
DriveId string
|
DriveId string
|
||||||
ExcludeNames []string // 排除的文件名,包括文件夹和文件。即这些文件/文件夹不进行上传,支持正则表达式
|
ExcludeNames []string // 排除的文件名,包括文件夹和文件。即这些文件/文件夹不进行上传,支持正则表达式
|
||||||
BlockSize int64 // 分片大小
|
BlockSize int64 // 分片大小
|
||||||
UseInternalUrl bool // 是否使用内置链接
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -259,7 +258,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
|||||||
if opt.MaxRetry < 0 {
|
if opt.MaxRetry < 0 {
|
||||||
opt.MaxRetry = DefaultUploadMaxRetry
|
opt.MaxRetry = DefaultUploadMaxRetry
|
||||||
}
|
}
|
||||||
opt.UseInternalUrl = config.Config.TransferUrlType == 2
|
|
||||||
|
|
||||||
// 超时时间
|
// 超时时间
|
||||||
if opt.MaxTimeoutSec > 0 {
|
if opt.MaxTimeoutSec > 0 {
|
||||||
@@ -398,7 +396,6 @@ func RunUpload(localPaths []string, savePath string, opt *UploadOptions) {
|
|||||||
ShowProgress: opt.ShowProgress,
|
ShowProgress: opt.ShowProgress,
|
||||||
IsOverwrite: opt.IsOverwrite,
|
IsOverwrite: opt.IsOverwrite,
|
||||||
IsSkipSameName: opt.IsSkipSameName,
|
IsSkipSameName: opt.IsSkipSameName,
|
||||||
UseInternalUrl: opt.UseInternalUrl,
|
|
||||||
GlobalSpeedsStat: globalSpeedsStat,
|
GlobalSpeedsStat: globalSpeedsStat,
|
||||||
FileRecorder: fileRecorder,
|
FileRecorder: fileRecorder,
|
||||||
}, opt.MaxRetry)
|
}, opt.MaxRetry)
|
||||||
|
|||||||
@@ -93,7 +93,6 @@ type PanConfig struct {
|
|||||||
|
|
||||||
MaxDownloadRate int64 `json:"maxDownloadRate"` // 限制最大下载速度,单位 B/s, 即字节/每秒
|
MaxDownloadRate int64 `json:"maxDownloadRate"` // 限制最大下载速度,单位 B/s, 即字节/每秒
|
||||||
MaxUploadRate int64 `json:"maxUploadRate"` // 限制最大上传速度,单位 B/s, 即字节/每秒
|
MaxUploadRate int64 `json:"maxUploadRate"` // 限制最大上传速度,单位 B/s, 即字节/每秒
|
||||||
TransferUrlType int `json:"transferUrlType"` // 上传/下载URL类别,1-默认,2-阿里云ECS
|
|
||||||
|
|
||||||
SaveDir string `json:"saveDir"` // 下载储存路径
|
SaveDir string `json:"saveDir"` // 下载储存路径
|
||||||
|
|
||||||
|
|||||||
@@ -110,7 +110,6 @@ func (c *PanConfig) PrintTable() {
|
|||||||
[]string{"max_upload_parallel", strconv.Itoa(c.MaxUploadParallel), "1 ~ 20", "最大上传并发量,即同时上传文件最大数量"},
|
[]string{"max_upload_parallel", strconv.Itoa(c.MaxUploadParallel), "1 ~ 20", "最大上传并发量,即同时上传文件最大数量"},
|
||||||
[]string{"max_download_rate", showMaxRate(c.MaxDownloadRate), "", "限制单个文件最大下载速度, 0代表不限制"},
|
[]string{"max_download_rate", showMaxRate(c.MaxDownloadRate), "", "限制单个文件最大下载速度, 0代表不限制"},
|
||||||
[]string{"max_upload_rate", showMaxRate(c.MaxUploadRate), "", "限制单个文件最大上传速度, 0代表不限制"},
|
[]string{"max_upload_rate", showMaxRate(c.MaxUploadRate), "", "限制单个文件最大上传速度, 0代表不限制"},
|
||||||
[]string{"transfer_url_type", strconv.Itoa(c.TransferUrlType), "1-默认,2-阿里云ECS", "上传下载URL类别。除非在阿里云ECS(暂只支持经典网络)服务器中使用,不然请设置1"},
|
|
||||||
[]string{"savedir", c.SaveDir, "", "下载文件的储存目录"},
|
[]string{"savedir", c.SaveDir, "", "下载文件的储存目录"},
|
||||||
[]string{"proxy", c.Proxy, "", "设置代理, 支持 http/socks5 代理,例如: http://127.0.0.1:8888 或者 socks5://127.0.0.1:8889"},
|
[]string{"proxy", c.Proxy, "", "设置代理, 支持 http/socks5 代理,例如: http://127.0.0.1:8888 或者 socks5://127.0.0.1:8889"},
|
||||||
[]string{"local_addrs", c.LocalAddrs, "", "设置本地网卡地址, 多个地址用逗号隔开,例如: 127.0.0.1,192.168.100.126"},
|
[]string{"local_addrs", c.LocalAddrs, "", "设置本地网卡地址, 多个地址用逗号隔开,例如: 127.0.0.1,192.168.100.126"},
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
// you may not use this file except in compliance with the License.
|
// you may not use this file except in compliance with the License.
|
||||||
// You may obtain a copy of the License at
|
// You may obtain a copy of the License at
|
||||||
//
|
//
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
//
|
//
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
@@ -30,7 +30,7 @@ var (
|
|||||||
MaxParallelWorkerCount int = 3
|
MaxParallelWorkerCount int = 3
|
||||||
)
|
)
|
||||||
|
|
||||||
//Config 下载配置
|
// Config 下载配置
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Mode transfer.RangeGenMode // 下载Range分配模式
|
Mode transfer.RangeGenMode // 下载Range分配模式
|
||||||
MaxParallel int // 最大下载并发量
|
MaxParallel int // 最大下载并发量
|
||||||
@@ -41,11 +41,10 @@ type Config struct {
|
|||||||
InstanceStatePath string // 断点续传信息路径
|
InstanceStatePath string // 断点续传信息路径
|
||||||
TryHTTP bool // 是否尝试使用 http 连接
|
TryHTTP bool // 是否尝试使用 http 连接
|
||||||
ShowProgress bool // 是否展示下载进度条
|
ShowProgress bool // 是否展示下载进度条
|
||||||
UseInternalUrl bool // 是否使用内置链接
|
|
||||||
ExcludeNames []string // 排除的文件名,包括文件夹和文件。即这些文件/文件夹不进行下载,支持正则表达式
|
ExcludeNames []string // 排除的文件名,包括文件夹和文件。即这些文件/文件夹不进行下载,支持正则表达式
|
||||||
}
|
}
|
||||||
|
|
||||||
//NewConfig 返回默认配置
|
// NewConfig 返回默认配置
|
||||||
func NewConfig() *Config {
|
func NewConfig() *Config {
|
||||||
return &Config{
|
return &Config{
|
||||||
MaxParallel: 5,
|
MaxParallel: 5,
|
||||||
@@ -53,7 +52,7 @@ func NewConfig() *Config {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//Fix 修复配置信息, 使其合法
|
// Fix 修复配置信息, 使其合法
|
||||||
func (cfg *Config) Fix() {
|
func (cfg *Config) Fix() {
|
||||||
fixCacheSize(&cfg.CacheSize)
|
fixCacheSize(&cfg.CacheSize)
|
||||||
if cfg.MaxParallel < 1 {
|
if cfg.MaxParallel < 1 {
|
||||||
@@ -61,7 +60,7 @@ func (cfg *Config) Fix() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//Copy 拷贝新的配置
|
// Copy 拷贝新的配置
|
||||||
func (cfg *Config) Copy() *Config {
|
func (cfg *Config) Copy() *Config {
|
||||||
newCfg := *cfg
|
newCfg := *cfg
|
||||||
return &newCfg
|
return &newCfg
|
||||||
|
|||||||
@@ -411,9 +411,6 @@ func (der *Downloader) Execute() error {
|
|||||||
client.SetTimeout(10 * time.Minute)
|
client.SetTimeout(10 * time.Minute)
|
||||||
|
|
||||||
realUrl := durl.Url
|
realUrl := durl.Url
|
||||||
if der.config.UseInternalUrl {
|
|
||||||
realUrl = durl.InternalUrl
|
|
||||||
}
|
|
||||||
worker := NewWorker(k, der.driveId, der.fileInfo.FileId, realUrl, writer, der.globalSpeedsStat)
|
worker := NewWorker(k, der.driveId, der.fileInfo.FileId, realUrl, writer, der.globalSpeedsStat)
|
||||||
worker.SetClient(client)
|
worker.SetClient(client)
|
||||||
worker.SetPanClient(der.panClient)
|
worker.SetPanClient(der.panClient)
|
||||||
|
|||||||
@@ -38,7 +38,6 @@ type (
|
|||||||
|
|
||||||
// 网盘上传参数
|
// 网盘上传参数
|
||||||
uploadOpEntity *aliyunpan.CreateFileUploadResult
|
uploadOpEntity *aliyunpan.CreateFileUploadResult
|
||||||
useInternalUrl bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
EmptyReaderLen64 struct {
|
EmptyReaderLen64 struct {
|
||||||
@@ -53,13 +52,12 @@ func (e EmptyReaderLen64) Len() int64 {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewPanUpload(panClient *config.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult, useInternalUrl bool) uploader.MultiUpload {
|
func NewPanUpload(panClient *config.PanClient, targetPath, driveId string, uploadOpEntity *aliyunpan.CreateFileUploadResult) uploader.MultiUpload {
|
||||||
return &PanUpload{
|
return &PanUpload{
|
||||||
panClient: panClient,
|
panClient: panClient,
|
||||||
targetPath: targetPath,
|
targetPath: targetPath,
|
||||||
driveId: driveId,
|
driveId: driveId,
|
||||||
uploadOpEntity: uploadOpEntity,
|
uploadOpEntity: uploadOpEntity,
|
||||||
useInternalUrl: useInternalUrl,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,9 +76,6 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
|
|||||||
|
|
||||||
// check url expired or not
|
// check url expired or not
|
||||||
uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL
|
uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL
|
||||||
if pu.useInternalUrl {
|
|
||||||
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
|
|
||||||
}
|
|
||||||
if IsUrlExpired(uploadUrl) {
|
if IsUrlExpired(uploadUrl) {
|
||||||
// get renew upload url
|
// get renew upload url
|
||||||
infoList := make([]aliyunpan.FileUploadPartInfoParam, 0)
|
infoList := make([]aliyunpan.FileUploadPartInfoParam, 0)
|
||||||
@@ -183,9 +178,6 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
|
|||||||
|
|
||||||
// 上传一个分片数据
|
// 上传一个分片数据
|
||||||
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].UploadURL
|
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].UploadURL
|
||||||
if pu.useInternalUrl {
|
|
||||||
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
|
|
||||||
}
|
|
||||||
apiError := pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
|
apiError := pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
|
||||||
|
|
||||||
if respErr != nil {
|
if respErr != nil {
|
||||||
@@ -206,9 +198,6 @@ func (pu *PanUpload) UploadFile(ctx context.Context, partseq int, partOffset int
|
|||||||
// 获取新的上传URL重试一次
|
// 获取新的上传URL重试一次
|
||||||
pu.uploadOpEntity.PartInfoList[partseq] = guur.PartInfoList[0]
|
pu.uploadOpEntity.PartInfoList[partseq] = guur.PartInfoList[0]
|
||||||
uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL
|
uploadUrl := pu.uploadOpEntity.PartInfoList[partseq].UploadURL
|
||||||
if pu.useInternalUrl {
|
|
||||||
uploadUrl = pu.uploadOpEntity.PartInfoList[partseq].InternalUploadURL
|
|
||||||
}
|
|
||||||
apiError = pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
|
apiError = pu.panClient.OpenapiPanClient().UploadFileData(uploadUrl, uploadFunc)
|
||||||
} else if respErr.Err == uploader.UploadPartAlreadyExist {
|
} else if respErr.Err == uploader.UploadPartAlreadyExist {
|
||||||
// already upload
|
// already upload
|
||||||
|
|||||||
@@ -73,9 +73,6 @@ type (
|
|||||||
IsOverwrite bool // 覆盖已存在的文件,如果同名文件已存在则移到回收站里
|
IsOverwrite bool // 覆盖已存在的文件,如果同名文件已存在则移到回收站里
|
||||||
IsSkipSameName bool // 跳过已存在的文件,即使文件内容不一致(不检查SHA1)
|
IsSkipSameName bool // 跳过已存在的文件,即使文件内容不一致(不检查SHA1)
|
||||||
|
|
||||||
// 是否使用内置链接
|
|
||||||
UseInternalUrl bool
|
|
||||||
|
|
||||||
// 全局速度统计
|
// 全局速度统计
|
||||||
GlobalSpeedsStat *speeds.Speeds
|
GlobalSpeedsStat *speeds.Speeds
|
||||||
|
|
||||||
@@ -165,7 +162,7 @@ func (utu *UploadTaskUnit) upload() (result *taskframework.TaskUnitRunResult) {
|
|||||||
// 阿里云盘默认就是分片上传,每一个分片对应一个part_info
|
// 阿里云盘默认就是分片上传,每一个分片对应一个part_info
|
||||||
// 但是不支持分片同时上传,必须单线程,并且按照顺序从1开始一个一个上传
|
// 但是不支持分片同时上传,必须单线程,并且按照顺序从1开始一个一个上传
|
||||||
muer := uploader.NewMultiUploader(
|
muer := uploader.NewMultiUploader(
|
||||||
NewPanUpload(utu.PanClient, utu.SavePath, utu.DriveId, utu.LocalFileChecksum.UploadOpEntity, utu.UseInternalUrl),
|
NewPanUpload(utu.PanClient, utu.SavePath, utu.DriveId, utu.LocalFileChecksum.UploadOpEntity),
|
||||||
rio.NewFileReaderAtLen64(utu.LocalFileChecksum.GetFile()), &uploader.MultiUploaderConfig{
|
rio.NewFileReaderAtLen64(utu.LocalFileChecksum.GetFile()), &uploader.MultiUploaderConfig{
|
||||||
Parallel: utu.Parallel,
|
Parallel: utu.Parallel,
|
||||||
BlockSize: utu.BlockSize,
|
BlockSize: utu.BlockSize,
|
||||||
|
|||||||
@@ -300,9 +300,6 @@ func (f *FileActionTask) downloadFile(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
downloadUrl := durl.Url
|
downloadUrl := durl.Url
|
||||||
if f.syncItem.UseInternalUrl {
|
|
||||||
downloadUrl = durl.InternalUrl
|
|
||||||
}
|
|
||||||
worker := downloader.NewWorker(0, f.syncItem.PanFile.DriveId, f.syncItem.PanFile.FileId, downloadUrl, writer, nil)
|
worker := downloader.NewWorker(0, f.syncItem.PanFile.DriveId, f.syncItem.PanFile.FileId, downloadUrl, writer, nil)
|
||||||
|
|
||||||
// 限速
|
// 限速
|
||||||
@@ -554,9 +551,6 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
|
|||||||
// 检测链接是否过期
|
// 检测链接是否过期
|
||||||
// check url expired or not
|
// check url expired or not
|
||||||
uploadUrl := f.syncItem.UploadEntity.PartInfoList[f.syncItem.UploadPartSeq].UploadURL
|
uploadUrl := f.syncItem.UploadEntity.PartInfoList[f.syncItem.UploadPartSeq].UploadURL
|
||||||
if f.syncItem.UseInternalUrl {
|
|
||||||
uploadUrl = f.syncItem.UploadEntity.PartInfoList[f.syncItem.UploadPartSeq].InternalUploadURL
|
|
||||||
}
|
|
||||||
if panupload.IsUrlExpired(uploadUrl) {
|
if panupload.IsUrlExpired(uploadUrl) {
|
||||||
// get renew upload url
|
// get renew upload url
|
||||||
logger.Verbosef("链接过期,获取新的上传链接: %s\n", targetPanFilePath)
|
logger.Verbosef("链接过期,获取新的上传链接: %s\n", targetPanFilePath)
|
||||||
@@ -584,7 +578,7 @@ func (f *FileActionTask) uploadFile(ctx context.Context) error {
|
|||||||
// 创建分片上传器
|
// 创建分片上传器
|
||||||
// 阿里云盘默认就是分片上传,每一个分片对应一个part_info
|
// 阿里云盘默认就是分片上传,每一个分片对应一个part_info
|
||||||
// 但是不支持分片同时上传,必须单线程,并且按照顺序从1开始一个一个上传
|
// 但是不支持分片同时上传,必须单线程,并且按照顺序从1开始一个一个上传
|
||||||
worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity, f.syncItem.UseInternalUrl)
|
worker := panupload.NewPanUpload(f.panClient, f.syncItem.getPanFileFullPath(), f.syncItem.DriveId, f.syncItem.UploadEntity)
|
||||||
|
|
||||||
// 限速配置
|
// 限速配置
|
||||||
var rateLimit *speeds.RateLimit
|
var rateLimit *speeds.RateLimit
|
||||||
|
|||||||
@@ -347,7 +347,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if file.IsFolder() {
|
if file.IsFolder() {
|
||||||
@@ -379,7 +378,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(fileActionTask)
|
f.addToSyncDb(fileActionTask)
|
||||||
@@ -420,7 +418,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
}
|
}
|
||||||
if file.IsFolder() {
|
if file.IsFolder() {
|
||||||
if localFolderQueue != nil {
|
if localFolderQueue != nil {
|
||||||
@@ -451,7 +448,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(fileActionTask)
|
f.addToSyncDb(fileActionTask)
|
||||||
@@ -492,7 +488,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(deletePanFile)
|
f.addToSyncDb(deletePanFile)
|
||||||
@@ -517,7 +512,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(deletePanFile)
|
f.addToSyncDb(deletePanFile)
|
||||||
@@ -586,7 +580,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(uploadLocalFile)
|
f.addToSyncDb(uploadLocalFile)
|
||||||
@@ -603,7 +596,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(downloadPanFile)
|
f.addToSyncDb(downloadPanFile)
|
||||||
@@ -644,7 +636,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(uploadLocalFile)
|
f.addToSyncDb(uploadLocalFile)
|
||||||
@@ -661,7 +652,6 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
|
|||||||
DriveId: f.task.DriveId,
|
DriveId: f.task.DriveId,
|
||||||
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
DownloadBlockSize: f.syncOption.FileDownloadBlockSize,
|
||||||
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
UploadBlockSize: f.syncOption.FileUploadBlockSize,
|
||||||
UseInternalUrl: f.syncOption.UseInternalUrl,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
f.addToSyncDb(downloadPanFile)
|
f.addToSyncDb(downloadPanFile)
|
||||||
|
|||||||
@@ -1,78 +0,0 @@
|
|||||||
package syncdrive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"github.com/tickstep/aliyunpan-api/aliyunpan"
|
|
||||||
"github.com/tickstep/aliyunpan/internal/utils"
|
|
||||||
"github.com/tickstep/library-go/logger"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFileActionMgrStart(t *testing.T) {
|
|
||||||
refreshToken := "1640cc2d4ea...cb4d6242161a7"
|
|
||||||
webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
|
|
||||||
if err != nil {
|
|
||||||
fmt.Println("get acccess token error")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// pan client
|
|
||||||
panClient := aliyunpan.NewPanClient(*webToken, aliyunpan.AppLoginToken{}, aliyunpan.AppConfig{
|
|
||||||
AppId: "25dzX3vbYqktVxyX",
|
|
||||||
DeviceId: "E75459EXhOTkI5ZI6S3qDHA3",
|
|
||||||
UserId: "",
|
|
||||||
Nonce: 0,
|
|
||||||
PublicKey: "",
|
|
||||||
}, aliyunpan.SessionConfig{
|
|
||||||
DeviceName: "Chrome浏览器",
|
|
||||||
ModelName: "Windows网页版",
|
|
||||||
})
|
|
||||||
user, _ := panClient.GetUserInfo()
|
|
||||||
task := &SyncTask{
|
|
||||||
Id: "5b2d7c10-e927-4e72-8f9d-5abb3bb04814",
|
|
||||||
DriveId: user.FileDriveId,
|
|
||||||
LocalFolderPath: "D:\\smb\\feny\\goprojects\\dev\\NS游戏备份",
|
|
||||||
PanFolderPath: "/sync_drive",
|
|
||||||
Mode: "sync",
|
|
||||||
LastSyncTime: "2022-05-26 19:29:19",
|
|
||||||
|
|
||||||
syncDbFolderPath: "D:\\smb\\feny\\goprojects\\dev\\sync_drive",
|
|
||||||
panClient: panClient,
|
|
||||||
syncOption: SyncOption{
|
|
||||||
FileDownloadParallel: 2,
|
|
||||||
FileUploadParallel: 2,
|
|
||||||
FileDownloadBlockSize: int64(256 * 1024),
|
|
||||||
FileUploadBlockSize: aliyunpan.DefaultChunkSize,
|
|
||||||
UseInternalUrl: false,
|
|
||||||
MaxDownloadRate: 0,
|
|
||||||
MaxUploadRate: 0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
task.setupDb()
|
|
||||||
|
|
||||||
ft := NewFileActionTaskManager(task)
|
|
||||||
ft.Start()
|
|
||||||
|
|
||||||
//go func() {
|
|
||||||
// time.Sleep(10 * time.Second)
|
|
||||||
// task.Stop()
|
|
||||||
//}()
|
|
||||||
time.Sleep(50 * time.Minute)
|
|
||||||
ft.Stop()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFileTime(t *testing.T) {
|
|
||||||
ts := utils.ParseTimeStr("2013-01-11 12:15:07")
|
|
||||||
//ts = time.Now()
|
|
||||||
if err := os.Chtimes("/Volumes/Downloads/dev/upload/password.key", ts, ts); err != nil {
|
|
||||||
logger.Verbosef(err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalFileTime(t *testing.T) {
|
|
||||||
if file, er := os.Stat("/Volumes/Downloads/dev/upload/password.key"); er == nil {
|
|
||||||
fmt.Println(file.ModTime().Format("2006-01-02 15:04:05"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -133,7 +133,6 @@ type (
|
|||||||
StatusUpdateTime string `json:"statusUpdateTime"`
|
StatusUpdateTime string `json:"statusUpdateTime"`
|
||||||
|
|
||||||
DriveId string `json:"driveId"`
|
DriveId string `json:"driveId"`
|
||||||
UseInternalUrl bool `json:"useInternalUrl"`
|
|
||||||
DownloadRange *transfer.Range `json:"downloadRange"`
|
DownloadRange *transfer.Range `json:"downloadRange"`
|
||||||
DownloadBlockSize int64 `json:"downloadBlockSize"`
|
DownloadBlockSize int64 `json:"downloadBlockSize"`
|
||||||
UploadRange *transfer.Range `json:"uploadRange"`
|
UploadRange *transfer.Range `json:"uploadRange"`
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ type (
|
|||||||
FileUploadParallel int // 文件上传并发数
|
FileUploadParallel int // 文件上传并发数
|
||||||
FileDownloadBlockSize int64 // 文件下载分片大小
|
FileDownloadBlockSize int64 // 文件下载分片大小
|
||||||
FileUploadBlockSize int64 // 文件上传分片大小
|
FileUploadBlockSize int64 // 文件上传分片大小
|
||||||
UseInternalUrl bool // 是否使用阿里ECS内部链接
|
|
||||||
|
|
||||||
MaxDownloadRate int64 // 限制最大下载速度
|
MaxDownloadRate int64 // 限制最大下载速度
|
||||||
MaxUploadRate int64 // 限制最大上传速度
|
MaxUploadRate int64 // 限制最大上传速度
|
||||||
|
|||||||
Reference in New Issue
Block a user