Studio/aliyunpan
5
0
Fork 0
mirror of https://gitee.com/zhangsken/aliyunpan.git synced 2026-10-16 10:29:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

add sync config

Browse Source
This commit is contained in:
xiaoyaofenfen
2022-06-10 13:55:00 +08:00
parent f1c6379e33
commit debaf655f0
8 changed files with 197 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/command/sync.go
+46 -11
View File
@@ -15,6 +15,7 @@ package command
import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan/cmder"
"github.com/tickstep/aliyunpan/internal/config"
"github.com/tickstep/aliyunpan/internal/syncdrive"
@@ -36,13 +37,15 @@ func CmdSync() cli.Command {
备份功能支持一下三种模式：
1. upload
备份本地文件，即上传本地文件到网盘，始终保持本地文件有一个完整的备份在网盘
2. download
备份云盘文件，即下载网盘文件到本地，始终保持网盘的文件有一个完整的备份在本地
3. sync
双向备份，保持网盘文件和本地文件严格一致
示例:
1. 将本地的 C:\Users\Administrator\Video 整个目录备份到网盘 /视频 目录
注意区别反斜杠 "\" 和 斜杠 "/" !!!
aliyunpan-go backup C:/Users/Administrator/Video /视频
`,
Category: "阿里云盘",
Before: cmder.ReloadConfigFunc,
@@ -51,25 +54,56 @@ func CmdSync() cli.Command {
fmt.Println("未登录账号")
return nil
}
blockSize := int64(c.Int("bs") * 1024)
RunSync(blockSize)
dp := c.Int("dp")
if dp == 0 {
dp = config.Config.MaxDownloadParallel
}
if dp == 0 {
dp = 5
}
up := c.Int("up")
if up == 0 {
up = config.Config.MaxUploadParallel
}
if up == 0 {
up = 5
}
downloadBlockSize := int64(config.Config.CacheSize)
if downloadBlockSize == 0 {
downloadBlockSize = int64(256 * 1024)
}
uploadBlockSize := int64(c.Int("ubs") * 1024)
if uploadBlockSize == 0 {
uploadBlockSize = aliyunpan.DefaultChunkSize
}
RunSync(up, dp, uploadBlockSize, downloadBlockSize)
return nil
},
Flags: []cli.Flag{
cli.BoolFlag{
Name: "log",
Usage: "开启log输出",
cli.IntFlag{
Name: "dp",
Usage: "Download Parallel, 下载并发数量，即可以同时并发下载多少个文件。0代表跟从配置文件设置（取值范围:1 ~ 10）",
Value: 0,
},
cli.IntFlag{
Name: "bs",
Usage: "block size，上传分片大小，单位KB。推荐值：1024 ~ 10240",
Name: "up",
Usage: "upload parallel, 上传并发数量，即可以同时并发上传多少个文件。0代表跟从配置文件设置（取值范围:1 ~ 10）",
Value: 0,
},
cli.IntFlag{
Name: "ubs",
Usage: "upload block size，上传分片大小，单位KB。推荐值：1024 ~ 10240",
Value: 10240,
},
},
}
}
func RunSync(uploadBlockSize int64) {
func RunSync(fileDownloadParallel, fileUploadParallel int, downloadBlockSize, uploadBlockSize int64) {
activeUser := GetActiveUser()
panClient := activeUser.PanClient()
@@ -92,7 +126,8 @@ func RunSync(uploadBlockSize int64) {
}
fmt.Println("启动同步备份进程")
syncMgr := syncdrive.NewSyncTaskManager(activeUser.DriveList.GetFileDriveId(), panClient, syncFolderRootPath)
syncMgr := syncdrive.NewSyncTaskManager(activeUser.DriveList.GetFileDriveId(), panClient, syncFolderRootPath,
fileDownloadParallel, fileUploadParallel, downloadBlockSize, uploadBlockSize)
if _, e := syncMgr.Start(); e != nil {
fmt.Println("启动任务失败：", e)
return
internal/syncdrive/file_action_task.go
+2 -3
View File
@@ -30,7 +30,6 @@ type (
syncFileDb SyncFileDb
panClient *aliyunpan.PanClient
blockSize int64
syncItem *SyncFileItem
@@ -200,7 +199,7 @@ func (f *FileActionTask) downloadFile(ctx context.Context) error {
if f.syncItem.DownloadRange == nil {
f.syncItem.DownloadRange = &transfer.Range{
Begin: 0,
End: f.blockSize,
End: f.syncItem.DownloadBlockSize,
}
}
worker.SetRange(f.syncItem.DownloadRange) // 分片
@@ -238,7 +237,7 @@ func (f *FileActionTask) downloadFile(ctx context.Context) error {
// 下一个分片
f.syncItem.DownloadRange.Begin = f.syncItem.DownloadRange.End
f.syncItem.DownloadRange.End += f.blockSize
f.syncItem.DownloadRange.End += f.syncItem.DownloadBlockSize
// 存储状态
f.syncFileDb.Update(f.syncItem)
internal/syncdrive/file_action_task_mgr.go
+104 -98
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
mapset "github.com/deckarep/golang-set"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/aliyunpan/internal/localfile"
"github.com/tickstep/aliyunpan/internal/waitgroup"
"github.com/tickstep/aliyunpan/library/collection"
@@ -52,11 +51,11 @@ func NewFileActionTaskManager(task *SyncTask) *FileActionTaskManager {
task: task,
fileInProcessQueue: collection.NewFifoQueue(),
fileDownloadParallel: 2,
fileUploadParallel: 2,
fileDownloadParallel: task.fileDownloadParallel,
fileUploadParallel: task.fileUploadParallel,
fileDownloadBlockSize: int64(10 * 1024 * 1024),
fileUploadBlockSize: aliyunpan.DefaultChunkSize,
fileDownloadBlockSize: task.fileDownloadBlockSize,
fileUploadBlockSize: task.fileUploadBlockSize,
}
}
@@ -247,15 +246,16 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
}
fileActionTask := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionDownload,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: file,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionDownload,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: file,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(fileActionTask)
@@ -264,15 +264,16 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
if f.task.Mode == DownloadOnly || f.task.Mode == SyncTwoWay {
fileActionTask := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionDeleteLocal,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: file,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionDeleteLocal,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: file,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(fileActionTask)
@@ -297,15 +298,16 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
}
fileActionTask := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionUpload,
Status: SyncFileStatusCreate,
LocalFile: file,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionUpload,
Status: SyncFileStatusCreate,
LocalFile: file,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(fileActionTask)
@@ -314,15 +316,16 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
if f.task.Mode == UploadOnly || f.task.Mode == SyncTwoWay {
fileActionTask := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionDeletePan,
Status: SyncFileStatusCreate,
LocalFile: file,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionDeletePan,
Status: SyncFileStatusCreate,
LocalFile: file,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(fileActionTask)
@@ -353,15 +356,16 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
// 删除对应的云盘文件
deletePanFile := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionDeletePan,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionDeletePan,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(deletePanFile)
@@ -376,15 +380,16 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
// 删除对应的本地文件
deletePanFile := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionDeleteLocal,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionDeleteLocal,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(deletePanFile)
@@ -434,30 +439,32 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
if f.task.Mode == UploadOnly {
uploadLocalFile := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionUpload,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionUpload,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(uploadLocalFile)
} else if f.task.Mode == DownloadOnly {
downloadPanFile := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionDownload,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionDownload,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(downloadPanFile)
@@ -465,30 +472,32 @@ func (f *FileActionTaskManager) doFileDiffRoutine(panFiles PanFileList, localFil
if localFile.UpdateTimeUnix() > panFile.UpdateTimeUnix() { // upload file
uploadLocalFile := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionUpload,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionUpload,
Status: SyncFileStatusCreate,
LocalFile: localFile,
PanFile: nil,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(uploadLocalFile)
} else if localFile.UpdateTimeUnix() < panFile.UpdateTimeUnix() { // download file
downloadPanFile := &FileActionTask{
syncItem: &SyncFileItem{
Action: SyncFileActionDownload,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
UploadBlockSize: f.fileUploadBlockSize,
Action: SyncFileActionDownload,
Status: SyncFileStatusCreate,
LocalFile: nil,
PanFile: panFile,
StatusUpdateTime: "",
PanFolderPath: f.task.PanFolderPath,
LocalFolderPath: f.task.LocalFolderPath,
DriveId: f.task.DriveId,
DownloadBlockSize: f.fileDownloadBlockSize,
UploadBlockSize: f.fileUploadBlockSize,
},
}
f.addToSyncDb(downloadPanFile)
@@ -548,7 +557,6 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
panFileDb: f.task.panFileDb,
syncFileDb: f.task.syncFileDb,
panClient: f.task.panClient,
blockSize: int64(10485760),
syncItem: file,
panFolderCreateMutex: f.folderCreateMutex,
}
@@ -564,7 +572,6 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
panFileDb: f.task.panFileDb,
syncFileDb: f.task.syncFileDb,
panClient: f.task.panClient,
blockSize: int64(10485760),
syncItem: file,
panFolderCreateMutex: f.folderCreateMutex,
}
@@ -582,7 +589,6 @@ func (f *FileActionTaskManager) getFromSyncDb(act SyncFileAction) *FileActionTas
panFileDb: f.task.panFileDb,
syncFileDb: f.task.syncFileDb,
panClient: f.task.panClient,
blockSize: int64(10485760),
syncItem: file,
panFolderCreateMutex: f.folderCreateMutex,
}
internal/syncdrive/file_action_task_mgr_test.go
+9 -4
View File
@@ -11,7 +11,7 @@ import (
)
func TestFileActionMgrStart(t *testing.T) {
refreshToken := "ac1010f6396...585338b533bd4ab"
refreshToken := "1640cc2d4ea...cb4d6242161a7"
webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
if err != nil {
fmt.Println("get acccess token error")
@@ -24,15 +24,20 @@ func TestFileActionMgrStart(t *testing.T) {
task := &SyncTask{
Id: "5b2d7c10-e927-4e72-8f9d-5abb3bb04814",
DriveId: user.FileDriveId,
LocalFolderPath: "/Volumes/Downloads/dev/folder",
LocalFolderPath: "D:\\smb\\feny\\goprojects\\dev\\NS游戏备份",
PanFolderPath: "/sync_drive",
Mode: "sync",
LastSyncTime: "2022-05-26 19:29:19",
syncDbFolderPath: "/Volumes/Downloads/dev/sync_drive",
syncDbFolderPath: "D:\\smb\\feny\\goprojects\\dev\\sync_drive",
panClient: panClient,
fileDownloadParallel: 2,
fileUploadParallel: 2,
fileDownloadBlockSize: int64(256 * 1024),
fileUploadBlockSize: aliyunpan.DefaultChunkSize,
}
task.setup()
task.setupDb()
ft := NewFileActionTaskManager(task)
ft.Start()
internal/syncdrive/sync_db.go
+6 -5
View File
@@ -129,11 +129,12 @@ type (
PanFolderPath string `json:"panFolderPath"`
StatusUpdateTime string `json:"statusUpdateTime"`
DriveId string `json:"driveId"`
UseInternalUrl bool `json:"useInternalUrl"`
DownloadRange *transfer.Range `json:"downloadRange"`
UploadRange *transfer.Range `json:"uploadRange"`
UploadEntity *aliyunpan.CreateFileUploadResult `json:"uploadEntity"`
DriveId string `json:"driveId"`
UseInternalUrl bool `json:"useInternalUrl"`
DownloadRange *transfer.Range `json:"downloadRange"`
DownloadBlockSize int64 `json:"downloadBlockSize"`
UploadRange *transfer.Range `json:"uploadRange"`
UploadEntity *aliyunpan.CreateFileUploadResult `json:"uploadEntity"`
// UploadPartSeq 上传序号，从0开始
UploadPartSeq int `json:"uploadPartSeq"`
UploadBlockSize int64 `json:"uploadBlockSize"`
internal/syncdrive/sync_task.go
+5
View File
@@ -46,6 +46,11 @@ type (
panClient *aliyunpan.PanClient
fileDownloadParallel int
fileUploadParallel int
fileDownloadBlockSize int64
fileUploadBlockSize int64
fileActionTaskManager *FileActionTaskManager
}
)
internal/syncdrive/sync_task_mgr.go
+18 -2
View File
@@ -14,7 +14,13 @@ import (
type (
// SyncTaskManager 同步任务管理器
SyncTaskManager struct {
syncDriveConfig *SyncDriveConfig
syncDriveConfig *SyncDriveConfig
fileDownloadParallel int
fileUploadParallel int
fileDownloadBlockSize int64
fileUploadBlockSize int64
DriveId string
PanClient *aliyunpan.PanClient
SyncConfigFolderPath string
@@ -31,11 +37,17 @@ var (
ErrSyncTaskListEmpty error = fmt.Errorf("no sync task")
)
func NewSyncTaskManager(driveId string, panClient *aliyunpan.PanClient, syncConfigFolderPath string) *SyncTaskManager {
func NewSyncTaskManager(driveId string, panClient *aliyunpan.PanClient, syncConfigFolderPath string, fileDownloadParallel, fileUploadParallel int, fileDownloadBlockSize, fileUploadBlockSize int64) *SyncTaskManager {
return &SyncTaskManager{
DriveId: driveId,
PanClient: panClient,
SyncConfigFolderPath: syncConfigFolderPath,
fileDownloadParallel: fileDownloadParallel,
fileUploadParallel: fileUploadParallel,
fileDownloadBlockSize: fileDownloadBlockSize,
fileUploadBlockSize: fileUploadBlockSize,
}
}
@@ -103,6 +115,10 @@ func (m *SyncTaskManager) Start() (bool, error) {
task.DriveId = m.DriveId
task.syncDbFolderPath = m.SyncConfigFolderPath
task.panClient = m.PanClient
task.fileUploadParallel = m.fileUploadParallel
task.fileDownloadParallel = m.fileDownloadParallel
task.fileUploadBlockSize = m.fileUploadBlockSize
task.fileDownloadBlockSize = m.fileDownloadBlockSize
if e := task.Start(); e != nil {
logger.Verboseln(e)
fmt.Println("start sync task error: {}", task.Id)
internal/syncdrive/sync_task_mgr_test.go
+7 -3
View File
@@ -8,7 +8,7 @@ import (
)
func TestStart(t *testing.T) {
refreshToken := "ac1010f63...9585338b533bd4ab"
refreshToken := "1640cc2d4ea...6b8ccb4d6242161a7"
webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
if err != nil {
fmt.Println("get acccess token error")
@@ -22,10 +22,14 @@ func TestStart(t *testing.T) {
manager := NewSyncTaskManager(
user.FileDriveId,
panClient,
"/Volumes/Downloads/dev/sync_drive",
"D:\\smb\\feny\\goprojects\\dev\\sync_drive",
1,
1,
int64(256*1024),
aliyunpan.DefaultChunkSize,
)
manager.Start()
time.Sleep(5 * time.Minute)
time.Sleep(30 * time.Minute)
manager.Stop()
}