add download action for sync

internal/syncdrive/bolt_db.go

@@ -24,6 +24,10 @@ type (
 	}
 )
 
+const (
+	DirBucketPrefix string = "##:@@"
+)
+
 func NewBoltDb(dbFilePath string) *BoltDb {
 	return &BoltDb{
 		Path:   dbFilePath,
@@ -47,26 +51,22 @@ func (b *BoltDb) addOneItem(rootBucket *bolt.Bucket, item *BoltItem) (bool, erro
 		if p == "" {
 			continue
 		}
-		bkt, _ = bkt.CreateBucketIfNotExists([]byte(p))
+		bkt, _ = bkt.CreateBucketIfNotExists([]byte(DirBucketPrefix + p))
 	}
 	if bkt == nil {
 		return false, fmt.Errorf("create or get bucket error")
 	}
 
 	fileName := path.Base(item.FilePath)
+	if e := bkt.Put([]byte(fileName), []byte(item.Data)); e != nil {
+		return false, e
+	}
 	var err error
 	if item.IsFolder {
-		bkt, err = bkt.CreateBucketIfNotExists([]byte(fileName))
+		bkt, err = bkt.CreateBucketIfNotExists([]byte(DirBucketPrefix + fileName))
 		if err != nil {
 			return false, err
 		}
-		if e := bkt.Put([]byte(DefaultDirKeyName), []byte(item.Data)); e != nil {
-			return false, e
-		}
-	} else {
-		if e := bkt.Put([]byte(fileName), []byte(item.Data)); e != nil {
-			return false, e
-		}
 	}
 	return true, nil
 }
@@ -168,7 +168,7 @@ func (b *BoltDb) Get(filePath string) (string, error) {
 		return "", ErrItemNotExisted
 	}
 	for _, p := range parts[:len(parts)-1] {
-		bkt = bkt.Bucket([]byte(p))
+		bkt = bkt.Bucket([]byte(DirBucketPrefix + p))
 		if bkt == nil {
 			return "", ErrItemNotExisted
 		}
@@ -177,14 +177,7 @@ func (b *BoltDb) Get(filePath string) (string, error) {
 		return "", ErrItemNotExisted
 	}
 
-	dirBucket := bkt.Bucket([]byte(parts[len(parts)-1]))
-	var data []byte
-	if dirBucket != nil {
-		// is dir
-		data = dirBucket.Get([]byte(DefaultDirKeyName))
-	} else {
-		data = bkt.Get([]byte(parts[len(parts)-1]))
-	}
+	data := bkt.Get([]byte(parts[len(parts)-1]))
 	if data != nil {
 		return string(data), nil
 	}
@@ -218,26 +211,25 @@ func (b *BoltDb) GetFileList(filePath string) ([]string, error) {
 	if bkt == nil {
 		return dataList, ErrItemNotExisted
 	}
-	for _, p := range parts[:len(parts)-1] {
-		bkt = bkt.Bucket([]byte(p))
+	if len(parts) > 0 {
+		for _, p := range parts[:len(parts)-1] {
+			bkt = bkt.Bucket([]byte(DirBucketPrefix + p))
+			if bkt == nil {
+				return dataList, ErrItemNotExisted
+			}
+		}
+		bkt = bkt.Bucket([]byte(DirBucketPrefix + parts[len(parts)-1]))
 		if bkt == nil {
 			return dataList, ErrItemNotExisted
 		}
 	}
+
 	if bkt == nil {
 		return dataList, ErrItemNotExisted
 	}
 
-	dirBucket := bkt.Bucket([]byte(parts[len(parts)-1]))
-	if dirBucket == nil {
-		return dataList, ErrItemNotExisted
-	}
-
-	c := dirBucket.Cursor()
+	c := bkt.Cursor()
 	for k, v := c.First(); k != nil; k, v = c.Next() {
-		if string(k) == DefaultDirKeyName {
-			continue
-		}
 		dataList = append(dataList, string(v))
 	}
 	return dataList, nil
@@ -273,7 +265,7 @@ func (b *BoltDb) Delete(filePath string) (bool, error) {
 		return false, ErrItemNotExisted
 	}
 	for _, p := range parts[:len(parts)-1] {
-		bkt = bkt.Bucket([]byte(p))
+		bkt = bkt.Bucket([]byte(DirBucketPrefix + p))
 		if bkt == nil {
 			return false, ErrItemNotExisted
 		}
@@ -282,16 +274,16 @@ func (b *BoltDb) Delete(filePath string) (bool, error) {
 		return false, ErrItemNotExisted
 	}
 
-	targetName := []byte(parts[len(parts)-1])
-	dirBucket := bkt.Bucket(targetName)
+	targetName := parts[len(parts)-1]
+	dirBucket := bkt.Bucket([]byte(DirBucketPrefix + targetName))
 	var er error
 	if dirBucket != nil {
 		// is dir, delete bucket
-		er = bkt.DeleteBucket(targetName)
-	} else {
-		// is file, delete item
-		er = bkt.Delete(targetName)
+		er = bkt.DeleteBucket([]byte(DirBucketPrefix + targetName))
 	}
+	// delete item
+	er = bkt.Delete([]byte(targetName))
+
 	// Commit the transaction and check for error.
 	if err := tx.Commit(); err != nil {
 		return false, err
@@ -300,7 +292,7 @@ func (b *BoltDb) Delete(filePath string) (bool, error) {
 }
 
 // Update 更新数据项，数据项不存在返回错误
-func (b *BoltDb) Update(filePath string, isFolder bool, data string) (bool, error) {
+func (b *BoltDb) Update(filePath string, data string) (bool, error) {
 	filePath = FormatFilePath(filePath)
 	b.locker.Lock()
 	defer b.locker.Unlock()
@@ -334,19 +326,8 @@ func (b *BoltDb) Update(filePath string, isFolder bool, data string) (bool, erro
 
 	// update content
 	fileName := path.Base(filePath)
-	if isFolder {
-		bkt = bkt.Bucket([]byte(fileName))
-		if bkt == nil {
-			return false, ErrItemNotExisted
-		}
-		if e := bkt.Put([]byte(DefaultDirKeyName), []byte(data)); e != nil {
-			return false, e
-		}
-	} else {
-		// is file
-		if e := bkt.Put([]byte(fileName), []byte(data)); e != nil {
-			return false, e
-		}
+	if e := bkt.Put([]byte(fileName), []byte(data)); e != nil {
+		return false, e
 	}
 
 	// Commit the transaction and check for error.

internal/syncdrive/bolt_db_test.go

@@ -3,14 +3,38 @@ package syncdrive
 import (
 	"context"
 	"fmt"
-	"path"
+	"github.com/tickstep/bolt"
 	"strings"
 	"testing"
 	"time"
 )
 
 func TestPath(t *testing.T) {
-	fmt.Println(path.Dir("/dsgsdghsd/dsgsdh/eg34/dghsh"))
+	db, err := bolt.Open("D:\\smb\\feny\\goprojects\\dev\\sync_drive\\5b2d7c10-e927-4e72-8f9d-5abb3bb04814\\test.db", 0600, &bolt.Options{Timeout: 5 * time.Second})
+	if err != nil {
+		return
+	}
+	defer db.Close()
+
+	// add item
+	// Start a writable transaction.
+	tx, err := db.Begin(true)
+	if err != nil {
+		return
+	}
+	defer tx.Rollback()
+
+	rootBucket, _ := tx.CreateBucketIfNotExists([]byte("/"))
+	if e := rootBucket.Put([]byte("test"), []byte("test value")); e != nil {
+		return
+	}
+	_, e := rootBucket.CreateBucketIfNotExists([]byte("test"))
+	println(e)
+
+	// Commit the transaction and check for error.
+	if err := tx.Commit(); err != nil {
+		return
+	}
 }
 
 func dosomething(ctx context.Context) {
@@ -39,6 +63,16 @@ func TestBolt(t *testing.T) {
 	localFileDb := NewLocalSyncDb("D:\\smb\\feny\\goprojects\\dev\\sync_drive\\local.db")
 	localFileDb.Open()
 	defer localFileDb.Close()
+	localFileDb.Add(&LocalFileItem{
+		FileName:      "dev",
+		FileSize:      0,
+		FileType:      "folder",
+		CreatedAt:     "2022-05-12 10:21:14",
+		UpdatedAt:     "2022-05-12 10:21:14",
+		FileExtension: ".db",
+		Sha1Hash:      "",
+		Path:          "D:\\smb\\feny\\goprojects\\dev",
+	})
 	localFileDb.Add(&LocalFileItem{
 		FileName:      "file1.db",
 		FileSize:      0,

internal/syncdrive/file_action_task.go

@@ -1,37 +1,199 @@
 package syncdrive
 
+import (
+	"context"
+	"fmt"
+	"github.com/tickstep/aliyunpan-api/aliyunpan"
+	"github.com/tickstep/aliyunpan/internal/file/downloader"
+	"github.com/tickstep/aliyunpan/internal/utils"
+	"github.com/tickstep/aliyunpan/library/requester/transfer"
+	"github.com/tickstep/library-go/logger"
+	"github.com/tickstep/library-go/requester"
+	"os"
+	"path"
+	"strings"
+	"sync"
+	"time"
+)
+
 type (
 	FileAction     string
 	FileActionTask struct {
-		Action    FileAction
-		LocalFile *LocalFileItem
-		PanFile   *PanFileItem
+		localFileDb LocalSyncDb
+		panFileDb   PanSyncDb
+		syncFileDb  SyncFileDb
+
+		panClient *aliyunpan.PanClient
+		blockSize int64
+
+		syncItem *SyncFileItem
 	}
 )
 
-const (
-	DownloadFile    FileAction = "DownloadFile"
-	UploadFile      FileAction = "UploadFile"
-	DeleteLocalFile FileAction = "DeleteLocalFile"
-	DeletePanFile   FileAction = "DeletePanFile"
-)
+func (f *FileActionTask) HashCode() string {
+	postfix := ""
+	if f.syncItem.Action == SyncFileActionDownload {
+		postfix = strings.ReplaceAll(f.syncItem.PanFile.Path, "\\", "/")
+	} else if f.syncItem.Action == SyncFileActionUpload {
+		postfix = strings.ReplaceAll(f.syncItem.LocalFile.Path, "\\", "/")
+	}
+	return string(f.syncItem.Action) + postfix
+}
 
-func (f *FileActionTask) DoAction() error {
+func (f *FileActionTask) DoAction(ctx context.Context) error {
+	fmt.Println("\nfile action task")
+	fmt.Println(f.syncItem)
+	if f.syncItem.Action == SyncFileActionUpload {
+		if e := f.uploadFile(); e != nil {
+			return e
+		}
+	}
+	if f.syncItem.Action == SyncFileActionDownload {
+		if e := f.downloadFile(); e != nil {
+			// TODO: retry / cleanup downloading file
+			return e
+		} else {
+			if b, er := utils.PathExists(f.syncItem.getLocalFileFullPath()); er == nil && b {
+				// file existed
+				// remove old local file
+				logger.Verbosef("delete local old file")
+				os.Remove(f.syncItem.getLocalFileFullPath())
+				time.Sleep(200 * time.Millisecond)
+			}
+
+			// rename downloading file into local file
+			os.Rename(f.syncItem.getLocalFileDownloadingFullPath(), f.syncItem.getLocalFileFullPath())
+			time.Sleep(200 * time.Millisecond)
+
+			// change modify time of local file
+			if err := os.Chtimes(f.syncItem.getLocalFileFullPath(), f.syncItem.PanFile.UpdateTime(), f.syncItem.PanFile.UpdateTime()); err != nil {
+				logger.Verbosef(err.Error())
+			}
+			time.Sleep(200 * time.Millisecond)
+
+			// save local file info
+			if file, er := os.Stat(f.syncItem.getLocalFileFullPath()); er == nil {
+				f.localFileDb.Add(&LocalFileItem{
+					FileName:      file.Name(),
+					FileSize:      file.Size(),
+					FileType:      "file",
+					CreatedAt:     file.ModTime().Format("2006-01-02 15:04:05"),
+					UpdatedAt:     file.ModTime().Format("2006-01-02 15:04:05"),
+					FileExtension: path.Ext(file.Name()),
+					Sha1Hash:      f.syncItem.PanFile.Sha1Hash,
+					Path:          f.syncItem.getLocalFileFullPath(),
+				})
+			}
+		}
+	}
 	return nil
 }
 
-func (f *FileActionTask) DownloadFile() error {
+func (f *FileActionTask) downloadFile() error {
+	// check local file existed or not
+	if b, e := utils.PathExists(f.syncItem.getLocalFileFullPath()); e == nil && b {
+		// file existed
+		logger.Verbosef("delete local old file")
+		os.Remove(f.syncItem.getLocalFileFullPath())
+		time.Sleep(200 * time.Millisecond)
+	}
+
+	durl, apierr := f.panClient.GetFileDownloadUrl(&aliyunpan.GetFileDownloadUrlParam{
+		DriveId: f.syncItem.PanFile.DriveId,
+		FileId:  f.syncItem.PanFile.FileId,
+	})
+	time.Sleep(time.Duration(200) * time.Millisecond)
+	if apierr != nil {
+		logger.Verbosef("ERROR: get download url error: %s\n", f.syncItem.PanFile.FileId)
+		return apierr
+	}
+	if durl == nil || durl.Url == "" {
+		logger.Verbosef("无法获取有效的下载链接: %+v\n", durl)
+		f.syncItem.Status = SyncFileStatusFailed
+		f.syncItem.StatusUpdateTime = utils.NowTimeStr()
+		f.syncFileDb.Update(f.syncItem)
+		return fmt.Errorf("无法获取有效的下载链接")
+	}
+	if durl.Url == aliyunpan.IllegalDownloadUrl {
+		logger.Verbosef("无法获取有效的下载链接: %+v\n", durl)
+		f.syncItem.Status = SyncFileStatusIllegal
+		f.syncItem.StatusUpdateTime = utils.NowTimeStr()
+		f.syncFileDb.Update(f.syncItem)
+		return fmt.Errorf("文件非法，无法下载")
+	}
+	localDir := path.Dir(f.syncItem.getLocalFileFullPath())
+	if b, e := utils.PathExists(localDir); e == nil && !b {
+		os.MkdirAll(localDir, 0600)
+		time.Sleep(200 * time.Millisecond)
+	}
+	writer, file, err := downloader.NewDownloaderWriterByFilename(f.syncItem.getLocalFileDownloadingFullPath(), os.O_CREATE|os.O_WRONLY, 0666)
+	if err != nil {
+		return fmt.Errorf("%s, %s", "初始化下载发生错误", err)
+	}
+	defer file.Close()
+	if f.syncItem.PanFile.FileSize == 0 {
+		// zero file
+		f.syncItem.Status = SyncFileStatusSuccess
+		f.syncItem.StatusUpdateTime = utils.NowTimeStr()
+		f.syncFileDb.Update(f.syncItem)
+		return nil
+	}
+
+	worker := downloader.NewWorker(0, f.syncItem.PanFile.DriveId, f.syncItem.PanFile.FileId, durl.Url, writer, nil)
+
+	client := requester.NewHTTPClient()
+	client.SetKeepAlive(true)
+	client.SetTimeout(10 * time.Minute)
+	worker.SetClient(client)
+	worker.SetPanClient(f.panClient)
+
+	writeMu := &sync.Mutex{}
+	worker.SetWriteMutex(writeMu)
+	worker.SetTotalSize(f.syncItem.PanFile.FileSize)
+	worker.SetAcceptRange("bytes")
+	if f.syncItem.DownloadRange == nil {
+		f.syncItem.DownloadRange = &transfer.Range{
+			Begin: 0,
+			End:   f.blockSize,
+		}
+	}
+	worker.SetRange(f.syncItem.DownloadRange) // 分片
+
+	// update status
+	f.syncItem.Status = SyncFileStatusDownloading
+	f.syncItem.StatusUpdateTime = utils.NowTimeStr()
+	f.syncFileDb.Update(f.syncItem)
+
+	for {
+		if f.syncItem.DownloadRange.End > f.syncItem.PanFile.FileSize {
+			f.syncItem.DownloadRange.End = f.syncItem.PanFile.FileSize
+		}
+
+		// 下载分片
+		// TODO: 分片重试策略
+		worker.Execute()
+
+		if worker.GetStatus().StatusCode() == downloader.StatusCodeSuccessed {
+			if f.syncItem.DownloadRange.End == f.syncItem.PanFile.FileSize {
+				// finished
+				f.syncItem.Status = SyncFileStatusSuccess
+				f.syncItem.StatusUpdateTime = utils.NowTimeStr()
+				f.syncFileDb.Update(f.syncItem)
+				break
+			}
+
+			// 下一个分片
+			f.syncItem.DownloadRange.Begin = f.syncItem.DownloadRange.End
+			f.syncItem.DownloadRange.End += f.blockSize
+
+			// 存储状态
+			f.syncFileDb.Update(f.syncItem)
+		}
+	}
+
 	return nil
 }
 
-func (f *FileActionTask) UploadFile() error {
-	return nil
-}
-
-func (f *FileActionTask) DeleteLocalFile() error {
-	return nil
-}
-
-func (f *FileActionTask) DeletePanFile() error {
+func (f *FileActionTask) uploadFile() error {
 	return nil
 }

internal/syncdrive/file_action_task_mgr.go

@@ -1,28 +1,477 @@
 package syncdrive
 
+import (
+	"context"
+	"fmt"
+	mapset "github.com/deckarep/golang-set"
+	"github.com/tickstep/aliyunpan/internal/localfile"
+	"github.com/tickstep/aliyunpan/internal/waitgroup"
+	"github.com/tickstep/aliyunpan/library/collection"
+	"github.com/tickstep/library-go/logger"
+	"path"
+	"strings"
+	"sync"
+	"time"
+)
+
 type (
-	FileActionTaskExecutor struct {
-		localFileDb LocalSyncDb
-		panFileDb   PanSyncDb
-	}
+	FileActionTaskList []*FileActionTask
 
 	FileActionTaskManager struct {
-		FileActionTaskList []*FileActionTask
-		task               *SyncTask
+		mutex *sync.Mutex
+
+		task       *SyncTask
+		wg         *waitgroup.WaitGroup
+		ctx        context.Context
+		cancelFunc context.CancelFunc
+	}
+
+	localFileSet struct {
+		items           LocalFileList
+		localFolderPath string
+	}
+	panFileSet struct {
+		items         PanFileList
+		panFolderPath string
 	}
 )
 
 func NewFileActionTaskManager(task *SyncTask) *FileActionTaskManager {
 	return &FileActionTaskManager{
-		task:               task,
-		FileActionTaskList: []*FileActionTask{},
+		mutex: &sync.Mutex{},
+		task:  task,
 	}
 }
 
-func (f *FileActionTaskManager) StartSync() error {
+func (f *FileActionTaskManager) Start() error {
+	if f.ctx != nil {
+		return fmt.Errorf("task have starting")
+	}
+	f.wg = waitgroup.NewWaitGroup(2)
+
+	var cancel context.CancelFunc
+	f.ctx, cancel = context.WithCancel(context.Background())
+	f.cancelFunc = cancel
+
+	go f.doFileDiffRoutine(f.ctx)
+	go f.fileActionTaskExecutor(f.ctx)
 	return nil
 }
 
-func (f *FileActionTaskManager) StopSync() error {
+func (f *FileActionTaskManager) Stop() error {
+	if f.ctx == nil {
+		return nil
+	}
+	// cancel all sub task & process
+	f.cancelFunc()
+
+	// wait for finished
+	f.wg.Wait()
+
+	f.ctx = nil
+	f.cancelFunc = nil
+
 	return nil
 }
+
+// getPanPathFromLocalPath 通过本地文件路径获取网盘文件的对应路径
+func (f *FileActionTaskManager) getPanPathFromLocalPath(localPath string) string {
+	localPath = strings.ReplaceAll(localPath, "\\", "/")
+	localRootPath := strings.ReplaceAll(f.task.LocalFolderPath, "\\", "/")
+
+	relativePath := strings.TrimPrefix(localPath, localRootPath)
+	return path.Join(path.Clean(f.task.PanFolderPath), relativePath)
+}
+
+// getLocalPathFromPanPath 通过网盘文件路径获取对应的本地文件的对应路径
+func (f *FileActionTaskManager) getLocalPathFromPanPath(panPath string) string {
+	panPath = strings.ReplaceAll(panPath, "\\", "/")
+	panRootPath := strings.ReplaceAll(f.task.PanFolderPath, "\\", "/")
+
+	relativePath := strings.TrimPrefix(panPath, panRootPath)
+	return path.Join(path.Clean(f.task.LocalFolderPath), relativePath)
+}
+
+// doFileDiffRoutine 对比网盘文件和本地文件信息，差异化上传或者下载文件
+func (f *FileActionTaskManager) doFileDiffRoutine(ctx context.Context) {
+	localFolderQueue := collection.NewFifoQueue()
+	panFolderQueue := collection.NewFifoQueue()
+
+	// init root folder
+	if localRootFolder, e := f.task.localFileDb.Get(f.task.LocalFolderPath); e == nil {
+		localFolderQueue.Push(localRootFolder)
+	} else {
+		logger.Verboseln(e)
+		return
+	}
+	if panRootFolder, e := f.task.panFileDb.Get(f.task.PanFolderPath); e == nil {
+		panFolderQueue.Push(panRootFolder)
+	} else {
+		logger.Verboseln(e)
+		return
+	}
+
+	f.wg.AddDelta()
+	defer f.wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			// cancel routine & done
+			logger.Verboseln("file diff routine done")
+			return
+		default:
+			logger.Verboseln("do file diff process")
+			localFiles := LocalFileList{}
+			panFiles := PanFileList{}
+			var err error
+
+			// iterator local folder
+			objLocal := localFolderQueue.Pop()
+			if objLocal != nil {
+				localItem := objLocal.(*LocalFileItem)
+				localFiles, err = f.task.localFileDb.GetFileList(localItem.Path)
+				if err != nil {
+					localFiles = LocalFileList{}
+				}
+				panFiles, err = f.task.panFileDb.GetFileList(f.getPanPathFromLocalPath(localItem.Path))
+				if err != nil {
+					panFiles = PanFileList{}
+				}
+			} else {
+				// iterator pan folder
+				objPan := panFolderQueue.Pop()
+				if objPan != nil {
+					panItem := objPan.(*PanFileItem)
+					panFiles, err = f.task.panFileDb.GetFileList(panItem.Path)
+					if err != nil {
+						panFiles = PanFileList{}
+					}
+					localFiles, err = f.task.localFileDb.GetFileList(f.getLocalPathFromPanPath(panItem.Path))
+					if err != nil {
+						localFiles = LocalFileList{}
+					}
+				}
+			}
+
+			// empty loop
+			if len(panFiles) == 0 && len(localFiles) == 0 {
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+
+			localFilesSet := &localFileSet{
+				items:           localFiles,
+				localFolderPath: f.task.LocalFolderPath,
+			}
+			panFilesSet := &panFileSet{
+				items:         panFiles,
+				panFolderPath: f.task.PanFolderPath,
+			}
+			localFilesNeedToUpload := localFilesSet.Difference(panFilesSet)
+			panFilesNeedToDownload := panFilesSet.Difference(localFilesSet)
+			localFilesNeedToCheck, panFilesNeedToCheck := localFilesSet.Intersection(panFilesSet)
+
+			// download file from pan drive
+			if f.task.Mode != UploadOnly {
+				for _, file := range panFilesNeedToDownload {
+					if file.IsFolder() {
+						panFolderQueue.PushUnique(file)
+						continue
+					}
+					fileActionTask := &FileActionTask{
+						syncItem: &SyncFileItem{
+							Action:           SyncFileActionDownload,
+							Status:           SyncFileStatusCreate,
+							LocalFile:        nil,
+							PanFile:          file,
+							StatusUpdateTime: "",
+							PanFolderPath:    f.task.PanFolderPath,
+							LocalFolderPath:  f.task.LocalFolderPath,
+						},
+					}
+					f.addToQueue(fileActionTask)
+				}
+			}
+
+			// upload file to pan drive
+			if f.task.Mode != DownloadOnly {
+				for _, file := range localFilesNeedToUpload {
+					if file.IsFolder() {
+						localFolderQueue.PushUnique(file)
+						continue
+					}
+					fileActionTask := &FileActionTask{
+						syncItem: &SyncFileItem{
+							Action:           SyncFileActionUpload,
+							Status:           SyncFileStatusCreate,
+							LocalFile:        file,
+							PanFile:          nil,
+							StatusUpdateTime: "",
+							PanFolderPath:    f.task.PanFolderPath,
+							LocalFolderPath:  f.task.LocalFolderPath,
+						},
+					}
+					f.addToQueue(fileActionTask)
+				}
+			}
+
+			// compare file to decide download / upload
+			for idx, _ := range localFilesNeedToCheck {
+				localFile := localFilesNeedToCheck[idx]
+				panFile := panFilesNeedToCheck[idx]
+				if localFile.IsFolder() {
+					localFolderQueue.PushUnique(localFile)
+					continue
+				}
+
+				if localFile.Sha1Hash == "" {
+					// calc sha1
+					fileSum := localfile.NewLocalFileEntity(localFile.Path)
+					fileSum.Sum(localfile.CHECKSUM_SHA1) // block operation
+					localFile.Sha1Hash = fileSum.SHA1
+					fileSum.Close()
+
+					// save sha1
+					f.task.localFileDb.Update(localFile)
+				}
+
+				if strings.ToLower(panFile.Sha1Hash) == strings.ToLower(localFile.Sha1Hash) {
+					// do nothing
+					logger.Verboseln("no need to update file: ", localFile.Path)
+					continue
+				}
+
+				if localFile.UpdateTimeUnix() > panFile.UpdateTimeUnix() { // upload file
+					uploadLocalFile := &FileActionTask{
+						syncItem: &SyncFileItem{
+							Action:           SyncFileActionUpload,
+							Status:           SyncFileStatusCreate,
+							LocalFile:        localFile,
+							PanFile:          nil,
+							StatusUpdateTime: "",
+							PanFolderPath:    f.task.PanFolderPath,
+							LocalFolderPath:  f.task.LocalFolderPath,
+						},
+					}
+					f.addToQueue(uploadLocalFile)
+				} else if localFile.UpdateTimeUnix() < panFile.UpdateTimeUnix() { // download file
+					downloadPanFile := &FileActionTask{
+						syncItem: &SyncFileItem{
+							Action:           SyncFileActionDownload,
+							Status:           SyncFileStatusCreate,
+							LocalFile:        nil,
+							PanFile:          panFile,
+							StatusUpdateTime: "",
+							PanFolderPath:    f.task.PanFolderPath,
+							LocalFolderPath:  f.task.LocalFolderPath,
+						},
+					}
+					f.addToQueue(downloadPanFile)
+				}
+			}
+
+			time.Sleep(100 * time.Millisecond)
+		}
+	}
+}
+
+func (f *FileActionTaskManager) addToQueue(fileTask *FileActionTask) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	if itemInDb, e := f.task.syncFileDb.Get(fileTask.syncItem.Id()); e == nil && itemInDb != nil {
+		if itemInDb.Status == SyncFileStatusCreate || itemInDb.Status == SyncFileStatusDownloading || itemInDb.Status == SyncFileStatusUploading {
+			return
+		}
+		if itemInDb.Status == SyncFileStatusSuccess {
+			if (time.Now().Unix() - itemInDb.StatusUpdateTimeUnix()) < TimeSecondsOf5Minute {
+				// 少于5分钟，不同步，减少同步频次
+				return
+			}
+		}
+	}
+
+	// 进入任务队列
+	f.task.syncFileDb.Add(fileTask.syncItem)
+}
+
+func (f *FileActionTaskManager) getFromQueue(act SyncFileAction) *FileActionTask {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+
+	if act == SyncFileActionDownload {
+		if files, e := f.task.syncFileDb.GetFileList(SyncFileStatusDownloading); e == nil {
+			if len(files) > 0 {
+				return &FileActionTask{
+					localFileDb: f.task.localFileDb,
+					panFileDb:   f.task.panFileDb,
+					syncFileDb:  f.task.syncFileDb,
+					panClient:   f.task.panClient,
+					blockSize:   int64(10485760),
+					syncItem:    files[0],
+				}
+			}
+		}
+	} else if act == SyncFileActionUpload {
+		if files, e := f.task.syncFileDb.GetFileList(SyncFileStatusUploading); e == nil {
+			if len(files) > 0 {
+				return &FileActionTask{
+					localFileDb: f.task.localFileDb,
+					panFileDb:   f.task.panFileDb,
+					syncFileDb:  f.task.syncFileDb,
+					panClient:   f.task.panClient,
+					blockSize:   int64(10485760),
+					syncItem:    files[0],
+				}
+			}
+		}
+	}
+
+	if files, e := f.task.syncFileDb.GetFileList(SyncFileStatusCreate); e == nil {
+		if len(files) > 0 {
+			for _, file := range files {
+				if file.Action == act {
+					return &FileActionTask{
+						localFileDb: f.task.localFileDb,
+						panFileDb:   f.task.panFileDb,
+						syncFileDb:  f.task.syncFileDb,
+						panClient:   f.task.panClient,
+						blockSize:   int64(10485760),
+						syncItem:    file,
+					}
+				}
+			}
+		}
+	}
+
+	// TODO: failed file retry?
+
+	return nil
+}
+
+// fileActionTaskExecutor 异步执行文件操作
+func (f *FileActionTaskManager) fileActionTaskExecutor(ctx context.Context) {
+	f.wg.AddDelta()
+	defer f.wg.Done()
+
+	for {
+		select {
+		case <-ctx.Done():
+			// cancel routine & done
+			logger.Verboseln("file executor routine done")
+			return
+		default:
+			logger.Verboseln("do file executor process")
+
+			// do upload
+			uploadItem := f.getFromQueue(SyncFileActionUpload)
+			if uploadItem != nil {
+				uploadItem.DoAction(ctx)
+			}
+
+			// do download
+			downloadItem := f.getFromQueue(SyncFileActionDownload)
+			if downloadItem != nil {
+				downloadItem.DoAction(ctx)
+			}
+
+			// delay
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
+}
+
+// getRelativePath 获取文件的相对路径
+func (l *localFileSet) getRelativePath(localPath string) string {
+	localPath = strings.ReplaceAll(localPath, "\\", "/")
+	localRootPath := strings.ReplaceAll(l.localFolderPath, "\\", "/")
+	relativePath := strings.TrimPrefix(localPath, localRootPath)
+	return path.Clean(relativePath)
+}
+
+// Intersection 交集
+func (l *localFileSet) Intersection(other *panFileSet) (LocalFileList, PanFileList) {
+	localFilePathSet := mapset.NewThreadUnsafeSet()
+	relativePathLocalMap := map[string]*LocalFileItem{}
+	for _, item := range l.items {
+		rp := l.getRelativePath(item.Path)
+		relativePathLocalMap[rp] = item
+		localFilePathSet.Add(rp)
+	}
+
+	localFileList := LocalFileList{}
+	panFileList := PanFileList{}
+	for _, item := range other.items {
+		rp := other.getRelativePath(item.Path)
+		if localFilePathSet.Contains(rp) {
+			localFileList = append(localFileList, relativePathLocalMap[rp])
+			panFileList = append(panFileList, item)
+		}
+	}
+	return localFileList, panFileList
+}
+
+// Difference 差集
+func (l *localFileSet) Difference(other *panFileSet) LocalFileList {
+	panFilePathSet := mapset.NewThreadUnsafeSet()
+	for _, item := range other.items {
+		rp := other.getRelativePath(item.Path)
+		panFilePathSet.Add(rp)
+	}
+
+	localFileList := LocalFileList{}
+	for _, item := range l.items {
+		rp := l.getRelativePath(item.Path)
+		if !panFilePathSet.Contains(rp) {
+			localFileList = append(localFileList, item)
+		}
+	}
+	return localFileList
+}
+
+// getRelativePath 获取文件的相对路径
+func (p *panFileSet) getRelativePath(panPath string) string {
+	panPath = strings.ReplaceAll(panPath, "\\", "/")
+	panRootPath := strings.ReplaceAll(p.panFolderPath, "\\", "/")
+	relativePath := strings.TrimPrefix(panPath, panRootPath)
+	return path.Clean(relativePath)
+}
+
+// Intersection 交集
+func (p *panFileSet) Intersection(other *localFileSet) (PanFileList, LocalFileList) {
+	localFilePathSet := mapset.NewThreadUnsafeSet()
+	relativePathLocalMap := map[string]*LocalFileItem{}
+	for _, item := range other.items {
+		rp := other.getRelativePath(item.Path)
+		relativePathLocalMap[rp] = item
+		localFilePathSet.Add(rp)
+	}
+
+	localFileList := LocalFileList{}
+	panFileList := PanFileList{}
+	for _, item := range p.items {
+		rp := p.getRelativePath(item.Path)
+		if localFilePathSet.Contains(rp) {
+			localFileList = append(localFileList, relativePathLocalMap[rp])
+			panFileList = append(panFileList, item)
+		}
+	}
+	return panFileList, localFileList
+}
+
+// Difference 差集
+func (p *panFileSet) Difference(other *localFileSet) PanFileList {
+	localFilePathSet := mapset.NewThreadUnsafeSet()
+	for _, item := range other.items {
+		rp := other.getRelativePath(item.Path)
+		localFilePathSet.Add(rp)
+	}
+
+	panFileList := PanFileList{}
+	for _, item := range p.items {
+		rp := p.getRelativePath(item.Path)
+		if !localFilePathSet.Contains(rp) {
+			panFileList = append(panFileList, item)
+		}
+	}
+	return panFileList
+}

internal/syncdrive/file_action_task_mgr_test.go

@@ -0,0 +1,60 @@
+package syncdrive
+
+import (
+	"fmt"
+	"github.com/tickstep/aliyunpan-api/aliyunpan"
+	"github.com/tickstep/aliyunpan/internal/utils"
+	"github.com/tickstep/library-go/logger"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestFileActionMgrStart(t *testing.T) {
+	refreshToken := "ac1010f6396...585338b533bd4ab"
+	webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
+	if err != nil {
+		fmt.Println("get acccess token error")
+		return
+	}
+
+	// pan client
+	panClient := aliyunpan.NewPanClient(*webToken, aliyunpan.AppLoginToken{})
+	user, _ := panClient.GetUserInfo()
+	task := &SyncTask{
+		Id:              "5b2d7c10-e927-4e72-8f9d-5abb3bb04814",
+		DriveId:         user.FileDriveId,
+		LocalFolderPath: "/Volumes/Downloads/dev/folder",
+		PanFolderPath:   "/sync_drive",
+		Mode:            "sync",
+		LastSyncTime:    "2022-05-26 19:29:19",
+
+		syncDbFolderPath: "/Volumes/Downloads/dev/sync_drive",
+		panClient:        panClient,
+	}
+	task.setup()
+
+	ft := NewFileActionTaskManager(task)
+	ft.Start()
+
+	//go func() {
+	//	time.Sleep(10 * time.Second)
+	//	task.Stop()
+	//}()
+	time.Sleep(50 * time.Minute)
+	ft.Stop()
+}
+
+func TestFileTime(t *testing.T) {
+	ts := utils.ParseTimeStr("2013-01-11 12:15:07")
+	//ts = time.Now()
+	if err := os.Chtimes("/Volumes/Downloads/dev/upload/password.key", ts, ts); err != nil {
+		logger.Verbosef(err.Error())
+	}
+}
+
+func TestLocalFileTime(t *testing.T) {
+	if file, er := os.Stat("/Volumes/Downloads/dev/upload/password.key"); er == nil {
+		fmt.Println(file.ModTime().Format("2006-01-02 15:04:05"))
+	}
+}

internal/syncdrive/sync_constants.go

@@ -0,0 +1,12 @@
+package syncdrive
+
+const (
+	// DownloadingFileSuffix 下载中文件后缀
+	DownloadingFileSuffix string = ".aliyunpan"
+
+	// TimeSecondsOfOneMinute 一分钟秒数
+	TimeSecondsOfOneMinute int64 = 60
+
+	// TimeSecondsOf5Minute 5分钟秒数
+	TimeSecondsOf5Minute int64 = 5 * TimeSecondsOfOneMinute
+)

internal/syncdrive/sync_db.go

@@ -3,6 +3,11 @@ package syncdrive
 import (
 	"fmt"
 	"github.com/tickstep/aliyunpan-api/aliyunpan"
+	"github.com/tickstep/aliyunpan/internal/utils"
+	"github.com/tickstep/aliyunpan/library/requester/transfer"
+	"path"
+	"strings"
+	"time"
 )
 
 type (
@@ -99,6 +104,51 @@ type (
 		// Close 关闭数据库
 		Close() (bool, error)
 	}
+
+	SyncFileAction string
+	SyncFileStatus string
+	SyncFileItem   struct {
+		Action    SyncFileAction `json:"action"`
+		Status    SyncFileStatus `json:"status"`
+		LocalFile *LocalFileItem `json:"localFile"`
+		PanFile   *PanFileItem   `json:"panFile"`
+		// LocalFolderPath 本地目录
+		LocalFolderPath string `json:"localFolderPath"`
+		// PanFolderPath 云盘目录
+		PanFolderPath    string          `json:"panFolderPath"`
+		DownloadRange    *transfer.Range `json:"downloadRange"`
+		StatusUpdateTime string          `json:"statusUpdateTime"`
+	}
+	SyncFileList []*SyncFileItem
+
+	SyncFileDb interface {
+		// Open 打开并准备数据库
+		Open() (bool, error)
+		// Add 存储一个数据项
+		Add(item *SyncFileItem) (bool, error)
+		// Get 获取一个数据项
+		Get(id string) (*SyncFileItem, error)
+		// GetFileList 获取文件夹下的所有的文件列表
+		GetFileList(Status SyncFileStatus) (SyncFileList, error)
+		// Delete 删除一个数据项，如果是文件夹，则会删除文件夹下面所有的文件列表
+		Delete(id string) (bool, error)
+		// Update 更新一个数据项数据
+		Update(item *SyncFileItem) (bool, error)
+		// Close 关闭数据库
+		Close() (bool, error)
+	}
+)
+
+const (
+	SyncFileStatusCreate      SyncFileStatus = "create"
+	SyncFileStatusUploading   SyncFileStatus = "uploading"
+	SyncFileStatusDownloading SyncFileStatus = "downloading"
+	SyncFileStatusFailed      SyncFileStatus = "failed"
+	SyncFileStatusSuccess     SyncFileStatus = "success"
+	SyncFileStatusIllegal     SyncFileStatus = "illegal"
+
+	SyncFileActionDownload SyncFileAction = "download"
+	SyncFileActionUpload   SyncFileAction = "upload"
 )
 
 var (
@@ -125,6 +175,12 @@ func NewPanFileItem(fe *aliyunpan.FileEntity) *PanFileItem {
 	}
 }
 
+func (item *PanFileItem) Id() string {
+	sb := &strings.Builder{}
+	fmt.Fprintf(sb, "%s%s", strings.ReplaceAll(item.Path, "\\", "/"), item.UpdatedAt)
+	return utils.Md5Str(sb.String())
+}
+
 func (item *PanFileItem) FormatFileName() string {
 	return item.FileName
 }
@@ -137,10 +193,28 @@ func (item *PanFileItem) IsFolder() bool {
 	return item.FileType == "folder"
 }
 
+func (item *PanFileItem) UpdateTimeUnix() int64 {
+	return item.UpdateTime().Unix()
+}
+
+func (item *PanFileItem) UpdateTime() time.Time {
+	return utils.ParseTimeStr(item.UpdatedAt)
+}
+
+func (item *PanFileItem) HashCode() string {
+	return item.Path
+}
+
 func NewPanSyncDb(dbFilePath string) PanSyncDb {
 	return interface{}(newPanSyncDbBolt(dbFilePath)).(PanSyncDb)
 }
 
+func (item *LocalFileItem) Id() string {
+	sb := &strings.Builder{}
+	fmt.Fprintf(sb, "%s%s", strings.ReplaceAll(item.Path, "\\", "/"), item.UpdatedAt)
+	return utils.Md5Str(sb.String())
+}
+
 func (item *LocalFileItem) FormatFileName() string {
 	return item.FileName
 }
@@ -153,6 +227,82 @@ func (item *LocalFileItem) IsFolder() bool {
 	return item.FileType == "folder"
 }
 
+func (item *LocalFileItem) UpdateTimeUnix() int64 {
+	return item.UpdateTime().Unix()
+}
+
+func (item *LocalFileItem) UpdateTime() time.Time {
+	return utils.ParseTimeStr(item.UpdatedAt)
+}
+
+func (item *LocalFileItem) HashCode() string {
+	return item.Path
+}
+
 func NewLocalSyncDb(dbFilePath string) LocalSyncDb {
 	return interface{}(newLocalSyncDbBolt(dbFilePath)).(LocalSyncDb)
 }
+
+func (l LocalFileList) FindFileByPath(filePath string) *LocalFileItem {
+	for _, item := range l {
+		if filePath == item.Path {
+			return item
+		}
+	}
+	return nil
+}
+
+func (p PanFileList) FindFileByPath(filePath string) *PanFileItem {
+	for _, item := range p {
+		if strings.ReplaceAll(filePath, "\\", "/") == item.Path {
+			return item
+		}
+	}
+	return nil
+}
+
+func (item *SyncFileItem) Id() string {
+	sb := &strings.Builder{}
+	if item.Action == SyncFileActionDownload {
+		fmt.Fprintf(sb, "%s%s", string(item.Action), item.PanFile.Id())
+	} else if item.Action == SyncFileActionUpload {
+		fmt.Fprintf(sb, "%s%s", string(item.Action), item.LocalFile.Id())
+	}
+	return utils.Md5Str(sb.String())
+}
+
+func (item *SyncFileItem) StatusUpdateTimeUnix() int64 {
+	if ts, er := time.Parse("2006-01-02 15:04:05", item.StatusUpdateTime); er != nil {
+		return ts.Unix()
+	}
+	return 0
+}
+
+// getPanFullPath 获取网盘文件的路径
+func (item *SyncFileItem) getPanFileFullPath() string {
+	localPath := item.LocalFile.Path
+	localPath = strings.ReplaceAll(localPath, "\\", "/")
+	localRootPath := strings.ReplaceAll(item.LocalFolderPath, "\\", "/")
+
+	relativePath := strings.TrimPrefix(localPath, localRootPath)
+	return path.Join(path.Clean(item.PanFolderPath), relativePath)
+}
+
+// getLocalFullPath 获取本地文件的路径
+func (item *SyncFileItem) getLocalFileFullPath() string {
+	panPath := item.PanFile.Path
+	panPath = strings.ReplaceAll(panPath, "\\", "/")
+	panRootPath := strings.ReplaceAll(item.PanFolderPath, "\\", "/")
+
+	relativePath := strings.TrimPrefix(panPath, panRootPath)
+	return path.Join(path.Clean(item.LocalFolderPath), relativePath)
+}
+
+// getLocalFileDownloadingFullPath 获取本地文件下载时的路径
+func (item *SyncFileItem) getLocalFileDownloadingFullPath() string {
+	return item.getLocalFileFullPath() + DownloadingFileSuffix
+}
+
+func NewSyncFileDb(dbFilePath string) SyncFileDb {
+	return interface{}(newSyncFileDbBolt(dbFilePath)).(SyncFileDb)
+}

internal/syncdrive/sync_db_bolt.go

@@ -21,10 +21,13 @@ type (
 		db     *BoltDb
 		locker *sync.Mutex
 	}
-)
 
-const (
-	DefaultDirKeyName string = "."
+	// SyncFileDbBolt 存储同步文件状态信息的数据库
+	SyncFileDbBolt struct {
+		Path   string
+		db     *BoltDb
+		locker *sync.Mutex
+	}
 )
 
 func newPanSyncDbBolt(dbFilePath string) *PanSyncDbBolt {
@@ -174,7 +177,7 @@ func (p *PanSyncDbBolt) Update(item *PanFileItem) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	return p.db.Update(item.Path, item.IsFolder(), string(data))
+	return p.db.Update(item.Path, string(data))
 }
 
 // Close 关闭数据库
@@ -327,10 +330,141 @@ func (p *LocalSyncDbBolt) Update(item *LocalFileItem) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	return p.db.Update(item.Path, item.IsFolder(), string(data))
+	return p.db.Update(item.Path, string(data))
 }
 
 // Close 关闭数据库
 func (p *LocalSyncDbBolt) Close() (bool, error) {
 	return true, nil
 }
+
+func newSyncFileDbBolt(dbFilePath string) *SyncFileDbBolt {
+	return &SyncFileDbBolt{
+		Path:   dbFilePath,
+		locker: &sync.Mutex{},
+	}
+}
+
+// Open 打开并准备数据库
+func (s *SyncFileDbBolt) Open() (bool, error) {
+	return true, nil
+}
+
+// Add 存储一个数据项
+func (s *SyncFileDbBolt) Add(item *SyncFileItem) (bool, error) {
+	if item == nil {
+		return false, fmt.Errorf("item is nil")
+	}
+	s.locker.Lock()
+	defer s.locker.Unlock()
+
+	s.db = NewBoltDb(s.Path)
+	s.db.Open()
+	defer s.db.Close()
+
+	data, err := json.Marshal(item)
+	if err != nil {
+		return false, err
+	}
+	return s.db.Add(&BoltItem{
+		FilePath: "/" + item.Id(),
+		IsFolder: false,
+		Data:     string(data),
+	})
+}
+
+// Get 获取一个数据项
+func (s *SyncFileDbBolt) Get(id string) (*SyncFileItem, error) {
+	filePath := "/" + id
+	if filePath == "" {
+		return nil, fmt.Errorf("item is nil")
+	}
+	s.locker.Lock()
+	defer s.locker.Unlock()
+
+	s.db = NewBoltDb(s.Path)
+	s.db.Open()
+	defer s.db.Close()
+
+	data, err := s.db.Get(filePath)
+	if err == nil && data != "" {
+		item := &SyncFileItem{}
+		if err := json.Unmarshal([]byte(data), item); err != nil {
+			return nil, err
+		}
+		return item, nil
+	}
+	return nil, err
+}
+
+// GetFileList 获取文件夹下的所有的文件列表
+func (s *SyncFileDbBolt) GetFileList(Status SyncFileStatus) (SyncFileList, error) {
+	filePath := "/"
+	if filePath == "" {
+		return nil, fmt.Errorf("item is nil")
+	}
+	s.locker.Lock()
+	defer s.locker.Unlock()
+
+	s.db = NewBoltDb(s.Path)
+	s.db.Open()
+	defer s.db.Close()
+
+	panFileList := SyncFileList{}
+	dataList, err := s.db.GetFileList(filePath)
+	if err == nil && len(dataList) > 0 {
+		for _, data := range dataList {
+			if data == "" {
+				continue
+			}
+			item := &SyncFileItem{}
+			if err := json.Unmarshal([]byte(data), item); err != nil {
+				return nil, err
+			}
+			if item.Status == Status {
+				panFileList = append(panFileList, item)
+			}
+		}
+		return panFileList, nil
+	}
+	return nil, err
+}
+
+// Delete 删除一个数据项，如果是文件夹，则会删除文件夹下面所有的文件列表
+func (s *SyncFileDbBolt) Delete(id string) (bool, error) {
+	filePath := "/" + id
+	if filePath == "" {
+		return false, fmt.Errorf("item is nil")
+	}
+	s.locker.Lock()
+	defer s.locker.Unlock()
+
+	s.db = NewBoltDb(s.Path)
+	s.db.Open()
+	defer s.db.Close()
+	return s.db.Delete(filePath)
+}
+
+// Update 更新一个数据项数据
+func (s *SyncFileDbBolt) Update(item *SyncFileItem) (bool, error) {
+	if item == nil {
+		return false, fmt.Errorf("item is nil")
+	}
+	s.locker.Lock()
+	defer s.locker.Unlock()
+
+	s.db = NewBoltDb(s.Path)
+	s.db.Open()
+	defer s.db.Close()
+
+	data, err := json.Marshal(item)
+	if err != nil {
+		return false, err
+	}
+	return s.db.Update("/"+item.Id(), string(data))
+}
+
+// Close 关闭数据库
+func (s *SyncFileDbBolt) Close() (bool, error) {
+	return true, nil
+}

internal/syncdrive/sync_db_test.go

@@ -197,3 +197,118 @@ func TestLocalGetFileList(t *testing.T) {
 
 	fmt.Println(b.GetFileList("D:/smb/feny/goprojects/dl/a761171495"))
 }
+
+func TestSyncDbAdd(t *testing.T) {
+	b := NewSyncFileDb("D:\\smb\\feny\\goprojects\\dev\\sync_drive\\sync.db")
+	b.Open()
+	defer b.Close()
+
+	b.Add(&SyncFileItem{
+		Action:    SyncFileActionDownload,
+		Status:    SyncFileStatusCreate,
+		LocalFile: nil,
+		PanFile: &PanFileItem{
+			Category:      "others",
+			Crc64Hash:     "16173291050517323365",
+			CreatedAt:     "2021-07-09 23:17:52",
+			DomainId:      "bj29",
+			DriveId:       "19519221",
+			FileExtension: "apk",
+			FileId:        "60e868a0a005315fe2b149b4ade47b2df8bdccee",
+			FileName:      "10000996@yunpan-release.apk",
+			FileSize:      61029089,
+			FileType:      "file",
+			ParentFileId:  "60f29e3ac420ae8c08e645db8b80b881e2a3633c",
+			Path:          "/sync_drive/10000996@yunpan-release.apk",
+			Sha1Hash:      "05A7597967C3C947D84564A7D55309DEA56ED985",
+			UpdatedAt:     "2021-07-21 17:05:49",
+			UploadId:      "rapid-9141d56d-21c3-4fcc-b41d-5df7bf47540f",
+		},
+		StatusUpdateTime: "",
+	})
+	b.Add(&SyncFileItem{
+		Action:    SyncFileActionDownload,
+		Status:    SyncFileStatusCreate,
+		LocalFile: nil,
+		PanFile: &PanFileItem{
+			Category:      "others",
+			Crc64Hash:     "16173291050517323365",
+			CreatedAt:     "2021-07-09 23:17:52",
+			DomainId:      "bj29",
+			DriveId:       "19519221",
+			FileExtension: "apk",
+			FileId:        "60e868a0a005315fe2b149b4ade47b2df8bdccee",
+			FileName:      "10000996@yunpan-release-1.apk",
+			FileSize:      61029089,
+			FileType:      "file",
+			ParentFileId:  "60f29e3ac420ae8c08e645db8b80b881e2a3633c",
+			Path:          "/sync_drive/10000996@yunpan-release-1.apk",
+			Sha1Hash:      "05A7597967C3C947D84564A7D55309DEA56ED985",
+			UpdatedAt:     "2021-07-21 17:05:49",
+			UploadId:      "rapid-9141d56d-21c3-4fcc-b41d-5df7bf47540f",
+		},
+		StatusUpdateTime: "",
+	})
+	b.Add(&SyncFileItem{
+		Action: SyncFileActionUpload,
+		Status: SyncFileStatusCreate,
+		LocalFile: &LocalFileItem{
+			FileName:      "f1.txt",
+			FileSize:      1542,
+			FileType:      "file",
+			CreatedAt:     "2020-12-12 12:51:12",
+			UpdatedAt:     "2020-12-12 12:51:12",
+			FileExtension: ".txt",
+			Sha1Hash:      "",
+			Path:          "D:\\smb\\feny\\goprojects\\dev\\fo\\f1.txt",
+		},
+		PanFile:          nil,
+		StatusUpdateTime: "",
+	})
+	//b.AddUnique(&SyncFileItem{
+	//	Action:    SyncFileActionDownload,
+	//	Status:    SyncFileStatusCreate,
+	//	LocalFile: nil,
+	//	PanFile: &PanFileItem{
+	//		Category:      "others",
+	//		Crc64Hash:     "16173291050517323365",
+	//		CreatedAt:     "2021-07-09 23:17:52",
+	//		DomainId:      "bj29",
+	//		DriveId:       "19519221",
+	//		FileExtension: "apk",
+	//		FileId:        "60e868a0a005315fe2b149b4ade47b2df8bdccee",
+	//		FileName:      "10000996@yunpan-release-1.apk",
+	//		FileSize:      61029089,
+	//		FileType:      "file",
+	//		ParentFileId:  "60f29e3ac420ae8c08e645db8b80b881e2a3633c",
+	//		Path:          "/sync_drive/10000996@yunpan-release-1.apk",
+	//		Sha1Hash:      "05A7597967C3C947D84564A7D55309DEA56ED985",
+	//		UpdatedAt:     "2021-07-21 17:05:49",
+	//		UploadId:      "rapid-9141d56d-21c3-4fcc-b41d-5df7bf47540f",
+	//	},
+	//	StatusUpdateTime: "",
+	//})
+	//b.AddUnique(&SyncFileItem{
+	//	Action:    SyncFileActionDownload,
+	//	Status:    SyncFileStatusCreate,
+	//	LocalFile: nil,
+	//	PanFile: &PanFileItem{
+	//		Category:      "others",
+	//		Crc64Hash:     "16173291050517323365",
+	//		CreatedAt:     "2021-07-09 23:17:52",
+	//		DomainId:      "bj29",
+	//		DriveId:       "19519221",
+	//		FileExtension: "apk",
+	//		FileId:        "60e868a0a005315fe2b149b4ade47b2df8bdccee",
+	//		FileName:      "10000996@yunpan-release-2.apk",
+	//		FileSize:      61029089,
+	//		FileType:      "file",
+	//		ParentFileId:  "60f29e3ac420ae8c08e645db8b80b881e2a3633c",
+	//		Path:          "/sync_drive/10000996@yunpan-release-2.apk",
+	//		Sha1Hash:      "05A7597967C3C947D84564A7D55309DEA56ED985",
+	//		UpdatedAt:     "2021-07-21 17:05:49",
+	//		UploadId:      "rapid-9141d56d-21c3-4fcc-b41d-5df7bf47540f",
+	//	},
+	//	StatusUpdateTime: "",
+	//})
+}

internal/syncdrive/sync_task.go

@@ -38,12 +38,15 @@ type (
 		syncDbFolderPath string
 		localFileDb      LocalSyncDb
 		panFileDb        PanSyncDb
+		syncFileDb       SyncFileDb
 
 		wg         *waitgroup.WaitGroup
 		ctx        context.Context
 		cancelFunc context.CancelFunc
 
 		panClient *aliyunpan.PanClient
+
+		fileActionTaskManager *FileActionTaskManager
 	}
 )
 
@@ -76,19 +79,40 @@ func (t *SyncTask) String() string {
 	return builder.String()
 }
 
+func (t *SyncTask) setup() error {
+	t.localFileDb = NewLocalSyncDb(t.localSyncDbFullPath())
+	t.panFileDb = NewPanSyncDb(t.panSyncDbFullPath())
+	t.syncFileDb = NewSyncFileDb(t.syncFileDbFullPath())
+	if _, e := t.localFileDb.Open(); e != nil {
+		return e
+	}
+	if _, e := t.panFileDb.Open(); e != nil {
+		return e
+	}
+	return nil
+}
+
 // Start 启动同步任务
 func (t *SyncTask) Start() error {
 	if t.ctx != nil {
 		return fmt.Errorf("task have starting")
 	}
 	t.localFileDb = NewLocalSyncDb(t.localSyncDbFullPath())
-	t.panFileDb = newPanSyncDbBolt(t.panSyncDbFullPath())
+	t.panFileDb = NewPanSyncDb(t.panSyncDbFullPath())
+	t.syncFileDb = NewSyncFileDb(t.syncFileDbFullPath())
 	if _, e := t.localFileDb.Open(); e != nil {
 		return e
 	}
 	if _, e := t.panFileDb.Open(); e != nil {
 		return e
 	}
+	if _, e := t.syncFileDb.Open(); e != nil {
+		return e
+	}
+
+	if t.fileActionTaskManager == nil {
+		t.fileActionTaskManager = NewFileActionTaskManager(t)
+	}
 
 	t.wg = waitgroup.NewWaitGroup(2)
 
@@ -98,6 +122,8 @@ func (t *SyncTask) Start() error {
 
 	go t.scanLocalFile(t.ctx)
 	go t.scanPanFile(t.ctx)
+
+	//t.fileActionTaskManager.Start()
 	return nil
 }
 
@@ -115,6 +141,9 @@ func (t *SyncTask) Stop() error {
 	t.ctx = nil
 	t.cancelFunc = nil
 
+	// stop file action task (block routine)
+	t.fileActionTaskManager.Stop()
+
 	// release resources
 	if t.localFileDb != nil {
 		t.localFileDb.Close()
@@ -122,6 +151,9 @@ func (t *SyncTask) Stop() error {
 	if t.panFileDb != nil {
 		t.panFileDb.Close()
 	}
+	if t.syncFileDb != nil {
+		t.syncFileDb.Close()
+	}
 
 	// record the sync time
 	t.LastSyncTime = utils.NowTimeStr()
@@ -146,6 +178,15 @@ func (t *SyncTask) localSyncDbFullPath() string {
 	return path.Join(dir, "local.bolt")
 }
 
+// syncFileDbFullPath 同步文件过程数据库
+func (t *SyncTask) syncFileDbFullPath() string {
+	dir := path.Join(t.syncDbFolderPath, t.Id)
+	if b, _ := utils.PathExists(dir); !b {
+		os.MkdirAll(dir, 0600)
+	}
+	return path.Join(dir, "sync.bolt")
+}
+
 func newLocalFileItem(file os.FileInfo, fullPath string) *LocalFileItem {
 	ft := "file"
 	if file.IsDir() {
@@ -227,6 +268,11 @@ func (t *SyncTask) scanLocalFile(ctx context.Context) {
 			}
 			localFileAppendList := LocalFileList{}
 			for _, file := range files {
+				if strings.HasSuffix(file.Name(), DownloadingFileSuffix) {
+					// 下载中文件，跳过
+					continue
+				}
+
 				localFile := newLocalFileItem(file, item.path+"/"+file.Name())
 				localFileInDb, _ := t.localFileDb.Get(localFile.Path)
 				if localFileInDb == nil {
@@ -234,6 +280,9 @@ func (t *SyncTask) scanLocalFile(ctx context.Context) {
 					localFileAppendList = append(localFileAppendList, localFile)
 				} else {
 					// update newest info into DB
+					if localFile.UpdateTimeUnix() > localFileInDb.UpdateTimeUnix() {
+						localFileInDb.Sha1Hash = ""
+					}
 					localFileInDb.UpdatedAt = localFile.UpdatedAt
 					localFileInDb.CreatedAt = localFile.CreatedAt
 					localFileInDb.FileSize = localFile.FileSize

internal/syncdrive/sync_task_mgr.go

@@ -112,6 +112,8 @@ func (m *SyncTaskManager) Start() (bool, error) {
 	}
 	// save config file
 	ioutil.WriteFile(m.configFilePath(), []byte(utils.ObjectToJsonStr(m.syncDriveConfig, true)), 0600)
+
+	// TODO: refresh panClient token to keep access token alive
 	return true, nil
 }
 

internal/syncdrive/sync_task_mgr_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestStart(t *testing.T) {
-	refreshToken := "927e2a3da30646d8a787e7f11c0fdf1f"
+	refreshToken := "ac1010f63...9585338b533bd4ab"
 	webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
 	if err != nil {
 		fmt.Println("get acccess token error")
@@ -26,6 +26,6 @@ func TestStart(t *testing.T) {
 	)
 
 	manager.Start()
-	time.Sleep(1 * time.Minute)
+	time.Sleep(5 * time.Minute)
 	manager.Stop()
 }

internal/syncdrive/sync_task_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestSyncTask(t *testing.T) {
-	refreshToken := "84c6499b7c9344109a2fe4f6733c3afd"
+	refreshToken := "84c6499b7...9a2fe4f6733c3afd"
 	webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
 	if err != nil {
 		fmt.Println("get acccess token error")
@@ -19,9 +19,9 @@ func TestSyncTask(t *testing.T) {
 	panClient := aliyunpan.NewPanClient(*webToken, aliyunpan.AppLoginToken{})
 	user, _ := panClient.GetUserInfo()
 	task := SyncTask{
-		Id:              "840f28af799747848c0b3155e0bdfeab",
+		Id:              "5b2d7c10-e927-4e72-8f9d-5abb3bb04814",
 		DriveId:         user.FileDriveId,
-		LocalFolderPath: "D:\\smb\\feny\\goprojects\\dev\\upload",
+		LocalFolderPath: "D:\\smb\\feny\\goprojects\\dev\\NS游戏备份",
 		PanFolderPath:   "/sync_drive",
 		Mode:            "sync",
 		LastSyncTime:    "",

internal/utils/utils.go

@@ -15,6 +15,7 @@ package utils
 
 import (
 	"compress/gzip"
+	"crypto/md5"
 	"flag"
 	"fmt"
 	jsoniter "github.com/json-iterator/go"
@@ -202,3 +203,22 @@ func UuidStr() string {
 func NowTimeStr() string {
 	return time.Now().Format("2006-01-02 15:04:05")
 }
+
+// ParseTimeStr 反解析时间字符串
+func ParseTimeStr(value string) time.Time {
+	cz := time.FixedZone("CST", 8*3600) // 东8区
+	if t, e := time.ParseInLocation("2006-01-02 15:04:05", value, cz); e == nil {
+		return t
+	}
+	return time.Now()
+}
+
+// Md5Str MD5哈希计算
+func Md5Str(text string) string {
+	h := md5.New()
+	h.Write([]byte(text))
+	re := h.Sum(nil)
+	sb := &strings.Builder{}
+	fmt.Fprintf(sb, "%x\n", re)
+	return strings.ToLower(sb.String())
+}

internal/utils/utils_test.go

@@ -23,3 +23,7 @@ func TestConvertTime(t *testing.T) {
 func TestUuidStr(t *testing.T) {
 	fmt.Println(UuidStr())
 }
+
+func TestMd5Str(t *testing.T) {
+	fmt.Println(Md5Str("123456"))
+}

library/collection/queue.go

@@ -1,6 +1,12 @@
 package collection
 
-import "sync"
+import (
+	"sync"
+)
+
+type QueueItem interface {
+	HashCode() string
+}
 
 type Queue struct {
 	queueList []interface{}
@@ -21,6 +27,21 @@ func (q *Queue) Push(item interface{}) {
 	q.queueList = append(q.queueList, item)
 }
 
+func (q *Queue) PushUnique(item interface{}) {
+	q.mutex.Lock()
+	defer q.mutex.Unlock()
+	if q.queueList == nil {
+		q.queueList = []interface{}{}
+	} else {
+		for _, qItem := range q.queueList {
+			if item.(QueueItem).HashCode() == qItem.(QueueItem).HashCode() {
+				return
+			}
+		}
+	}
+	q.queueList = append(q.queueList, item)
+}
+
 func (q *Queue) Pop() interface{} {
 	q.mutex.Lock()
 	defer q.mutex.Unlock()