refactor sync drive library

This commit is contained in:
xiaoyaofenfen
2022-05-13 15:34:44 +08:00
parent 01b9929c4b
commit 24765ba2af
5 changed files with 481 additions and 294 deletions
+315
View File
@@ -0,0 +1,315 @@
package syncdrive
import (
"fmt"
"github.com/tickstep/bolt"
"path"
"strings"
"sync"
"time"
)
type (
// BoltDb 存储本地文件信息的数据库
BoltDb struct {
Path string
db *bolt.DB
locker *sync.Mutex
}
)
func NewBoltDb(dbFilePath string) *BoltDb {
return &BoltDb{
Path: dbFilePath,
locker: &sync.Mutex{},
}
}
func (p *BoltDb) Open() (bool, error) {
db, err := bolt.Open(p.Path, 0600, &bolt.Options{Timeout: 5 * time.Second})
if err != nil {
return false, err
}
p.db = db
return true, nil
}
// Add 增加一个数据项
func (p *BoltDb) Add(filePath string, isFolder bool, data string) (bool, error) {
filePath = FormatFilePath(filePath)
p.locker.Lock()
defer p.locker.Unlock()
// add item
// Start a writable transaction.
tx, err := p.db.Begin(true)
if err != nil {
return false, err
}
defer tx.Rollback()
parts := strings.Split(filePath, "/")
bkt, er := tx.CreateBucketIfNotExists([]byte("/"))
if er != nil {
return false, er
}
for _, p := range parts[:len(parts)-1] {
if p == "" {
continue
}
bkt, _ = bkt.CreateBucketIfNotExists([]byte(p))
}
if bkt == nil {
return false, fmt.Errorf("create or get bucket error")
}
fileName := path.Base(filePath)
if isFolder {
bkt, err = bkt.CreateBucketIfNotExists([]byte(fileName))
if err != nil {
return false, err
}
if e := bkt.Put([]byte(DefaultDirKeyName), []byte(data)); e != nil {
return false, e
}
} else {
if e := bkt.Put([]byte(fileName), []byte(data)); e != nil {
return false, e
}
}
// Commit the transaction and check for error.
if err := tx.Commit(); err != nil {
return false, err
}
return true, nil
}
// Get 获取一个数据项,数据项不存在返回错误
func (p *BoltDb) Get(filePath string) (string, error) {
filePath = FormatFilePath(filePath)
if filePath == "" {
return "", fmt.Errorf("item is nil")
}
p.locker.Lock()
defer p.locker.Unlock()
tx, err := p.db.Begin(false)
if err != nil {
return "", err
}
partsOrg := strings.Split(filePath, "/")
parts := []string{}
for _, p := range partsOrg {
if p == "" {
continue
}
parts = append(parts, p)
}
bkt := tx.Bucket([]byte("/"))
if bkt == nil {
return "", ErrItemNotExisted
}
for _, p := range parts[:len(parts)-1] {
bkt = bkt.Bucket([]byte(p))
if bkt == nil {
return "", ErrItemNotExisted
}
}
if bkt == nil {
return "", ErrItemNotExisted
}
dirBucket := bkt.Bucket([]byte(parts[len(parts)-1]))
var data []byte
if dirBucket != nil {
// is dir
data = dirBucket.Get([]byte(DefaultDirKeyName))
} else {
data = bkt.Get([]byte(parts[len(parts)-1]))
}
if data != nil {
return string(data), nil
}
return "", ErrItemNotExisted
}
func (p *BoltDb) GetFileList(filePath string) ([]string, error) {
dataList := []string{}
filePath = FormatFilePath(filePath)
if filePath == "" {
return dataList, fmt.Errorf("item is nil")
}
p.locker.Lock()
defer p.locker.Unlock()
tx, err := p.db.Begin(false)
if err != nil {
return dataList, err
}
partsOrg := strings.Split(filePath, "/")
parts := []string{}
for _, p := range partsOrg {
if p == "" {
continue
}
parts = append(parts, p)
}
bkt := tx.Bucket([]byte("/"))
if bkt == nil {
return dataList, ErrItemNotExisted
}
for _, p := range parts[:len(parts)-1] {
bkt = bkt.Bucket([]byte(p))
if bkt == nil {
return dataList, ErrItemNotExisted
}
}
if bkt == nil {
return dataList, ErrItemNotExisted
}
dirBucket := bkt.Bucket([]byte(parts[len(parts)-1]))
if dirBucket == nil {
return dataList, ErrItemNotExisted
}
c := dirBucket.Cursor()
for k, v := c.First(); k != nil; k, v = c.Next() {
if string(k) == DefaultDirKeyName {
continue
}
dataList = append(dataList, string(v))
}
return dataList, nil
}
// Delete 删除一个数据项
func (p *BoltDb) Delete(filePath string) (bool, error) {
filePath = FormatFilePath(filePath)
if filePath == "" {
return false, fmt.Errorf("item is nil")
}
p.locker.Lock()
defer p.locker.Unlock()
tx, err := p.db.Begin(true)
if err != nil {
return false, err
}
defer tx.Rollback()
partsOrg := strings.Split(filePath, "/")
parts := []string{}
for _, p := range partsOrg {
if p == "" {
continue
}
parts = append(parts, p)
}
// get parent node
bkt := tx.Bucket([]byte("/"))
if bkt == nil {
return false, ErrItemNotExisted
}
for _, p := range parts[:len(parts)-1] {
bkt = bkt.Bucket([]byte(p))
if bkt == nil {
return false, ErrItemNotExisted
}
}
if bkt == nil {
return false, ErrItemNotExisted
}
targetName := []byte(parts[len(parts)-1])
dirBucket := bkt.Bucket(targetName)
var er error
if dirBucket != nil {
// is dir, delete bucket
er = bkt.DeleteBucket(targetName)
} else {
// is file, delete item
er = bkt.Delete(targetName)
}
// Commit the transaction and check for error.
if err := tx.Commit(); err != nil {
return false, err
}
return er == nil, nil
}
// Update 更新数据项,数据项不存在返回错误
func (p *BoltDb) Update(filePath string, isFolder bool, data string) (bool, error) {
filePath = FormatFilePath(filePath)
p.locker.Lock()
defer p.locker.Unlock()
// update item
// Start a writable transaction.
tx, err := p.db.Begin(true)
if err != nil {
return false, err
}
defer tx.Rollback()
// get parent bucket of update item
parts := strings.Split(filePath, "/")
bkt := tx.Bucket([]byte("/"))
if bkt == nil {
return false, ErrItemNotExisted
}
for _, p := range parts[:len(parts)-1] {
if p == "" {
continue
}
bkt = bkt.Bucket([]byte(p))
if bkt == nil {
return false, ErrItemNotExisted
}
}
if bkt == nil {
return false, ErrItemNotExisted
}
// update content
fileName := path.Base(filePath)
if isFolder {
bkt = bkt.Bucket([]byte(fileName))
if bkt == nil {
return false, ErrItemNotExisted
}
if e := bkt.Put([]byte(DefaultDirKeyName), []byte(data)); e != nil {
return false, e
}
} else {
// is file
if e := bkt.Put([]byte(fileName), []byte(data)); e != nil {
return false, e
}
}
// Commit the transaction and check for error.
if err := tx.Commit(); err != nil {
return false, err
}
return true, nil
}
// Close 关闭数据库
func (p *BoltDb) Close() (bool, error) {
p.locker.Lock()
defer p.locker.Unlock()
if p.db != nil {
if e := p.db.Close(); e != nil {
return false, e
}
}
return true, nil
}
+11
View File
@@ -0,0 +1,11 @@
package syncdrive
import (
"fmt"
"path"
"testing"
)
func TestPath(t *testing.T) {
fmt.Println(path.Dir("/dsgsdghsd/dsgsdh/eg34/dghsh"))
}
+96 -15
View File
@@ -1,23 +1,104 @@
package syncdrive
type (
//SyncItem interface {
// FileName() string
// FilePath() string
// IsDir() bool
//}
SyncItem struct {
FileName string
FilePath string
IsDir bool
}
import (
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
)
SyncDb interface {
type (
// PanFileItem 网盘文件信息
PanFileItem struct {
// 网盘ID
DriveId string `json:"driveId"`
// 域ID
DomainId string `json:"domainId"`
// FileId 文件ID
FileId string `json:"fileId"`
// FileName 文件名
FileName string `json:"fileName"`
// FileSize 文件大小
FileSize int64 `json:"fileSize"`
// 文件类别 folder / file
FileType string `json:"fileType"`
// 创建时间
CreatedAt string `json:"createdAt"`
// 最后修改时间
UpdatedAt string `json:"updatedAt"`
// 后缀名,例如:dmg
FileExtension string `json:"fileExtension"`
// 文件上传ID
UploadId string `json:"uploadId"`
// 父文件夹ID
ParentFileId string `json:"parentFileId"`
// 内容CRC64校验值,只有文件才会有
Crc64Hash string `json:"crc64Hash"`
// 内容Hash值,只有文件才会有
Sha1Hash string `json:"sha1Hash"`
// FilePath 文件的完整路径
Path string `json:"path"`
// Category 文件分类,例如:image/video/doc/others
Category string `json:"category"`
}
PanFileList []*PanFileItem
PanSyncDb interface {
// Open 打开并准备数据库
Open() (bool, error)
Add(item *SyncItem) (bool, error)
Get(filePath string) (*SyncItem, error)
// Add 存储一个数据项
Add(item *PanFileItem) (bool, error)
// Get 获取一个数据项
Get(filePath string) (*PanFileItem, error)
// GetFileList 获取文件夹下的所有的文件列表
GetFileList(filePath string) (PanFileList, error)
// Delete 删除一个数据项,如果是文件夹,则会删除文件夹下面所有的文件列表
Delete(filePath string) (bool, error)
Update(filePath string) (bool, error)
// Update 更新一个数据项数据
Update(item *PanFileItem) (bool, error)
// Close 关闭数据库
Close() (bool, error)
}
// LocalFileItem 本地文件信息
LocalFileItem struct {
}
)
var (
ErrItemNotExisted error = fmt.Errorf("item is not existed")
)
func NewPanFileItem(fe *aliyunpan.FileEntity) *PanFileItem {
return &PanFileItem{
DriveId: fe.DriveId,
DomainId: fe.DomainId,
FileId: fe.FileId,
FileName: fe.FileName,
FileSize: fe.FileSize,
FileType: fe.FileType,
CreatedAt: fe.CreatedAt,
UpdatedAt: fe.UpdatedAt,
FileExtension: fe.FileExtension,
UploadId: fe.UploadId,
ParentFileId: fe.ParentFileId,
Crc64Hash: fe.Crc64Hash,
Sha1Hash: fe.ContentHash,
Path: fe.Path,
Category: fe.Category,
}
}
func (item *PanFileItem) FormatFileName() string {
return item.FileName
}
func (item *PanFileItem) FormatFilePath() string {
return FormatFilePath(item.Path)
}
func (item *PanFileItem) IsFolder() bool {
return item.FileType == "folder"
}
func NewPanSyncDb(dbFilePath string) PanSyncDb {
return interface{}(newPanSyncDbBolt(dbFilePath)).(PanSyncDb)
}
+50 -278
View File
@@ -3,61 +3,21 @@ package syncdrive
import (
"encoding/json"
"fmt"
"github.com/tickstep/aliyunpan-api/aliyunpan"
"github.com/tickstep/bolt"
"strings"
"sync"
"time"
)
type (
// PanFileItem 网盘文件信息
PanFileItem struct {
// 网盘ID
DriveId string `json:"driveId"`
// 域ID
DomainId string `json:"domainId"`
// FileId 文件ID
FileId string `json:"fileId"`
// FileName 文件名
FileName string `json:"fileName"`
// FileSize 文件大小
FileSize int64 `json:"fileSize"`
// 文件类别 folder / file
FileType string `json:"fileType"`
// 创建时间
CreatedAt string `json:"createdAt"`
// 最后修改时间
UpdatedAt string `json:"updatedAt"`
// 后缀名,例如:dmg
FileExtension string `json:"fileExtension"`
// 文件上传ID
UploadId string `json:"uploadId"`
// 父文件夹ID
ParentFileId string `json:"parentFileId"`
// 内容CRC64校验值,只有文件才会有
Crc64Hash string `json:"crc64Hash"`
// 内容Hash值,只有文件才会有
Sha1Hash string `json:"sha1Hash"`
// FilePath 文件的完整路径
Path string `json:"path"`
// Category 文件分类,例如:image/video/doc/others
Category string `json:"category"`
}
// PanSyncDb 存储网盘文件信息的数据库
PanSyncDb struct {
// PanSyncDbBolt 存储网盘文件信息的数据库
PanSyncDbBolt struct {
Path string
db *bolt.DB
db *BoltDb
locker *sync.Mutex
}
// LocalFileItem 本地文件信息
LocalFileItem struct {
}
// LocalSyncDb 存储本地文件信息的数据库
LocalSyncDb struct {
// LocalSyncDbBolt 存储本地文件信息的数据库
LocalSyncDbBolt struct {
Path string
db *bolt.DB
locker *sync.Mutex
@@ -68,117 +28,35 @@ const (
DefaultDirKeyName string = "."
)
var (
ErrItemNotExisted error = fmt.Errorf("item is not existed")
)
func NewPanFileItem(fe *aliyunpan.FileEntity) *PanFileItem {
return &PanFileItem{
DriveId: fe.DriveId,
DomainId: fe.DomainId,
FileId: fe.FileId,
FileName: fe.FileName,
FileSize: fe.FileSize,
FileType: fe.FileType,
CreatedAt: fe.CreatedAt,
UpdatedAt: fe.UpdatedAt,
FileExtension: fe.FileExtension,
UploadId: fe.UploadId,
ParentFileId: fe.ParentFileId,
Crc64Hash: fe.Crc64Hash,
Sha1Hash: fe.ContentHash,
Path: fe.Path,
Category: fe.Category,
}
}
func (item *PanFileItem) FormatFileName() string {
return item.FileName
}
func (item *PanFileItem) FormatFilePath() string {
return FormatFilePath(item.Path)
}
func (item *PanFileItem) IsFolder() bool {
return item.FileType == "folder"
}
func NewPanSyncDb(dbFilePath string) *PanSyncDb {
return &PanSyncDb{
func newPanSyncDbBolt(dbFilePath string) *PanSyncDbBolt {
return &PanSyncDbBolt{
Path: dbFilePath,
locker: &sync.Mutex{},
}
}
func (p *PanSyncDb) Open() (bool, error) {
db, err := bolt.Open(p.Path, 0600, &bolt.Options{Timeout: 5 * time.Second})
if err != nil {
return false, err
}
p.db = db
return true, nil
func (p *PanSyncDbBolt) Open() (bool, error) {
p.db = NewBoltDb(p.Path)
return p.db.Open()
}
// Add 增加一个数据项
func (p *PanSyncDb) Add(item *PanFileItem) (bool, error) {
func (p *PanSyncDbBolt) Add(item *PanFileItem) (bool, error) {
if item == nil {
return false, fmt.Errorf("item is nil")
}
p.locker.Lock()
defer p.locker.Unlock()
// add item
// Start a writable transaction.
tx, err := p.db.Begin(true)
data, err := json.Marshal(item)
if err != nil {
return false, err
}
defer tx.Rollback()
parts := strings.Split(item.FormatFilePath(), "/")
bkt, er := tx.CreateBucketIfNotExists([]byte("/"))
if er != nil {
return false, er
}
for _, p := range parts[:len(parts)-1] {
if p == "" {
continue
}
bkt, _ = bkt.CreateBucketIfNotExists([]byte(p))
}
if bkt == nil {
return false, fmt.Errorf("create or get bucket error")
}
rs, err := json.Marshal(item)
if err != nil {
return false, err
}
if item.IsFolder() {
bkt, err = bkt.CreateBucketIfNotExists([]byte(item.FormatFileName()))
if err != nil {
return false, err
}
if e := bkt.Put([]byte(DefaultDirKeyName), rs); e != nil {
return false, e
}
} else {
if e := bkt.Put([]byte(item.FormatFileName()), rs); e != nil {
return false, e
}
}
// Commit the transaction and check for error.
if err := tx.Commit(); err != nil {
return false, err
}
return true, nil
return p.db.Add(item.Path, item.IsFolder(), string(data))
}
// Get 获取一个数据项,数据项不存在返回错误
func (p *PanSyncDb) Get(filePath string) (*PanFileItem, error) {
func (p *PanSyncDbBolt) Get(filePath string) (*PanFileItem, error) {
filePath = FormatFilePath(filePath)
if filePath == "" {
return nil, fmt.Errorf("item is nil")
@@ -186,54 +64,42 @@ func (p *PanSyncDb) Get(filePath string) (*PanFileItem, error) {
p.locker.Lock()
defer p.locker.Unlock()
tx, err := p.db.Begin(false)
if err != nil {
return nil, err
}
partsOrg := strings.Split(filePath, "/")
parts := []string{}
for _, p := range partsOrg {
if p == "" {
continue
}
parts = append(parts, p)
}
bkt := tx.Bucket([]byte("/"))
if bkt == nil {
return nil, ErrItemNotExisted
}
for _, p := range parts[:len(parts)-1] {
bkt = bkt.Bucket([]byte(p))
if bkt == nil {
return nil, ErrItemNotExisted
}
}
if bkt == nil {
return nil, ErrItemNotExisted
}
dirBucket := bkt.Bucket([]byte(parts[len(parts)-1]))
var data []byte
if dirBucket != nil {
// is dir
data = dirBucket.Get([]byte(DefaultDirKeyName))
} else {
data = bkt.Get([]byte(parts[len(parts)-1]))
}
if data != nil {
data, err := p.db.Get(filePath)
if err == nil && data != "" {
item := &PanFileItem{}
if err := json.Unmarshal(data, item); err != nil {
if err := json.Unmarshal([]byte(data), item); err != nil {
return nil, err
}
return item, nil
}
return nil, ErrItemNotExisted
return nil, err
}
func (p *PanSyncDbBolt) GetFileList(filePath string) (PanFileList, error) {
filePath = FormatFilePath(filePath)
if filePath == "" {
return nil, fmt.Errorf("item is nil")
}
p.locker.Lock()
defer p.locker.Unlock()
panFileList := PanFileList{}
dataList, err := p.db.GetFileList(filePath)
if err == nil && len(dataList) > 0 {
for _, data := range dataList {
item := &PanFileItem{}
if err := json.Unmarshal([]byte(data), item); err != nil {
return nil, err
}
panFileList = append(panFileList, item)
}
return panFileList, nil
}
return nil, err
}
// Delete 删除一个数据项
func (p *PanSyncDb) Delete(filePath string) (bool, error) {
func (p *PanSyncDbBolt) Delete(filePath string) (bool, error) {
filePath = FormatFilePath(filePath)
if filePath == "" {
return false, fmt.Errorf("item is nil")
@@ -241,121 +107,27 @@ func (p *PanSyncDb) Delete(filePath string) (bool, error) {
p.locker.Lock()
defer p.locker.Unlock()
tx, err := p.db.Begin(true)
if err != nil {
return false, err
}
defer tx.Rollback()
partsOrg := strings.Split(filePath, "/")
parts := []string{}
for _, p := range partsOrg {
if p == "" {
continue
}
parts = append(parts, p)
}
// get parent node
bkt := tx.Bucket([]byte("/"))
if bkt == nil {
return false, ErrItemNotExisted
}
for _, p := range parts[:len(parts)-1] {
bkt = bkt.Bucket([]byte(p))
if bkt == nil {
return false, ErrItemNotExisted
}
}
if bkt == nil {
return false, ErrItemNotExisted
}
targetName := []byte(parts[len(parts)-1])
dirBucket := bkt.Bucket(targetName)
var er error
if dirBucket != nil {
// is dir, delete bucket
er = bkt.DeleteBucket(targetName)
} else {
// is file, delete item
er = bkt.Delete(targetName)
}
// Commit the transaction and check for error.
if err := tx.Commit(); err != nil {
return false, err
}
return er == nil, nil
return p.db.Delete(filePath)
}
// Update 更新数据项,数据项不存在返回错误
func (p *PanSyncDb) Update(item *PanFileItem) (bool, error) {
func (p *PanSyncDbBolt) Update(item *PanFileItem) (bool, error) {
if item == nil {
return false, fmt.Errorf("item is nil")
}
p.locker.Lock()
defer p.locker.Unlock()
// update item
// Start a writable transaction.
tx, err := p.db.Begin(true)
data, err := json.Marshal(item)
if err != nil {
return false, err
}
defer tx.Rollback()
// get parent bucket of update item
parts := strings.Split(item.FormatFilePath(), "/")
bkt := tx.Bucket([]byte("/"))
if bkt == nil {
return false, ErrItemNotExisted
}
for _, p := range parts[:len(parts)-1] {
if p == "" {
continue
}
bkt = bkt.Bucket([]byte(p))
if bkt == nil {
return false, ErrItemNotExisted
}
}
if bkt == nil {
return false, ErrItemNotExisted
}
// update content
rs, err := json.Marshal(item)
if err != nil {
return false, err
}
if item.IsFolder() {
bkt = bkt.Bucket([]byte(item.FormatFileName()))
if bkt == nil {
return false, ErrItemNotExisted
}
if e := bkt.Put([]byte(DefaultDirKeyName), rs); e != nil {
return false, e
}
} else {
// is file
if e := bkt.Put([]byte(item.FormatFileName()), rs); e != nil {
return false, e
}
}
// Commit the transaction and check for error.
if err := tx.Commit(); err != nil {
return false, err
}
return true, nil
return p.db.Update(item.Path, item.IsFolder(), string(data))
}
// Close 关闭数据库
func (p *PanSyncDb) Close() (bool, error) {
if p.db != nil {
if e := p.db.Close(); e != nil {
return false, e
}
}
return true, nil
func (p *PanSyncDbBolt) Close() (bool, error) {
p.locker.Lock()
defer p.locker.Unlock()
return p.db.Close()
}
@@ -11,7 +11,7 @@ import (
func TestPanSyncDb(t *testing.T) {
// get access token
refreshToken := "39b6583...b662b2a522"
refreshToken := "c2b11bfc07...f090dc07ea59"
webToken, err := aliyunpan.GetAccessTokenFromRefreshToken(refreshToken)
if err != nil {
fmt.Println("get acccess token error")
@@ -52,3 +52,11 @@ func TestGet(t *testing.T) {
fmt.Println(b.Get("/Parallels Desktop/v17/部分电脑安装v17可能有问题,请退回v16版本.txt"))
}
func TestGetFileList(t *testing.T) {
b := NewPanSyncDb("D:\\smb\\feny\\goprojects\\dev\\pan.db")
b.Open()
defer b.Close()
fmt.Println(b.GetFileList("/Parallels Desktop/v17"))
}