From 3196cb41c4b30ccae2f9b335cba08eb31fe342c1 Mon Sep 17 00:00:00 2001 From: libsgh Date: Sat, 1 Jul 2023 18:48:18 +0800 Subject: [PATCH 1/4] =?UTF-8?q?:bug:=E3=80=8C139=E3=80=8Dfix=20login=20fai?= =?UTF-8?q?led?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pan/yun139/Yun139.go | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/pan/yun139/Yun139.go b/pan/yun139/Yun139.go index 1fe8888b..f6060991 100644 --- a/pan/yun139/Yun139.go +++ b/pan/yun139/Yun139.go @@ -3,6 +3,7 @@ package yun139 import ( "bytes" "crypto/md5" + "encoding/base64" "fmt" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" @@ -31,7 +32,7 @@ func (y Yun139) IsLogin(account *module.Account) bool { }, } resp, err := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/user/v1.0/qryUserExternInfo") status := jsoniter.Get(resp.Body(), "success").ToBool() @@ -42,9 +43,12 @@ func (y Yun139) IsLogin(account *module.Account) bool { } func (y Yun139) AuthLogin(account *module.Account) (string, error) { + authToken := util.GetBetweenStr(account.Password, "auth_token=", ";") + token := "Basic " + base64.StdEncoding.EncodeToString([]byte((fmt.Sprintf("pc:%s:%s", account.User, authToken)))) + account.RefreshToken = token isLogin, err := y.LoginCheck(*account) if isLogin { - return account.Password, err + return account.RefreshToken, err } return "", err } @@ -68,7 +72,7 @@ func (y Yun139) Files(account module.Account, fileId, path, sortColumn, sortOrde var filesResp Yun139FilesResp _, err = resty.New().R(). SetResult(&filesResp). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/catalog/v1.0/getDisk") if err == nil && filesResp.Success { @@ -220,7 +224,7 @@ func (y Yun139) UploadFiles(account module.Account, parentFileId string, files [ }}, } resp, err := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest") if err != nil { @@ -272,7 +276,7 @@ func (y Yun139) Rename(account module.Account, fileId, name string) (bool, inter "commonAccountInfo": base.KV{"account": account.User, "accountType": 1}, } resp, err := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/catalog/v1.0/updateCatalogInfo") if err != nil { @@ -287,7 +291,7 @@ func (y Yun139) Rename(account module.Account, fileId, name string) (bool, inter "commonAccountInfo": base.KV{"account": account.User, "accountType": 1}, } resp, err = resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/content/v1.0/updateContentInfo") if err != nil { @@ -324,7 +328,7 @@ func (y Yun139) QueryTaskInfo(account module.Account, taskId string) { }, } resp, _ := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/batchOprTask/v1.0/queryBatchOprTaskDetail") log.Debug("Task query:", resp.String()) @@ -339,7 +343,7 @@ func (y Yun139) Mkdir(account module.Account, parentFileId, name string) (bool, }, } resp, err := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/catalog/v1.0/createCatalogExt") if err != nil { @@ -367,7 +371,7 @@ func (y Yun139) CommonReq(account module.Account, actionType, taskType int, cata }, } resp, err := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask") return resp, err @@ -408,7 +412,7 @@ func (y Yun139) GetDownloadUrl(account module.Account, fileId string) (string, e "commonAccountInfo": base.KV{"account": account.User, "accountType": 1}, } resp, err := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/uploadAndDownload/v1.0/downloadRequest") status := jsoniter.Get(resp.Body(), "success").ToBool() @@ -424,7 +428,7 @@ func (y Yun139) GetSpaceSzie(account module.Account) (int64, int64) { "commonAccountInfo": base.KV{"account": account.User, "accountType": 1}, } resp, err := resty.New().R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/user/v1.0/getDiskInfo") if err != nil { @@ -450,7 +454,7 @@ func (y Yun139) LoginCheck(account module.Account) (bool, error) { }, } resp, err := base.Client.R(). - SetHeaders(y.CreateHeaders(body, account.Password)). + SetHeaders(y.CreateHeaders(body, account.RefreshToken)). SetBody(body). Post("https://yun.139.com/orchestration/personalCloud/user/v1.0/qryUserExternInfo") if err == nil && jsoniter.Get(resp.Body(), "success").ToBool() { @@ -476,7 +480,8 @@ func (y Yun139) CreateHeaders(body base.KV, cookie string) map[string]string { "x-DeviceInfo": "||9|6.5.2|chrome|95.0.4638.17|||linux unknow||zh-CN|||", "x-SvcType": "1", "referer": "https://yun.139.com/w/", - "Cookie": cookie, + "Authorization": cookie, + //"Cookie": cookie, } return headers } From 9a3bd51b5e796e47bd63fc04b64dbcafc0ce29e9 Mon Sep 17 00:00:00 2001 From: libsgh Date: Tue, 8 Aug 2023 11:37:56 +0800 Subject: [PATCH 2/4] :bug:Too many files cause caching to fail --- dao/db.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++++-- util/common.go | 15 ++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/dao/db.go b/dao/db.go index eaab39d5..c0f6b536 100644 --- a/dao/db.go +++ b/dao/db.go @@ -74,6 +74,8 @@ var InitConfigItems = []module.ConfigItem{ {"dav_password", "1234", "dav"}, {"proxy", "", "common"}, {"jwt_sign_key", uuid.NewV4().String(), "common"}, + {"enable_download_statistics", "0", "common"}, + {"show_download_info", "0", "common"}, } type Db interface { @@ -100,6 +102,7 @@ func InitDb() { DB.AutoMigrate(&module.HideFiles{}) DB.AutoMigrate(&module.Bypass{}) DB.AutoMigrate(&module.BypassAccounts{}) + DB.AutoMigrate(&module.DownloadStatistics{}) //init data var count int64 err := DB.Model(module.ConfigItem{}).Count(&count).Error @@ -108,7 +111,7 @@ func InitDb() { panic(err) } else if count == 0 { //DB.Create(&InitConfigItems) - rand.Seed(time.Now().UnixNano()) + rand.New(rand.NewSource(time.Now().UnixNano())) ApiToken := strconv.Itoa(rand.Intn(10000000)) configItem := module.ConfigItem{K: "api_token", V: ApiToken, G: "common"} DB.Create(configItem) @@ -178,6 +181,7 @@ func InitGlobalConfig() { module.GloablConfig = c c.CdnFiles = util.GetCdnFilesMap(c.Cdn, module.VERSION) c.ShareInfoList = GetShareInfoList() + c.DownloadStatisticsList = GetAllDownloadStatistics() module.GloablConfig = c RoundRobin() } @@ -399,7 +403,11 @@ func LoopCreateFiles(account module.Account, fileId, path string, hide, hasPwd i } } if len(fileNodes) > 0 { - DB.Create(&fileNodes) + //避免但文件夹文件过多导致同步失败 + groupNodes := util.Group(fileNodes, 500) + for _, nodes := range groupNodes { + DB.Create(&nodes) + } } if len(fileNodes) > 0 || (len(fileNodes) == 0 && err == nil) { RetryTasksCache.Remove(account.Id + fileId) @@ -867,3 +875,55 @@ func DeleteShareInfo(paths []string) { } InitGlobalConfig() } + +func DeleteStatistics(ids []string) { + for _, id := range ids { + DB.Where("id = ?", id).Delete(module.DownloadStatistics{}) + } + InitGlobalConfig() +} + +func SyncDownloadInfo(ac module.Account, fileNode module.FileNode) { + var downloadStatistics module.DownloadStatistics + err := DB.Where("id = ?", fileNode.FileId).First(&downloadStatistics).Error + if errors.Is(err, gorm.ErrRecordNotFound) { + DB.Create(module.DownloadStatistics{ + Id: fileNode.FileId, + AccountName: ac.Name, + FileName: fileNode.FileName, + LastDownloadTime: util.UTCTime(time.Now()), + Path: fileNode.Path, + Count: 1, + }) + } else { + count := downloadStatistics.Count + 1 + DB.Table("download_statistics").Where("id=?", fileNode.FileId).Update("count", count) + } + InitGlobalConfig() +} + +func GetDownloadCount(fns []module.FileNode) []module.FileNode { + if len(fns) > 0 { + fn := fns[0] + var downloadStatistics module.DownloadStatistics + err := DB.Where("id = ?", fns[0].FileId).First(&downloadStatistics).Error + extraData := fns[0].ExtraData + if extraData == nil { + extraData = make(map[string]interface{}) + } + if errors.Is(err, gorm.ErrRecordNotFound) { + extraData["DownloadCount"] = 0 + } else { + extraData["DownloadCount"] = downloadStatistics.Count + } + fn.ExtraData = extraData + return []module.FileNode{fn} + } + return fns +} + +func GetAllDownloadStatistics() []module.DownloadStatistics { + var data []module.DownloadStatistics + DB.Where("1=1").Find(&data) + return data +} diff --git a/util/common.go b/util/common.go index 0253acf0..b9c242ec 100644 --- a/util/common.go +++ b/util/common.go @@ -915,3 +915,18 @@ func ExeFilePath(def string) string { exPath := filepath.Dir(ex) return exPath } + +func Group(nodes []module.FileNode, subGroupLength int64) [][]module.FileNode { + max := int64(len(nodes)) + var segmens = make([][]module.FileNode, 0) + quantity := max / subGroupLength + remainder := max % subGroupLength + i := int64(0) + for i = int64(0); i < quantity; i++ { + segmens = append(segmens, nodes[i*subGroupLength:(i+1)*subGroupLength]) + } + if quantity == 0 || remainder != 0 { + segmens = append(segmens, nodes[i*subGroupLength:i*subGroupLength+remainder]) + } + return segmens +} From 4906fc6c48821596c60a6327e4607bf342446b06 Mon Sep 17 00:00:00 2001 From: libsgh Date: Tue, 8 Aug 2023 11:53:53 +0800 Subject: [PATCH 3/4] :bug:Too many files cause caching to fail --- module/entity.go | 109 ++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/module/entity.go b/module/entity.go index 104d99cd..0615046c 100644 --- a/module/entity.go +++ b/module/entity.go @@ -52,54 +52,57 @@ type Paths struct { IsCoShare int `json:"isCoShare"` } type Config struct { - Accounts []Account `json:"accounts" gorm:"-"` - SiteName string `json:"site_name"` - AccountChoose string `json:"account_choose"` - Theme string `json:"theme"` - PathPrefix string `json:"path_prefix""` //路径前缀 - AdminUser string `json:"admin_user""` - AdminPassword string `json:"admin_password""` - OnlyReferrer string `json:"only_referrer"` - EnableSafetyLink string `json:"enable_safety_link"` - IsNullReferrer string `json:"is_null_referrer"` - FaviconUrl string `json:"favicon_url"` //网站图标 - Footer string `json:"footer"` //网站底部信息 - Css string `json:"css"` //自定义css - Js string `json:"js"` //自定义js - EnablePreview string `json:"enable_preview"` //是否开启文件预览 - Image string `json:"image"` //图片 - Audio string `json:"audio"` //音频 - Video string `json:"video"` //视频 - Code string `json:"code"` //代码 - Doc string `json:"doc"` //文档 - Other string `json:"other"` //other - EnableLrc string `json:"enable_lrc"` //是否开启歌词 - LrcPath string `json:"lrc_path"` //歌词路径 - Subtitle string `json:"subtitle"` //字幕 - SubtitlePath string `json:"subtitle_path"` //字幕路径 - Danmuku string `json:"danmuku"` //弹幕 - DanmukuPath string `json:"danmuku_path"` //弹幕路径 - SColumn string `json:"s_column"` //排序字段 - SOrder string `json:"s_order"` //排序顺序 - PwdFiles []PwdFiles `json:"pwd_files"` //密码文件列表 - HideFiles map[string]string `json:"hide_files"` //隐藏文件 - AdminPath string `json:"admin_path"` //后台管理路径前缀 - Cdn string `json:"cdn"` //cdn - CdnFiles map[string]string `json:"-"` //cdn files - BypassList []Bypass `json:"bypass_list"` //分流列表 - EnableDav string `json:"enable_dav"` //dav enabled 1 disabled 0 - DavPath string `json:"dav_path"` //dav path - DavMode string `json:"dav_mode"` //0 read-only, 1 read-write - DavDownMode string `json:"dav_down_mode"` //0 302 downloadurl, 1 proxy - DavUser string `json:"dav_user"` //dav user - DavPassword string `json:"dav_password"` //dav password - Proxy string `json:"proxy"` //google api prxoy - Readme string `json:"readme"` //show or hide readme - Head string `json:"head"` //show or hide head - ShareInfoList []ShareInfo `json:"share_info_list"` //share info list - Access string `json:"access"` //access - ShortAction string `json:"short_action"` //access short link action - JwtSignKey string `json:"jwt_sign_key"` //jwt sign key + Accounts []Account `json:"accounts" gorm:"-"` + SiteName string `json:"site_name"` + AccountChoose string `json:"account_choose"` + Theme string `json:"theme"` + PathPrefix string `json:"path_prefix""` //路径前缀 + AdminUser string `json:"admin_user""` + AdminPassword string `json:"admin_password""` + OnlyReferrer string `json:"only_referrer"` + EnableSafetyLink string `json:"enable_safety_link"` + IsNullReferrer string `json:"is_null_referrer"` + FaviconUrl string `json:"favicon_url"` //网站图标 + Footer string `json:"footer"` //网站底部信息 + Css string `json:"css"` //自定义css + Js string `json:"js"` //自定义js + EnablePreview string `json:"enable_preview"` //是否开启文件预览 + Image string `json:"image"` //图片 + Audio string `json:"audio"` //音频 + Video string `json:"video"` //视频 + Code string `json:"code"` //代码 + Doc string `json:"doc"` //文档 + Other string `json:"other"` //other + EnableLrc string `json:"enable_lrc"` //是否开启歌词 + LrcPath string `json:"lrc_path"` //歌词路径 + Subtitle string `json:"subtitle"` //字幕 + SubtitlePath string `json:"subtitle_path"` //字幕路径 + Danmuku string `json:"danmuku"` //弹幕 + DanmukuPath string `json:"danmuku_path"` //弹幕路径 + SColumn string `json:"s_column"` //排序字段 + SOrder string `json:"s_order"` //排序顺序 + PwdFiles []PwdFiles `json:"pwd_files"` //密码文件列表 + HideFiles map[string]string `json:"hide_files"` //隐藏文件 + AdminPath string `json:"admin_path"` //后台管理路径前缀 + Cdn string `json:"cdn"` //cdn + CdnFiles map[string]string `json:"-"` //cdn files + BypassList []Bypass `json:"bypass_list"` //分流列表 + EnableDav string `json:"enable_dav"` //dav enabled 1 disabled 0 + DavPath string `json:"dav_path"` //dav path + DavMode string `json:"dav_mode"` //0 read-only, 1 read-write + DavDownMode string `json:"dav_down_mode"` //0 302 downloadurl, 1 proxy + DavUser string `json:"dav_user"` //dav user + DavPassword string `json:"dav_password"` //dav password + Proxy string `json:"proxy"` //google api prxoy + Readme string `json:"readme"` //show or hide readme + Head string `json:"head"` //show or hide head + ShareInfoList []ShareInfo `json:"share_info_list"` //share info list + DownloadStatisticsList []DownloadStatistics `json:"download_statistics_list"` //download statistics list + Access string `json:"access"` //access + ShortAction string `json:"short_action"` //access short link action + JwtSignKey string `json:"jwt_sign_key"` //jwt sign key + EnableDownloadStatistics string `json:"enable_download_statistics"` //enable download statistics + ShowDownloadInfo string `json:"show_download_info"` //show download info } type ConfigItem struct { K string `json:"k" gorm:"unique;not null"` //配置项key @@ -172,6 +175,16 @@ type Cache struct { Data interface{} `json:"data"` } +// File download statistics +type DownloadStatistics struct { + Id string `json:"id"` //数据库唯一主键（文件ID） + AccountName string `json:"account_name"` //文件所属账号 + FileName string `json:"file_name" gorm:"index:idx_fn"` //文件名称 + LastDownloadTime string `json:"last_op_time"` //最近一次下载时间 + Path string `json:"path" gorm:"index:idx_p"` //文件路径 + Count int64 `json:"count"` //下载次数 +} + type Cloud189 struct { Cloud189Session *resty.Client SessionKey string `json:"session_key"` From eecc1340d3de9f8587bfd12649f93b86eeb79287 Mon Sep 17 00:00:00 2001 From: libsgh Date: Tue, 21 Nov 2023 16:43:44 +0800 Subject: [PATCH 4/4] :bug:fonts cdn load error --- util/common.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/common.go b/util/common.go index b9c242ec..fdafb19f 100644 --- a/util/common.go +++ b/util/common.go @@ -473,7 +473,7 @@ func GetCdnFilesMap(cdn, version string) map[string]string { "natural@compare@js": "//fastly.jsdelivr.net/npm/natural-compare-lite@1.4.0/index.js", "bootstrap@css": "//cdn.staticfile.org/bootstrap/4.6.1/css/bootstrap.min.css", "bootstrap@js": "//cdn.staticfile.org/bootstrap/4.6.1/js/bootstrap.min.js", - "Material+Icons@css": "//fonts.sourcegcdn.com/icon?family=Material+Icons", + "Material+Icons@css": "//fonts.loli.net/icon?family=Material+Icons", }, "2": KV{ "mdui@css": jp + "/static/lib/mdui@1.0.2/css/mdui.min.css", @@ -513,7 +513,7 @@ func GetCdnFilesMap(cdn, version string) map[string]string { "natural@compare@js": jp + "/static/lib/natural-compare-lite@1.4.0/index.min.js", "bootstrap@css": jp + "/static/lib/bootstrap@4.6.1/css/bootstrap.min.css", "bootstrap@js": jp + "/static/lib/bootstrap@4.6.1/js/bootstrap.min.js", - "Material+Icons@css": "//fonts.sourcegcdn.com/icon?family=Material+Icons", + "Material+Icons@css": "//fonts.loli.net/icon?family=Material+Icons", }, } cdnKV := cdnMap["0"].(KV)