diff --git a/fileserver.go b/fileserver.go index 2fe740b3..88f2b4a0 100644 --- a/fileserver.go +++ b/fileserver.go @@ -23,6 +23,7 @@ import ( "runtime/debug" "strconv" "syscall" + // "strconv" "sync" @@ -150,13 +151,14 @@ type Server struct { } type FileInfo struct { - Name string - ReName string - Path string - Md5 string - Size int64 - Peers []string - Scene string + Name string + ReName string + Path string + Md5 string + Size int64 + Peers []string + Scene string + TimeStamp int64 } type Status struct { @@ -407,6 +409,16 @@ func (this *Common) IsExist(filename string) bool { return err == nil || os.IsExist(err) } +func (this *Common) Match(matcher string, content string) []string { + var result []string + if reg, err := regexp.Compile(matcher); err == nil { + + result = reg.FindAllString(content, -1) + + } + return result +} + func (this *Common) ReadBinFile(path string) ([]byte, error) { if this.IsExist(path) { fi, err := os.Open(path) @@ -471,6 +483,94 @@ func (this *Common) GetClientIp(r *http.Request) string { } +func (this *Server) RepairStat() { + + var ( + size int64 + count int64 + ) + defer func() { + if re := recover(); re != nil { + buffer := debug.Stack() + log.Error("RepairStat") + log.Error(re) + log.Error(string(buffer)) + fmt.Println(re) + } + }() + + handlefunc := func(file_path string, f os.FileInfo, err error) error { + + var ( + files []os.FileInfo + date []string + data []byte + content string + lines []string + count int64 + totalSize int64 + line string + cols []string + size int64 + ) + + if f.IsDir() { + + if files, err = ioutil.ReadDir(file_path); err != nil { + return err + } + + for _, file := range files { + count = 0 + size = 0 + if file.Name() == CONST_FILE_Md5_FILE_NAME { + if data, err = ioutil.ReadFile(file_path + "/" + file.Name()); err != nil { + log.Error(err) + continue + } + date = this.util.Match("\\d{8}", file_path) + if len(date) < 1 { + continue + } + content = string(data) + lines = strings.Split(content, "\n") + count = int64(len(lines)) + if count > 1 { + count = count - 1 + } + for _, line = range lines { + + cols = strings.Split(line, "|") + if len(cols) > 2 { + if size, err = strconv.ParseInt(cols[1], 10, 64); err != nil { + size = 0 + continue + } + totalSize = totalSize + size + } + + } + statMap.Put(date[0]+"_"+CONST_STAT_FILE_COUNT_KEY, count) + statMap.Put(date[0]+"_"+CONST_STAT_FILE_TOTAL_SIZE_KEY, totalSize) + statMap.AddCountInt64(CONST_STAT_FILE_COUNT_KEY, count) + statMap.AddCountInt64(CONST_STAT_FILE_TOTAL_SIZE_KEY, totalSize) + + } + + } + + } + + return nil + } + + statMap.Put(CONST_STAT_FILE_COUNT_KEY, count) + statMap.Put(CONST_STAT_FILE_TOTAL_SIZE_KEY, size) + + filepath.Walk(DATA_DIR, handlefunc) + +} + func (this *Server) DownloadFromPeer(peer string, fileInfo *FileInfo) { var ( err error @@ -742,6 +842,7 @@ func (this *Server) postFileToPeer(fileInfo *FileInfo, write_log bool) { b.Header("Sync-Path", fileInfo.Path) b.Param("name", filename) b.Param("md5", fileInfo.Md5) + b.Param("timestamp", fmt.Sprintf("%d", fileInfo.TimeStamp)) b.PostFile("file", fileInfo.Path+"/"+filename) result, err = b.String() if err != nil { @@ -790,12 +891,12 @@ func (this *Server) SaveFileMd5Log(fileInfo *FileInfo, filename string) { outname = fileInfo.ReName } - logpath = DATA_DIR + "/" + time.Now().Format("20060102") + logpath = DATA_DIR + "/" + time.Unix(fileInfo.TimeStamp, 0).Format("20060102") if _, err = os.Stat(logpath); err != nil { os.MkdirAll(logpath, 0777) } msg = fmt.Sprintf("%s|%d|%s\n", fileInfo.Md5, fileInfo.Size, fileInfo.Path+"/"+outname) - if tmpFile, err = os.OpenFile(DATA_DIR+"/"+time.Now().Format("20060102")+"/"+filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644); err != nil { + if tmpFile, err = os.OpenFile(logpath+"/"+filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644); err != nil { log.Error(err) return } @@ -824,24 +925,23 @@ func (this *Server) GetFileInfoByMd5(md5sum string) (*FileInfo, error) { func (this *Server) checkPeerFileExist(peer string, md5sum string) (*FileInfo, error) { var ( - err error + err error + fileInfo FileInfo ) req := httplib.Get(peer + fmt.Sprintf("/check_file_exist?md5=%s", md5sum)) req.SetTimeout(time.Second*5, time.Second*5) - var fileInfo FileInfo - - if err = req.ToJSON(&fileInfo); err == nil { - if fileInfo.Md5 == "" { - return &FileInfo{}, nil - } else { + if err = req.ToJSON(&fileInfo); err != nil { + return &FileInfo{}, err + } - return &fileInfo, nil - } + if fileInfo.Md5 == "" { + return &fileInfo, errors.New("not found") } - return &FileInfo{}, errors.New("file not found") + + return &fileInfo, nil } @@ -1012,9 +1112,10 @@ func (this *Server) IsPeer(r *http.Request) bool { func (this *Server) SyncFile(w http.ResponseWriter, r *http.Request) { var ( - err error - outPath string - outname string + err error + outPath string + outname string + // timestamp string fileInfo FileInfo tmpFile *os.File fi os.FileInfo @@ -1027,9 +1128,15 @@ func (this *Server) SyncFile(w http.ResponseWriter, r *http.Request) { } if r.Method == "POST" { + fileInfo.Path = r.Header.Get("Sync-Path") fileInfo.Md5 = r.PostFormValue("md5") fileInfo.Name = r.PostFormValue("name") + fileInfo.TimeStamp, err = strconv.ParseInt(r.PostFormValue("timestamp"), 10, 64) + if err != nil { + fileInfo.TimeStamp = time.Now().Unix() + log.Error(err) + } if uploadFile, _, err = r.FormFile("file"); err != nil { w.Write([]byte(err.Error())) log.Error(err) @@ -1082,12 +1189,14 @@ func (this *Server) SyncFile(w http.ResponseWriter, r *http.Request) { if _, err = io.Copy(tmpFile, uploadFile); err != nil { w.Write([]byte(err.Error())) log.Error(err) + return } if this.util.GetFileMd5(tmpFile) != fileInfo.Md5 { w.Write([]byte("md5 error")) tmpFile.Close() os.Remove(outPath) + return } @@ -1097,6 +1206,7 @@ func (this *Server) SyncFile(w http.ResponseWriter, r *http.Request) { fileInfo.Size = fi.Size() statMap.AddCountInt64(CONST_STAT_FILE_TOTAL_SIZE_KEY, fi.Size()) statMap.AddCountInt64(CONST_STAT_FILE_COUNT_KEY, 1) + } if fileInfo.Peers == nil { @@ -1213,6 +1323,7 @@ func (this *Server) Upload(w http.ResponseWriter, r *http.Request) { return } fileInfo.Peers = []string{} + fileInfo.TimeStamp = time.Now().Unix() if scene == "" { scene = Config().DefaultScene @@ -1435,8 +1546,6 @@ func (this *Server) Upload(w http.ResponseWriter, r *http.Request) { statMap.AddCountInt64(this.util.GetToDay()+"_"+CONST_STAT_FILE_COUNT_KEY, 1) } - // this.SaveStat() - this.SaveFileMd5Log(&fileInfo, CONST_FILE_Md5_FILE_NAME) p := strings.Replace(fileInfo.Path, STORE_DIR+"/", "", 1) @@ -1531,6 +1640,15 @@ func (this *Server) BenchMark(w http.ResponseWriter, r *http.Request) { util.WriteFile("time.txt", time.Since(t).String()) fmt.Println(time.Since(t).String()) } + +func (this *Server) RepairStatWeb(w http.ResponseWriter, r *http.Request) { + + this.RepairStat() + + w.Write([]byte("ok")) + +} + func (this *Server) Stat(w http.ResponseWriter, r *http.Request) { var ( min int64 @@ -1607,7 +1725,6 @@ func (this *Server) RegisterExit() { for s := range c { switch s { case syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT: - this.SaveStat() this.ldb.Close() log.Info("Exit", s) os.Exit(1) @@ -1892,6 +2009,7 @@ func (this *Server) Main() { util.RemoveEmptyDir(STORE_DIR) } }() + go server.RepairStat() go this.SaveStat() go this.Check() @@ -1901,6 +2019,7 @@ func (this *Server) Main() { http.HandleFunc("/delete", this.RemoveFile) http.HandleFunc("/sync", this.Sync) http.HandleFunc("/stat", this.Stat) + http.HandleFunc("/repair_stat", this.RepairStatWeb) http.HandleFunc("/status", this.Status) http.HandleFunc("/syncfile", this.SyncFile) http.HandleFunc("/"+Config().Group+"/", this.Download)