[WIP] On-demand filelist fetching

config/default_config.yml

@@ -107,6 +107,8 @@ torrents:
 torrents:
 # GenerationClientPort : Port used by the torrent client created during torrent generation
   generation_client_port: 50006
+# FilesFetchingClientPort: Port used by the client created by file fetching
+  files_fetching_client_port: 50005
 # FileStorage : Location of folder that will contain generated torrent files
   filestorage: ./downloads/
 # TorrentStorageLink : Url of torrent file download location (eg https://your.site/somewhere/%s)

config/structs.go

@@ -146,6 +146,7 @@ type TorrentsConfig struct {
 	Sort                          string            `yaml:"sort,omitempty"`
 	Tags                          Tags              `yaml:"tags,flow,omitempty"`
 	GenerationClientPort          int               `yaml:"generation_client_port,flow,omitempty"`
+	FilesFetchingClientPort       int               `yaml:"files_fetching_client_port,flow,omitempty"`
 }
 
 // UploadConfig : Config struct for uploading torrents

controllers/torrent/files.go

@@ -0,0 +1,78 @@
+package torrentController
+
+import (
+	"html/template"
+	"encoding/hex"
+	"net/http"
+	"strings"
+	"strconv"
+
+	"github.com/NyaaPantsu/nyaa/models/torrents"
+	"github.com/NyaaPantsu/nyaa/models"
+	"github.com/NyaaPantsu/nyaa/templates"
+	"github.com/NyaaPantsu/nyaa/utils/format"
+	"github.com/NyaaPantsu/nyaa/utils/filelist"
+	"github.com/Stephen304/goscrape"
+	"github.com/gin-gonic/gin"
+)
+
+func GetFilesHandler(c *gin.Context) {
+	id, _ := strconv.ParseInt(c.Param("id"), 10, 32)
+	torrent, err := torrents.FindByID(uint(id))	
+
+	if err != nil {
+		c.Status(http.StatusNotFound)
+		return
+	}
+
+
+	if len(torrent.FileList) == 0 {
+		var blankScrape models.Scrape
+		ScrapeFiles(format.InfoHashToMagnet(strings.TrimSpace(torrent.Hash), torrent.Name, GetTorrentTrackers(torrent)...), torrent, blankScrape, true)
+	}
+
+	folder := filelist.FileListToFolder(torrent.FileList, "root")
+	templates.TorrentFileList(c, torrent.ToJSON(), folder)
+}
+
+// ScrapeFiles : Scrape torrent files
+func ScrapeFiles(magnet string, torrent *models.Torrent, currentStats models.Scrape, statsExists bool) (error, []FileJSON) {
+	if client == nil {
+		err := initClient()
+		if err != nil {
+			return err, []FileJSON{}
+		}
+	}
+
+	t, _ := client.AddMagnet(magnet)
+	<-t.GotInfo()
+
+	infoHash := t.InfoHash()
+	dst := make([]byte, hex.EncodedLen(len(t.InfoHash())))
+	hex.Encode(dst, infoHash[:])
+
+	var UDP []string
+
+	for _, tracker := range t.Metainfo().AnnounceList[0] {
+		if strings.HasPrefix(tracker, "udp") {
+			UDP = append(UDP, tracker)
+		}
+	}
+	var results goscrape.Result
+	if len(UDP) != 0 {
+		udpscrape := goscrape.NewBulk(UDP)
+		results = udpscrape.ScrapeBulk([]string{torrent.Hash})[0]
+	}
+	t.Drop()
+	return nil, UpdateTorrentStats(torrent, results, currentStats, t.Files(), statsExists)
+}
+
+// FileJSON for file model in json, 
+type FileJSON struct {
+	Path       string         `json:"path"`
+	Filesize   template.HTML  `json:"filesize"`
+}
+
+func fileSize(filesize int64) template.HTML {
+	return template.HTML(format.FileSize(filesize))
+}

controllers/torrent/router.go

@@ -8,6 +8,7 @@ import (
 func init() {
 	router.Get().Any("/download/:hash", DownloadTorrent)
 	router.Get().Any("/stats/:id", GetStatsHandler)
+	router.Get().Any("/files/:id", GetFilesHandler)
 
 	torrentRoutes := router.Get().Group("/torrent", middlewares.LoggedInMiddleware())
 	{

controllers/torrent/stats.go

@@ -1,6 +1,7 @@
 package torrentController
 
 import (
+	"path/filepath"
 	"strconv"
 	"strings"
 	"net/url"
@@ -9,93 +10,176 @@ import (
 	"github.com/NyaaPantsu/nyaa/models/torrents"
 	"github.com/NyaaPantsu/nyaa/models"
 	"github.com/NyaaPantsu/nyaa/config"
+	"github.com/NyaaPantsu/nyaa/utils/log"
+	"github.com/NyaaPantsu/nyaa/utils/format"
 	"github.com/Stephen304/goscrape"
 	"github.com/gin-gonic/gin"
+
+	"github.com/anacrolix/dht"
+	"github.com/anacrolix/torrent"
+	"github.com/bradfitz/slice"
 )
 
+var client *torrent.Client
+
+func initClient() error {
+	clientConfig := torrent.Config{
+		DHTConfig: dht.ServerConfig{
+			StartingNodes: dht.GlobalBootstrapAddrs,
+		},
+		ListenAddr: ":" + strconv.Itoa(config.Get().Torrents.FilesFetchingClientPort),
+	}
+	cl, err := torrent.NewClient(&clientConfig)
+	if err != nil {
+		log.Errorf("error creating client: %s", err)
+		return err
+	}
+	client = cl
+	return nil
+}
+
 // ViewHeadHandler : Controller for getting torrent stats
 func GetStatsHandler(c *gin.Context) {
 	id, err := strconv.ParseInt(c.Param("id"), 10, 32)
 	if err != nil {
 		return
 	}
 
-	torrent, err := torrents.FindRawByID(uint(id))
-
+	updateTorrent, err := torrents.FindByID(uint(id))
+	
 	if err != nil {
 		return
 	}
 
 	var CurrentData models.Scrape
 	statsExists := !(models.ORM.Where("torrent_id = ?", id).Find(&CurrentData).RecordNotFound())
 
-	if statsExists {
+	if statsExists && c.Request.URL.Query()["files"] == nil {
 		//Stats already exist, we check if the torrent stats have been scraped already very recently and if so, we stop there to avoid abuse of the /stats/:id route
-		if (CurrentData.Seeders == 0 && CurrentData.Leechers == 0 && CurrentData.Completed == 0)  && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequencyUnknown {
+		if isEmptyScrape(CurrentData)  && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequencyUnknown {
 			//Unknown stats but has been scraped less than X minutes ago (X being the limit set in the config file)
 			return
 		}
-		if (CurrentData.Seeders != 0 || CurrentData.Leechers != 0 || CurrentData.Completed != 0) && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequency  {
+		if !isEmptyScrape(CurrentData) && time.Since(CurrentData.LastScrape).Minutes() <= config.Get().Scrape.MaxStatScrapingFrequency  {
 			//Known stats but has been scraped less than X minutes ago (X being the limit set in the config file)
 			return
 		}
 	}
 
-	var Trackers []string
-	if len(torrent.Trackers) > 3 {
-		for _, line := range strings.Split(torrent.Trackers[3:], "&tr=") {
-			tracker, error := url.QueryUnescape(line)
-			if error == nil && strings.HasPrefix(tracker, "udp") {
-				Trackers = append(Trackers, tracker)
-			}
-			//Cannot scrape from http trackers so don't put them in the array
-		}
-	}
+	Trackers := GetTorrentTrackers(updateTorrent)
 
-	for _, tracker := range config.Get().Torrents.Trackers.Default {
-		if !contains(Trackers, tracker) && strings.HasPrefix(tracker, "udp")  {
-			Trackers = append(Trackers, tracker)
+	var stats goscrape.Result
+	var torrentFiles []FileJSON
+
+	if c.Request.URL.Query()["files"] != nil {
+		if len(updateTorrent.FileList) > 0 {
+			return
 		}
+		err, torrentFiles = ScrapeFiles(format.InfoHashToMagnet(strings.TrimSpace(updateTorrent.Hash), updateTorrent.Name, Trackers...), updateTorrent, CurrentData, statsExists)
+		if err != nil {
+			return
+		}
+	} else {
+		//Single() returns an array which contain results for each torrent Hash it is fed, since we only feed him one we want to directly access the results
+		stats = goscrape.Single(Trackers, []string{
+		  updateTorrent.Hash,
+		})[0]
+		UpdateTorrentStats(updateTorrent, stats, CurrentData, []torrent.File{}, statsExists)
 	}
-
-	stats := goscrape.Single(Trackers, []string{
-	  torrent.Hash,
-	})[0]
-	//Single() returns an array which contain results for each torrent Hash it is fed, since we only feed him one we want to directly access the results
+
 
 	//If we put seeders on -1, the script instantly knows the fetching did not give any result, avoiding having to check all three stats below and in view.jet.html's javascript
-	if stats.Seeders == 0 && stats.Leechers == 0 && stats.Completed == 0  {
+	if isEmptyResult(stats) {
 		stats.Seeders = -1
 	}
 
 	c.JSON(200, gin.H{
  		"seeders": stats.Seeders,
  		"leechers": stats.Leechers,
  		"downloads": stats.Completed,
+		"filelist": torrentFiles,
+		"totalsize": fileSize(updateTorrent.Filesize),
  	})
 
+	return
+}
+
+// UpdateTorrentStats : Update stats & filelist if files are specified, otherwise just stats
+func UpdateTorrentStats(torrent *models.Torrent, stats goscrape.Result, currentStats models.Scrape, Files []torrent.File, statsExists bool) (JSONFilelist []FileJSON) {
 	if stats.Seeders == -1 {
 		stats.Seeders = 0
 	}
 
 	if !statsExists {
-		torrent.Scrape = torrent.Scrape.Create(uint(id), uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now())
-		//Create entry in the DB because none exist
+		torrent.Scrape = torrent.Scrape.Create(torrent.ID, uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now())
+		//Create a stat entry in the DB because none exist
 	} else {
 		//Entry in the DB already exists, simply update it
-		if (CurrentData.Seeders == 0 && CurrentData.Leechers == 0 && CurrentData.Completed == 0) || (stats.Seeders != 0 && stats.Leechers != 0 && stats.Completed != 0 ) {
-			torrent.Scrape = &models.Scrape{uint(id), uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now()}
+		if isEmptyScrape(currentStats) || !isEmptyResult(stats) {
+			torrent.Scrape = &models.Scrape{torrent.ID, uint32(stats.Seeders), uint32(stats.Leechers), uint32(stats.Completed), time.Now()}
 		} else {
-			torrent.Scrape = &models.Scrape{uint(id), uint32(CurrentData.Seeders), uint32(CurrentData.Leechers), uint32(CurrentData.Completed), time.Now()}
+			torrent.Scrape = &models.Scrape{torrent.ID, uint32(currentStats.Seeders), uint32(currentStats.Leechers), uint32(currentStats.Completed), time.Now()}
 		}
-		//Only overwrite stats if the old one are Unknown OR if the current ones are not unknown, preventing good stats from being turned into unknown own but allowing good stats to be updated to more reliable ones
+		//Only overwrite stats if the old one are Unknown OR if the new ones are not unknown, preventing good stats from being turned into unknown but allowing good stats to be updated to more reliable ones
 		torrent.Scrape.Update(false)
+	}
+
+	if len(Files) > 1 {
+		files, err := torrent.CreateFileList(Files)
 
+		if err != nil {
+			return
+		}
+
+		JSONFilelist = make([]FileJSON, 0, len(files))
+		for _, f := range files {
+			JSONFilelist = append(JSONFilelist, FileJSON{
+				Path:     filepath.Join(f.Path()...),
+				Filesize: fileSize(f.Filesize),
+			})
+		}
+
+		// Sort file list by lowercase filename
+		slice.Sort(JSONFilelist, func(i, j int) bool {
+			return strings.ToLower(JSONFilelist[i].Path) < strings.ToLower(JSONFilelist[j].Path)
+		})
+	} else if len(Files) == 1 {
+		torrent.Filesize = Files[0].Length()
+		torrent.Update(false)
 	}
 
 	return
 }
 
+// GetTorrentTrackers : Get the torrent trackers and add the default ones if they are missing
+func GetTorrentTrackers(torrent *models.Torrent) []string {
+	var Trackers []string
+	if len(torrent.Trackers) > 3 {
+		for _, line := range strings.Split(torrent.Trackers[3:], "&tr=") {
+			tracker, error := url.QueryUnescape(line)
+			if error == nil && strings.HasPrefix(tracker, "udp") {
+				Trackers = append(Trackers, tracker)
+			}
+			//Cannot scrape from http trackers only keep UDP ones
+		}
+	}
+
+	for _, tracker := range config.Get().Torrents.Trackers.Default {
+		if !contains(Trackers, tracker) && strings.HasPrefix(tracker, "udp") {
+			Trackers = append(Trackers, tracker)
+		}
+	}
+	return Trackers
+}
+
+func isEmptyResult(stats goscrape.Result) bool {
+	return stats.Seeders == 0 && stats.Leechers == 0 && stats.Completed == 0 
+}
+
+func isEmptyScrape(stats models.Scrape) bool {
+	return stats.Seeders == 0 && stats.Leechers == 0 && stats.Completed == 0 
+}
+
 func contains(s []string, e string) bool {
     for _, a := range s {
         if a == e {

models/file.go

@@ -52,12 +52,18 @@ func (f *File) SetPath(path []string) error {
 // Filename : Returns the filename of the file
 func (f *File) Filename() string {
 	path := f.Path()
+	if len(path) == 0 {
+		return ""
+	}
 	return path[len(path)-1]
 }
 
 // FilenameWithoutExtension : Returns the filename of the file without the extension
 func (f *File) FilenameWithoutExtension() string {
 	path := f.Path()
+	if len(path) == 0 {
+		return ""
+	}
 	fileName := path[len(path)-1]
 	index := strings.LastIndex(fileName, ".")
 
@@ -71,10 +77,13 @@ func (f *File) FilenameWithoutExtension() string {
 // FilenameExtension : Returns the extension of a filename, or an empty string
 func (f *File) FilenameExtension() string {
 	path := f.Path()
+	if len(path) == 0 {
+		return ""
+	}
 	fileName := path[len(path)-1]
 	index := strings.LastIndex(fileName, ".")
 
-	if index == -1 {
+	if index == -1 || index+1 == len(fileName){
 		return ""
 	}
 

models/torrent.go

@@ -21,6 +21,7 @@ import (
 	"github.com/NyaaPantsu/nyaa/utils/format"
 	"github.com/NyaaPantsu/nyaa/utils/log"
 	"github.com/NyaaPantsu/nyaa/utils/sanitize"
+	"github.com/anacrolix/torrent"
 	"github.com/bradfitz/slice"
 	"github.com/fatih/structs"
 )
@@ -457,6 +458,26 @@ func (t *Torrent) Update(unscope bool) (int, error) {
 	return http.StatusOK, nil
 }
 
+func (t *Torrent) CreateFileList(Files []torrent.File) ([]File, error) {
+	var createdFilelist []File
+	t.Filesize = 0
+
+	for _, uploadedFile := range Files {
+		file := File{TorrentID: t.ID, Filesize: uploadedFile.Length()}
+		err := file.SetPath(uploadedFile.FileInfo().Path)
+		if err != nil {
+			return []File{}, err
+		}
+		createdFilelist = append(createdFilelist, file)
+		t.Filesize  += uploadedFile.Length()
+		ORM.Create(&file)
+	}
+
+	t.FileList = createdFilelist
+	t.Update(false)
+	return createdFilelist, nil
+}
+
 // UpdateUnscope : Update a torrent based on model
 func (t *Torrent) UpdateUnscope() (int, error) {
 	return t.Update(true)