Add list template tags command (#4798)

* add list template tags command

* update readme

* misc changes to implementation

* misc

* misc update

---------

Co-authored-by: Sandeep Singh <[email protected]>
Co-authored-by: Ice3man <[email protected]>
Co-authored-by: sandeep <[email protected]>

README.md

@@ -137,6 +137,7 @@ TEMPLATES:
    -nss, -no-strict-syntax                disable strict syntax check on templates
    -td, -template-display                 displays the templates content
    -tl                                    list all available templates
+   -tgl                                   list all available tags
    -sign                                  signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable
    -code                                  enable loading code protocol-based templates
    -dut, -disable-unsigned-templates      disable running unsigned templates or templates with mismatched signature

cmd/nuclei/main.go

@@ -226,6 +226,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.NoStrictSyntax, "no-strict-syntax", "nss", false, "disable strict syntax check on templates"),
 		flagSet.BoolVarP(&options.TemplateDisplay, "template-display", "td", false, "displays the templates content"),
 		flagSet.BoolVar(&options.TemplateList, "tl", false, "list all available templates"),
+		flagSet.BoolVar(&options.TagList, "tgl", false, "list all available tags"),
 		flagSet.StringSliceVarConfigOnly(&options.RemoteTemplateDomainList, "remote-template-domain", []string{"cloud.projectdiscovery.io"}, "allowed domain list to load remote templates from"),
 		flagSet.BoolVar(&options.SignTemplates, "sign", false, "signs the templates with the private key defined in NUCLEI_SIGNATURE_PRIVATE_KEY env variable"),
 		flagSet.BoolVar(&options.EnableCodeTemplates, "code", false, "enable loading code protocol-based templates"),

internal/runner/runner.go

@@ -510,6 +510,23 @@ func (r *Runner) RunEnumeration() error {
 		return errors.Wrap(err, "Could not create loader.")
 	}
 
+	// list all templates or tags as specified by user.
+	// This uses a separate parser to reduce time taken as
+	// normally nuclei does a lot of compilation and stuff
+	// for templates, which we don't want for these simp
+	if r.options.TemplateList || r.options.TemplateDisplay || r.options.TagList {
+		if err := store.LoadTemplatesOnlyMetadata(); err != nil {
+			return err
+		}
+
+		if r.options.TagList {
+			r.listAvailableStoreTags(store)
+		} else {
+			r.listAvailableStoreTemplates(store)
+		}
+		os.Exit(0)
+	}
+
 	if r.options.Validate {
 		if err := store.ValidateTemplates(); err != nil {
 			return err
@@ -540,12 +557,6 @@ func (r *Runner) RunEnumeration() error {
 			_ = r.inputProvider.SetWithExclusions(host)
 		}
 	}
-	// list all templates
-	if r.options.TemplateList || r.options.TemplateDisplay {
-		r.listAvailableStoreTemplates(store)
-		os.Exit(0)
-	}
-
 	// display execution info like version , templates used etc
 	r.displayExecutionInfo(store)
 

internal/runner/templates.go

@@ -3,9 +3,11 @@ package runner
 import (
 	"bytes"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/alecthomas/chroma/quick"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/logrusorgru/aurora"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/loader"
@@ -71,6 +73,40 @@ func (r *Runner) listAvailableStoreTemplates(store *loader.Store) {
 	}
 }
 
+func (r *Runner) listAvailableStoreTags(store *loader.Store) {
+	gologger.Print().Msgf(
+		"\nListing available %v nuclei tags for %v",
+		config.DefaultConfig.TemplateVersion,
+		config.DefaultConfig.TemplatesDirectory,
+	)
+	tagsMap := make(map[string]int)
+	for _, tpl := range store.Templates() {
+		for _, tag := range tpl.Info.Tags.ToSlice() {
+			tagsMap[tag]++
+		}
+	}
+	type kv struct {
+		Key   string `json:"tag"`
+		Value int    `json:"count"`
+	}
+	var tagsList []kv
+	for k, v := range tagsMap {
+		tagsList = append(tagsList, kv{k, v})
+	}
+	sort.Slice(tagsList, func(i, j int) bool {
+		return tagsList[i].Value > tagsList[j].Value
+	})
+
+	for _, tag := range tagsList {
+		if r.options.JSONL {
+			marshalled, _ := jsoniter.Marshal(tag)
+			gologger.Silent().Msgf("%s\n", string(marshalled))
+		} else {
+			gologger.Silent().Msgf("%s (%d)\n", tag.Key, tag.Value)
+		}
+	}
+}
+
 func (r *Runner) highlightTemplate(body *[]byte) ([]byte, error) {
 	var buf bytes.Buffer
 	// YAML lexer, true color terminal formatter and monokai style

pkg/catalog/loader/loader.go

@@ -255,6 +255,46 @@ func init() {
 	templateIDPathMap = make(map[string]string)
 }
 
+// LoadTemplatesOnlyMetadata loads only the metadata of the templates
+func (store *Store) LoadTemplatesOnlyMetadata() error {
+	templatePaths, errs := store.config.Catalog.GetTemplatesPath(store.finalTemplates)
+	store.logErroredTemplates(errs)
+
+	filteredTemplatePaths := store.pathFilter.Match(templatePaths)
+
+	validPaths := make(map[string]struct{})
+	for templatePath := range filteredTemplatePaths {
+		loaded, err := store.config.ExecutorOptions.Parser.LoadTemplate(templatePath, store.tagFilter, nil, store.config.Catalog)
+		if loaded || store.pathFilter.MatchIncluded(templatePath) {
+			validPaths[templatePath] = struct{}{}
+		}
+		if err != nil {
+			if strings.Contains(err.Error(), templates.ErrExcluded.Error()) {
+				stats.Increment(templates.TemplatesExcludedStats)
+				if config.DefaultConfig.LogAllEvents {
+					gologger.Print().Msgf("[%v] %v\n", aurora.Yellow("WRN").String(), err.Error())
+				}
+				continue
+			}
+			gologger.Warning().Msg(err.Error())
+		}
+	}
+	parserItem, ok := store.config.ExecutorOptions.Parser.(*templates.Parser)
+	if !ok {
+		return errors.New("invalid parser")
+	}
+	templatesCache := parserItem.Cache()
+
+	for templatePath := range validPaths {
+		template, _, _ := templatesCache.Has(templatePath)
+		if template != nil {
+			template.Path = templatePath
+			store.templates = append(store.templates, template)
+		}
+	}
+	return nil
+}
+
 // ValidateTemplates takes a list of templates and validates them
 // erroring out on discovering any faulty templates.
 func (store *Store) ValidateTemplates() error {

pkg/templates/parser.go

@@ -33,6 +33,11 @@ func NewParser() *Parser {
 	return p
 }
 
+// Cache returns the parsed templates cache
+func (p *Parser) Cache() *Cache {
+	return p.parsedTemplatesCache
+}
+
 // LoadTemplate returns true if the template is valid and matches the filtering criteria.
 func (p *Parser) LoadTemplate(templatePath string, t any, extraTags []string, catalog catalog.Catalog) (bool, error) {
 	tagFilter, ok := t.(*TagFilter)

pkg/types/types.go

@@ -225,6 +225,8 @@ type Options struct {
 	TemplateDisplay bool
 	// TemplateList lists available templates
 	TemplateList bool
+	// TemplateList lists available tags
+	TagList bool
 	// HangMonitor enables nuclei hang monitoring
 	HangMonitor bool
 	// Stdin specifies whether stdin input was given to the process