diff --git a/.github/README.md b/.github/README.md index c8b1046b..9548a15f 100644 --- a/.github/README.md +++ b/.github/README.md @@ -86,6 +86,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | Set the output file. The result defaults to json format. | `-out output.json` | | `db` | `string` | Set the local vulnerability database file. It helps when you prefer to use your own vulnerability database. The format of the vulnerability database is shown below. If the cloud and local vulnerability databases are both set, the result of detection will merge both. | `-db db.json` | | `progress` | `bool` | Show the progress bar. | `-progress` | +| `dedup` | `bool` | Same result deduplication | `-dedup` | ------ @@ -142,7 +143,7 @@ opensca-cli -db db.json -path ${project_path} OpenSCA is an open source project, we appreciate your help! -To contribute, please read our [Contributing Guideline](./docs/Contributing%20Guideline-en%20v1.0.md). +To contribute, please read our [Contributing Guideline](../docs/Contributing%20Guideline-en%20v1.0.md). diff --git a/README.md b/README.md index bc274800..24a79fe1 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | 将检测结果保存到指定文件，根据后缀生成不同格式的文件，默认为 `json` 格式 | `-out output.json` | | `db` | `string` | 指定本地漏洞库文件，希望使用自己漏洞库时可用，漏洞库文件为 `json` 格式，具体格式会在之后给出;若同时使用云端漏洞库与本地漏洞库，漏洞查询结果取并集 | `-db db.json` | | `progress` | `bool` | 显示进度条 | `-progress` | +| `dedup` | `bool` | 相同组件去重 | `-dedup` | --- diff --git a/analyzer/engine/archive.go b/analyzer/engine/archive.go index 11da4c35..88d97a92 100644 --- a/analyzer/engine/archive.go +++ b/analyzer/engine/archive.go @@ -17,6 +17,7 @@ import ( "util/filter" "util/logs" "util/model" + "util/temp" "github.com/axgle/mahonia" "github.com/mholt/archiver" @@ -39,7 +40,6 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { filepath = strings.ReplaceAll(filepath, `\`, `/`) // 目录树根 root = model.NewDirTree() - root.Path = path.Base(filepath) var walker archiver.Walker if filter.Tar(filepath) { walker = archiver.NewTar() @@ -86,35 +86,27 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { // 支持解析的文件 root.AddFile(model.NewFileData(fileName, data)) } else if filter.AllPkg(fileName) { - // 支持检测的压缩包 - rootPath, _ := os.Executable() - rootPath = path.Dir(strings.ReplaceAll(rootPath, `\`, `/`)) - tempPath := path.Join(rootPath, ".temp_path") - // 创建临时文件夹 - os.Mkdir(tempPath, os.ModeDir) - targetPath := path.Join(tempPath, path.Base(fileName)) // 将压缩包解压到本地 - if out, err := os.Create(targetPath); err == nil { - _, err = out.Write(data) - out.Close() - if err != nil { - return errors.WithStack(err) - } - // 获取当前目录树 - dir := root.GetDir(fileName) - name := path.Base(fileName) - if _, ok := dir.SubDir[name]; !ok { - // 将压缩包的内容添加到当前目录树 - dir.DirList = append(dir.DirList, name) - dir.SubDir[name] = e.unArchiveFile(targetPath) - } - // 删除压缩包 - if err = os.Remove(targetPath); err != nil { + temp.DoInTempDir(func(tempdir string) { + targetPath := path.Join(tempdir, path.Base(fileName)) + if out, err := os.Create(targetPath); err == nil { + _, err = out.Write(data) + out.Close() + if err != nil { + logs.Error(err) + } + // 获取当前目录树 + dir := root.GetDir(fileName) + name := path.Base(fileName) + if _, ok := dir.SubDir[name]; !ok { + // 将压缩包的内容添加到当前目录树 + dir.DirList = append(dir.DirList, name) + dir.SubDir[name] = e.unArchiveFile(targetPath) + } + } else { logs.Error(err) } - } else { - logs.Error(err) - } + }) } } return nil diff --git a/analyzer/engine/engine.go b/analyzer/engine/engine.go index ed67841b..4a065050 100644 --- a/analyzer/engine/engine.go +++ b/analyzer/engine/engine.go @@ -24,6 +24,7 @@ import ( "analyzer/java" "analyzer/javascript" "analyzer/php" + "analyzer/python" "analyzer/ruby" "analyzer/rust" ) @@ -43,6 +44,9 @@ func NewEngine() Engine { rust.New(), golang.New(), erlang.New(), + // 暂不解析groovy文件 + // groovy.New(), + python.New(), }, } } @@ -97,7 +101,7 @@ func (e Engine) ParseFile(filepath string) (depRoot *model.DepTree, taskInfo rep // 获取漏洞 taskInfo.Error = vuln.SearchVuln(depRoot) // 是否仅保留漏洞组件 - if args.OnlyVuln { + if args.Config.OnlyVuln { root := model.NewDepTree(nil) q := model.NewQueue() q.Push(depRoot) diff --git a/analyzer/engine/parse.go b/analyzer/engine/parse.go index 5f61b5dd..9049daa3 100644 --- a/analyzer/engine/parse.go +++ b/analyzer/engine/parse.go @@ -7,6 +7,7 @@ package engine import ( "path" + "strings" "util/filter" "util/model" ) @@ -36,7 +37,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for _, d := range analyzer.ParseFiles(files) { depRoot.Children = append(depRoot.Children, d) d.Parent = depRoot - if d.Name != "" && d.Version.Ok() { + if d.Name != "" && !strings.ContainsAny(d.Vendor+d.Name, "${}") && d.Version.Ok() { d.Path = path.Join(d.Path, d.Dependency.String()) } // 标识为直接依赖 @@ -73,7 +74,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for len(q) > 0 { n := q[0] q = append(q[1:], n.Children...) - if n.Name == "" || !n.Version.Ok() { + if n.Name == "" || strings.ContainsAny(n.Vendor+n.Name, "${}") || !n.Version.Ok() { n.Move(n.Parent) } } diff --git a/analyzer/java/analyzer.go b/analyzer/java/analyzer.go index 6087b26d..10c058e8 100644 --- a/analyzer/java/analyzer.go +++ b/analyzer/java/analyzer.go @@ -35,7 +35,7 @@ func (Analyzer) GetLanguage() language.Type { // CheckFile Check if it is a parsable file func (Analyzer) CheckFile(filename string) bool { - return filter.JavaPom(filename) + return filter.JavaPom(filename) || filter.GroovyGradle(filename) } // pomTree pom文件树 @@ -182,9 +182,15 @@ func (a Analyzer) ParseFiles(files []*model.FileInfo) (deps []*model.DepTree) { p.Path = f.Name poms = append(poms, p) } + if filter.GroovyGradle(f.Name) { + dep := model.NewDepTree(nil) + dep.Path = f.Name + parseGradle(dep, f) + deps = append(deps, dep) + } } // 构建jar树 - deps = buildJarTree(jarMap) + deps = append(deps, buildJarTree(jarMap)...) // 构建pom树 deps = append(deps, buildPomTree(poms).parsePomTree(jarMap)...) return diff --git a/analyzer/java/ext.go b/analyzer/java/ext.go index cc447808..2afd04fa 100644 --- a/analyzer/java/ext.go +++ b/analyzer/java/ext.go @@ -18,17 +18,14 @@ import ( "util/enum/language" "util/logs" "util/model" + "util/temp" "github.com/pkg/errors" ) // MvnDepTree 调用mvn解析项目获取依赖树 func MvnDepTree(path string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(path) cmd := exec.Command("mvn", "dependency:tree", "--fail-never") out, _ := cmd.CombinedOutput() diff --git a/analyzer/java/gradle.go b/analyzer/java/gradle.go index 51f5e571..673cae18 100644 --- a/analyzer/java/gradle.go +++ b/analyzer/java/gradle.go @@ -6,9 +6,12 @@ import ( "encoding/json" "os" "os/exec" + "regexp" + "strings" "util/enum/language" "util/logs" "util/model" + "util/temp" ) //go:embed oss.gradle @@ -26,14 +29,10 @@ type gradleDep struct { // GradleDepTree 尝试获取 gradle 依赖树 func GradleDepTree(dirpath string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(dirpath) // 复制 oss.gradle - if err = os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { + if err := os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { logs.Warn(err) return } @@ -52,7 +51,7 @@ func GradleDepTree(dirpath string, root *model.DepTree) { data := out[startIndex+len(startTag) : endIndex] out = out[endIndex+1:] gdep := &gradleDep{MapDep: model.NewDepTree(root)} - err = json.Unmarshal(data, &gdep.Children) + err := json.Unmarshal(data, &gdep.Children) if err != nil { logs.Warn(err) } @@ -78,3 +77,35 @@ func GradleDepTree(dirpath string, root *model.DepTree) { } return } + +// parseGradle parse *.gradle or *.gradle.kts +func parseGradle(root *model.DepTree, file *model.FileInfo) { + regexs := []*regexp.Regexp{ + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?name: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?module: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`['"]([^\s:'"]+):([^\s:'"]+):([^\s:'"]+)['"]`), + } + for _, line := range strings.Split(string(file.Data), "\n") { + for _, re := range regexs { + match := re.FindStringSubmatch(line) + // 有捕获内容 + if len(match) == 4 && + // 不以注释开头 + !strings.HasPrefix(strings.TrimSpace(line), "/") && + // 不是测试组件 + !strings.Contains(strings.ToLower(line), "testimplementation") && + // 去掉非组件内容 + !strings.Contains(line, "//") { + ver := model.NewVersion(match[3]) + // 版本号正常 + if ver.Ok() { + dep := model.NewDepTree(root) + dep.Vendor = match[1] + dep.Name = match[2] + dep.Version = ver + break + } + } + } + } +} diff --git a/analyzer/java/pom.go b/analyzer/java/pom.go index ee4d686a..61ce5de0 100644 --- a/analyzer/java/pom.go +++ b/analyzer/java/pom.go @@ -129,7 +129,7 @@ func (p *Pom) GetProperty(key string) string { return p.Version case "${project.groupId}", "${groupId}", "${pom.groupId}": return p.GroupId - case "${project.artifactId}": + case "${project.artifactId}", "${artifactId}", "${pom.artifactId}": return p.ArtifactId case "${project.parent.version}", "${parent.version}": return p.Parent.Version diff --git a/analyzer/python/analyzer.go b/analyzer/python/analyzer.go new file mode 100644 index 00000000..047d3528 --- /dev/null +++ b/analyzer/python/analyzer.go @@ -0,0 +1,44 @@ +package python + +import ( + "util/enum/language" + "util/filter" + "util/model" +) + +type Analyzer struct { +} + +func New() Analyzer { + return Analyzer{} +} + +// GetLanguage get language of analyzer +func (Analyzer) GetLanguage() language.Type { + return language.Python +} + +// CheckFile check parsable file +func (Analyzer) CheckFile(filename string) bool { + return filter.PythonSetup(filename) || + filter.PythonPipfile(filename) || + filter.PythonPipfileLock(filename) +} + +// ParseFiles parse dependency from file +func (Analyzer) ParseFiles(files []*model.FileInfo) []*model.DepTree { + deps := []*model.DepTree{} + for _, f := range files { + dep := model.NewDepTree(nil) + dep.Path = f.Name + if filter.PythonSetup(f.Name) { + parseSetup(dep, f) + } else if filter.PythonPipfile(f.Name) { + parsePipfile(dep, f) + } else if filter.PythonPipfileLock(f.Name) { + parsePipfileLock(dep, f) + } + deps = append(deps, dep) + } + return deps +} diff --git a/analyzer/python/oss.py b/analyzer/python/oss.py new file mode 100644 index 00000000..4179e668 --- /dev/null +++ b/analyzer/python/oss.py @@ -0,0 +1,34 @@ +import re +import sys +import json + +def parse_setup_py(setup_py_path): + """解析setup.py文件""" + with open(setup_py_path, "r") as f: + pass_func = lambda **x: x + try: + import distutils + distutils.core.setup = pass_func + except Exception: + pass + try: + import setuptools + setuptools.setup = pass_func + except Exception: + pass + # 获取setup参数 + args = {} + code = re.sub('(?>oss_end'.format(json.dumps(info))) + +if __name__ == "__main__": + if len(sys.argv) > 1: + parse_setup_py(sys.argv[1]) \ No newline at end of file diff --git a/analyzer/python/pipfile.go b/analyzer/python/pipfile.go new file mode 100644 index 00000000..4b720c9c --- /dev/null +++ b/analyzer/python/pipfile.go @@ -0,0 +1,56 @@ +package python + +import ( + "encoding/json" + "util/logs" + "util/model" + + "github.com/BurntSushi/toml" +) + +// parsePipfile parse Pipfile file +func parsePipfile(root *model.DepTree, file *model.FileInfo) { + pip := struct { + DevPackages map[string]string `toml:"dev-packages"` + Packages map[string]string `toml:"packages"` + }{} + if err := toml.Unmarshal(file.Data, &pip); err != nil { + logs.Warn(err) + } + for name, version := range pip.Packages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } + for name, version := range pip.DevPackages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } +} + +// parsePipfileLock parse pipfile.lock file +func parsePipfileLock(root *model.DepTree, file *model.FileInfo) { + lock := struct { + Default map[string]struct { + Version string `json:"version"` + } `json:"default"` + }{} + err := json.Unmarshal(file.Data, &lock) + if err != nil { + logs.Warn(err) + } + names := []string{} + for n := range lock.Default { + names = append(names, n) + } + for _, n := range names { + v := lock.Default[n].Version + if v != "" { + dep := model.NewDepTree(root) + dep.Name = n + dep.Version = model.NewVersion(v) + } + } + return +} diff --git a/analyzer/python/setup.go b/analyzer/python/setup.go new file mode 100644 index 00000000..5ce39505 --- /dev/null +++ b/analyzer/python/setup.go @@ -0,0 +1,75 @@ +package python + +import ( + _ "embed" + "encoding/json" + "os" + "os/exec" + "path" + "strings" + "util/logs" + "util/model" + "util/temp" +) + +//go:embed oss.py +var ossPy []byte + +// oss.py 脚本输出的依赖结构 +type setupDep struct { + Name string `json:"name"` + Version string `json:"version"` + License string `json:"license"` + Packages []string `json:"packages"` + InstallRequires []string `json:"install_requires"` + Requires []string `json:"requires"` +} + +// parseSetup 解析 setup.py 文件 +func parseSetup(root *model.DepTree, file *model.FileInfo) { + temp.DoInTempDir(func(tempdir string) { + ossfile := path.Join(tempdir, "oss.py") + setupfile := path.Join(tempdir, "setup.py") + // 创建 oss.py + if err := os.WriteFile(ossfile, ossPy, 0444); err != nil { + logs.Warn(err) + return + } + // 创建 setup.py + if err := os.WriteFile(setupfile, file.Data, 0444); err != nil { + logs.Warn(err) + return + } + // 解析 setup.py + cmd := exec.Command("python", ossfile, setupfile) + out, _ := cmd.CombinedOutput() + startTag, endTag := `oss_start<<`, `>>oss_end` + startIndex, endIndex := strings.Index(string(out), startTag), strings.Index(string(out), endTag) + if startIndex == -1 || endIndex == -1 { + return + } else { + out = out[startIndex+len(startTag) : endIndex] + } + // 获取解析结果 + var dep setupDep + if err := json.Unmarshal(out, &dep); err != nil { + logs.Warn(err) + } + root.Name = dep.Name + root.Version = model.NewVersion(dep.Version) + root.Licenses = append(root.Licenses, dep.License) + for _, pkg := range [][]string{dep.Packages, dep.InstallRequires, dep.Requires} { + for _, p := range pkg { + index := strings.IndexAny(p, "=<>") + sub := model.NewDepTree(root) + if index > -1 { + sub.Name = p[:index] + sub.Version = model.NewVersion(p[index:]) + } else { + sub.Name = p + } + } + } + }) + return +} diff --git a/cli/main.go b/cli/main.go index 6d36bf58..796942c5 100644 --- a/cli/main.go +++ b/cli/main.go @@ -15,10 +15,12 @@ import ( "util/report" ) +var version string + func main() { args.Parse() - if len(args.Filepath) > 0 { - output(engine.NewEngine().ParseFile(args.Filepath)) + if len(args.Config.Path) > 0 { + output(engine.NewEngine().ParseFile(args.Config.Path)) } else { flag.PrintDefaults() } @@ -26,11 +28,12 @@ func main() { // output 输出结果 func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { + taskInfo.ToolVersion = version // 记录依赖 logs.Debug("\n" + depRoot.String()) // 输出结果 var reportFunc func(*model.DepTree, report.TaskInfo) []byte - switch path.Ext(args.Out) { + switch path.Ext(args.Config.Out) { case ".html": reportFunc = report.Html case ".json": @@ -38,8 +41,8 @@ func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { default: reportFunc = report.Json } - if args.Out != "" { - report.Save(reportFunc(depRoot, taskInfo), args.Out) + if args.Config.Out != "" { + report.Save(reportFunc(depRoot, taskInfo), args.Config.Out) } else { fmt.Println(string(reportFunc(depRoot, taskInfo))) } diff --git a/config.json b/config.json index 9381c40c..63fc6672 100644 --- a/config.json +++ b/config.json @@ -6,5 +6,6 @@ "out": "output.json", "cache": true, "vuln": false, - "progress": true + "progress": true, + "dedup": true } \ No newline at end of file diff --git a/util/args/args.go b/util/args/args.go index e9ae45ff..d0f54063 100644 --- a/util/args/args.go +++ b/util/args/args.go @@ -6,52 +6,64 @@ package args import ( + "encoding/json" "flag" + "fmt" + "io/ioutil" + "path" "strings" + "util/temp" ) var ( - // 配置文件路径 - Config string - // 解析文件路径 - Filepath string - // 云服务地址 - Url string - IP string - // 云服务token - Token string - // 开启本地缓存 - Cache bool - // 输出文件 - Out string - // 仅展示有漏洞的组件 - OnlyVuln bool - // 本地漏洞库文件路径 - VulnDB string - // display progress bar - ProgressBar bool + ConfigPath string + Config = struct { + // detect option + Path string `json:"path"` + Out string `json:"out"` + Cache bool `json:"cache"` + Bar bool `json:"progress"` + OnlyVuln bool `json:"vuln"` + Dedup bool `json:"dedup"` + // remote vuldb + Url string `json:"url"` + Token string `json:"token"` + // local vuldb + VulnDB string `json:"db"` + }{} ) func init() { - // 设置参数信息 - flag.StringVar(&Config, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") - flag.StringVar(&Filepath, "path", "", "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") - flag.StringVar(&Url, "url", "", "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") - flag.StringVar(&IP, "ip", "", "(待废弃,删除)与url作用相同,兼容旧版本参数") - flag.StringVar(&Token, "token", "", "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") - flag.BoolVar(&Cache, "cache", false, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") - flag.BoolVar(&OnlyVuln, "vuln", false, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") - flag.StringVar(&Out, "out", "", "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") - flag.StringVar(&VulnDB, "db", "", "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") - flag.BoolVar(&ProgressBar, "progress", false, "(可选) 显示进度条") + flag.StringVar(&ConfigPath, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") + flag.StringVar(&Config.Path, "path", Config.Path, "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") + flag.StringVar(&Config.Url, "url", Config.Url, "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") + flag.StringVar(&Config.Token, "token", Config.Token, "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") + flag.BoolVar(&Config.Cache, "cache", Config.Cache, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") + flag.BoolVar(&Config.OnlyVuln, "vuln", Config.OnlyVuln, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") + flag.StringVar(&Config.Out, "out", Config.Out, "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") + flag.StringVar(&Config.VulnDB, "db", Config.VulnDB, "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") + flag.BoolVar(&Config.Bar, "progress", Config.Bar, "(可选) 显示进度条") + flag.BoolVar(&Config.Dedup, "dedup", Config.Dedup, "(可选) 相同组件去重") } func Parse() { flag.Parse() - // 兼容旧版本，待废弃 - if Url == "" && IP != "" { - Url = IP + if ConfigPath != "" { + if data, err := ioutil.ReadFile(ConfigPath); err != nil { + fmt.Printf("load config file error: %s\n", err) + } else { + if err = json.Unmarshal(data, &Config); err != nil { + fmt.Printf("parse config file error: %s\n", err) + } + } + } else { + // 默认读取目录下的config.json文件 + if data, err := ioutil.ReadFile(path.Join(temp.GetPwd(), "config.json")); err == nil { + // 不处理错误 + json.Unmarshal(data, &Config) + } } - loadConfigFile() - Url = strings.TrimSuffix(Url, "/") + // 再次调用Parse, 优先使用cli参数 + flag.Parse() + Config.Url = strings.TrimSuffix(Config.Url, "/") } diff --git a/util/args/config.go b/util/args/config.go deleted file mode 100644 index 4456d84b..00000000 --- a/util/args/config.go +++ /dev/null @@ -1,63 +0,0 @@ -package args - -import ( - "encoding/json" - "os" - "util/logs" -) - -// loadConfigFile 加载配置文件 -func loadConfigFile() bool { - configFilePath := Config - if configFilePath == "" { - return false - } - if _, err := os.Stat(configFilePath); err != nil { - logs.Error(err) - return false - } - if data, err := os.ReadFile(configFilePath); err != nil { - logs.Error(err) - return false - } else { - config := struct { - Path string `json:"path"` - DB string `json:"db"` - Url string `json:"url"` - Token string `json:"token"` - Out string `json:"out"` - Cache *bool `json:"cache"` - OnlyVuln *bool `json:"vuln"` - ProgressBar *bool `json:"progress"` - }{} - if err = json.Unmarshal(data, &config); err != nil { - logs.Error(err) - return false - } - if Filepath == "" && config.Path != "" { - Filepath = config.Path - } - if VulnDB == "" && config.DB != "" { - VulnDB = config.DB - } - if Url == "" && config.Url != "" { - Url = config.Url - } - if Token == "" && config.Token != "" { - Token = config.Token - } - if Out == "" && config.Out != "" { - Out = config.Out - } - if !Cache && config.Cache != nil { - Cache = *config.Cache - } - if !OnlyVuln && config.OnlyVuln != nil { - OnlyVuln = *config.OnlyVuln - } - if !ProgressBar && config.ProgressBar != nil { - ProgressBar = *config.ProgressBar - } - return true - } -} diff --git a/util/bar/bar.go b/util/bar/bar.go index e7627b01..9085e4ee 100644 --- a/util/bar/bar.go +++ b/util/bar/bar.go @@ -37,7 +37,7 @@ func newBar(text string) *Bar { // Add add progress func (b *Bar) Add(n int) { - if !args.ProgressBar { + if !args.Config.Bar { return } if b.id == -1 { diff --git a/util/cache/cache.go b/util/cache/cache.go index 82011341..7e44a4d0 100644 --- a/util/cache/cache.go +++ b/util/cache/cache.go @@ -33,7 +33,7 @@ func init() { // save save cache file func save(filepath string, data []byte) { - if args.Cache { + if args.Config.Cache { if err := os.MkdirAll(path.Join(cacheDir, path.Dir(filepath)), os.ModeDir); err == nil { if f, err := os.Create(path.Join(cacheDir, filepath)); err == nil { defer f.Close() @@ -45,7 +45,7 @@ func save(filepath string, data []byte) { // load load cache file func load(filepath string) []byte { - if args.Cache { + if args.Config.Cache { if data, err := ioutil.ReadFile(path.Join(cacheDir, filepath)); err == nil { return data } else { diff --git a/util/client/client.go b/util/client/client.go index 046eb0a0..c5a76607 100644 --- a/util/client/client.go +++ b/util/client/client.go @@ -20,6 +20,7 @@ import ( "regexp" "util/args" "util/logs" + "util/temp" "github.com/pkg/errors" ) @@ -60,35 +61,31 @@ type DetectRequst struct { func GetClientId() string { // 默认id id := "XXXXXXXXXXXXXXXX" - if pwd, err := os.Getwd(); err != nil { - logs.Error(err) - } else { - // 尝试读取.key文件 - idFile := path.Join(pwd, ".key") - if _, err := os.Stat(idFile); err != nil { - // 文件不存在则生成随机ID并保存 - if f, err := os.Create(idFile); err != nil { - logs.Error(err) - } else { - defer f.Close() - const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - idbyte := []byte(id) - for i := range idbyte { - idbyte[i] = chars[mrand.Intn(26)] - } - f.Write(idbyte) - id = string(idbyte) - } + // 尝试读取.key文件 + idFile := path.Join(temp.GetPwd(), ".key") + if _, err := os.Stat(idFile); err != nil { + // 文件不存在则生成随机ID并保存 + if f, err := os.Create(idFile); err != nil { + logs.Error(err) } else { - // 文件存在则读取ID - idbyte, err := os.ReadFile(idFile) - if err != nil { - logs.Error(err) + defer f.Close() + const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + idbyte := []byte(id) + for i := range idbyte { + idbyte[i] = chars[mrand.Intn(26)] } - if len(idbyte) == 16 { - if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { - id = string(idbyte) - } + f.Write(idbyte) + id = string(idbyte) + } + } else { + // 文件存在则读取ID + idbyte, err := os.ReadFile(idFile) + if err != nil { + logs.Error(err) + } + if len(idbyte) == 16 { + if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { + id = string(idbyte) } } } @@ -109,11 +106,11 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // aes加密 ciphertext, tag := encrypt(reqbody, key, nonce) // 构建请求 - url := args.Url + "/oss-saas/api-v1/open-sca-client/detect" + url := args.Config.Url + "/oss-saas/api-v1/open-sca-client/detect" // 添加参数 param := DetectRequst{} param.ClientId = GetClientId() - param.Token = args.Token + param.Token = args.Config.Token param.Tag = base64.StdEncoding.EncodeToString(tag) param.Nonce = base64.StdEncoding.EncodeToString(nonce) // base64编码 @@ -168,14 +165,14 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // getAesKey 获取aes-key func getAesKey() (key []byte, err error) { - u, err := url.Parse(args.Url + "/oss-saas/api-v1/open-sca-client/aes-key") + u, err := url.Parse(args.Config.Url + "/oss-saas/api-v1/open-sca-client/aes-key") if err != nil { return key, err } // 设置参数 param := url.Values{} param.Set("clientId", GetClientId()) - param.Set("ossToken", args.Token) + param.Set("ossToken", args.Config.Token) u.RawQuery = param.Encode() // 发送请求 rep, err := http.Get(u.String()) diff --git a/util/enum/language/language.go b/util/enum/language/language.go index e132168d..94c71755 100644 --- a/util/enum/language/language.go +++ b/util/enum/language/language.go @@ -21,6 +21,7 @@ const ( Golang Rust Erlang + Python ) // String 语言类型 @@ -42,6 +43,8 @@ func (l Type) String() string { return "Rust" case Erlang: return "Erlang" + case Python: + return "Python" default: return "None" } @@ -66,6 +69,8 @@ func (l Type) Vuln() string { return "rust" case Erlang: return "" + case Python: + return "python" default: return "" } @@ -84,6 +89,7 @@ func init() { lm[Golang] = []string{"golang", "go", "gomod"} lm[Rust] = []string{"rust", "cargo"} lm[Erlang] = []string{"erlang", "rebar"} + lm[Python] = []string{"python", "pip", "pipy"} for t, ls := range lm { for _, l := range ls { lanMap[l] = t diff --git a/util/filter/file.go b/util/filter/file.go index 331c1a94..81ffe09f 100644 --- a/util/filter/file.go +++ b/util/filter/file.go @@ -82,5 +82,13 @@ var ( // groovy var ( - GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyGradle = filterFunc(strings.HasSuffix, ".gradle", ".gradle.kts") +) + +// python +var ( + PythonSetup = filterFunc(strings.HasSuffix, "setup.py") + PythonPipfile = filterFunc(strings.HasSuffix, "Pipfile") + PythonPipfileLock = filterFunc(strings.HasSuffix, "Pipfile.lock") ) diff --git a/util/model/dependency.go b/util/model/dependency.go index f6c5ba72..cfcfede6 100644 --- a/util/model/dependency.go +++ b/util/model/dependency.go @@ -75,7 +75,8 @@ type DepTree struct { // 是否为直接依赖 Direct bool `json:"direct"` // 依赖路径 - Path string `json:"path,omitempty"` + Path string `json:"-"` + Paths []string `json:"paths,omitempty"` // 唯一的组件id，用来标识不同组件 ID int64 `json:"-"` // 父组件 @@ -97,6 +98,7 @@ func NewDepTree(parent *DepTree) *DepTree { Dependency: NewDependency(), Vulnerabilities: []*Vuln{}, Path: "", + Paths: nil, Parent: parent, Children: []*DepTree{}, licenseMap: map[string]struct{}{}, diff --git a/util/model/version.go b/util/model/version.go index 1808e86f..01df4e2a 100644 --- a/util/model/version.go +++ b/util/model/version.go @@ -40,7 +40,7 @@ func (ver *Version) weight() (weight int) { func NewVersion(verStr string) *Version { verStr = strings.TrimSpace(verStr) ver := &Version{Nums: []int{}, Org: verStr} - verStr = strings.TrimLeft(verStr, "vV^") + verStr = strings.TrimLeft(verStr, "vV^~=<>") // 获取后缀 index := strings.Index(verStr, "-") if index != -1 { diff --git a/util/report/format.go b/util/report/format.go index 641e2962..4da0940e 100644 --- a/util/report/format.go +++ b/util/report/format.go @@ -1,8 +1,9 @@ package report import ( + "fmt" "os" - "strings" + "util/args" "util/enum/language" "util/logs" "util/model" @@ -10,6 +11,7 @@ import ( // 任务检查信息 type TaskInfo struct { + ToolVersion string `json:"tool_version"` AppName string `json:"app_name"` Size int64 `json:"size"` StartTime string `json:"start_time"` @@ -22,18 +24,52 @@ type TaskInfo struct { // format 按照输出内容格式化(不可逆) func format(dep *model.DepTree) { q := []*model.DepTree{dep} + // 保留要导出的数据 for len(q) > 0 { - node := q[0] - q = append(q[1:], node.Children...) - if node.Language != language.None { - node.LanguageStr = node.Language.String() + n := q[0] + q = append(q[1:], n.Children...) + if n.Language != language.None { + n.LanguageStr = n.Language.String() } - if node.Version != nil { - node.VersionStr = node.Version.Org + if n.Version != nil { + n.VersionStr = n.Version.Org + } + if n.Path != "" { + n.Paths = []string{n.Path} + } + n.Language = language.None + n.Version = nil + } + // 去重 + if args.Config.Dedup { + q = []*model.DepTree{dep} + dm := map[string]*model.DepTree{} + for len(q) > 0 { + n := q[0] + q = append(q[1:], n.Children...) + // 去重 + k := fmt.Sprintf("%s:%s@%s#%s", n.Vendor, n.Name, n.VersionStr, n.LanguageStr) + if d, ok := dm[k]; !ok { + dm[k] = n + } else { + // 已存在相同组件 + d.Paths = append(d.Paths, n.Path) + // 从父组件中移除当前组件 + if n.Parent != nil { + for i, c := range n.Parent.Children { + if c.ID == n.ID { + n.Parent.Children = append(n.Parent.Children[:i], n.Parent.Children[i+1:]...) + break + } + } + } + // 将当前组件的子组件转移到已存在组件的子依赖中 + d.Children = append(d.Children, n.Children...) + for _, c := range n.Children { + c.Parent = d + } + } } - node.Path = node.Path[strings.Index(node.Path, "/")+1:] - node.Language = language.None - node.Version = nil } } diff --git a/util/report/index.html b/util/report/index.html index 76160e38..90a4ce35 100644 --- a/util/report/index.html +++ b/util/report/index.html @@ -1,2 +1,2 @@ -OpenSCA开源组件检测报告
\ No newline at end of file +OpenSCA开源组件检测报告
\ No newline at end of file diff --git a/util/report/json.go b/util/report/json.go index e92a317f..addc7022 100644 --- a/util/report/json.go +++ b/util/report/json.go @@ -13,11 +13,11 @@ func Json(dep *model.DepTree, taskInfo TaskInfo) []byte { taskInfo.ErrorString = taskInfo.Error.Error() } if data, err := json.Marshal(struct { - *model.DepTree TaskInfo TaskInfo `json:"task_info"` + *model.DepTree }{ - DepTree: dep, TaskInfo: taskInfo, + DepTree: dep, }); err != nil { logs.Error(err) } else { diff --git a/util/temp/temp.go b/util/temp/temp.go new file mode 100644 index 00000000..765a1874 --- /dev/null +++ b/util/temp/temp.go @@ -0,0 +1,37 @@ +package temp + +import ( + "os" + "path" + "strconv" + "strings" + "time" + "util/logs" +) + +const tempdir = ".temp" + +func init() { + os.RemoveAll(path.Join(GetPwd(), tempdir)) +} + +// GetPwd 获取当前目录 +func GetPwd() string { + filepath, err := os.Executable() + if err != nil { + logs.Error(err) + return "" + } + return path.Dir(strings.ReplaceAll(filepath, `\`, `/`)) +} + +// DoInTempDir 在临时目录中执行 +func DoInTempDir(do func(tempdir string)) { + dir := path.Join(GetPwd(), tempdir, strconv.FormatInt(time.Now().UnixNano(), 10)) + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + logs.Warn(err) + } else { + defer os.RemoveAll(dir) + do(dir) + } +} diff --git a/util/vuln/local.go b/util/vuln/local.go index 61a23a03..90d62f80 100644 --- a/util/vuln/local.go +++ b/util/vuln/local.go @@ -29,9 +29,9 @@ var vulnDB map[string]map[string][]vulnInfo // loadVulnDB 加载本地漏洞 func loadVulnDB() { vulnDB = map[string]map[string][]vulnInfo{} - if args.VulnDB != "" { + if args.Config.VulnDB != "" { // 读取本地漏洞数据 - if data, err := ioutil.ReadFile(args.VulnDB); err != nil { + if data, err := ioutil.ReadFile(args.Config.VulnDB); err != nil { logs.Error(err) } else { // 解析本地漏洞 diff --git a/util/vuln/vuln.go b/util/vuln/vuln.go index 92c5d40c..2a178327 100644 --- a/util/vuln/vuln.go +++ b/util/vuln/vuln.go @@ -29,14 +29,14 @@ func SearchVuln(root *model.DepTree) (err error) { for i, d := range deps { ds[i] = d.Dependency } - if args.VulnDB != "" { + if args.Config.VulnDB != "" { localVulns = GetLocalVulns(ds) } - if args.Url != "" && args.Token != "" { + if args.Config.Url != "" && args.Config.Token != "" { serverVulns, err = GetServerVuln(ds) - } else if args.VulnDB == "" && args.Url == "" && args.Token != "" { + } else if args.Config.VulnDB == "" && args.Config.Url == "" && args.Config.Token != "" { err = errors.New("url is null") - } else if args.VulnDB == "" && args.Url != "" && args.Token == "" { + } else if args.Config.VulnDB == "" && args.Config.Url != "" && args.Config.Token == "" { err = errors.New("token is null") } for i, dep := range deps {