Merge pull request #21 from luotianqi777/dev-1.0.7

Dev 1.0.7

.github/README.md

@@ -86,6 +86,7 @@ opensca-cli -db db.json -path ${project_path}
 | `out`      | `string` | Set the output file. The result defaults to json format.                                                                                                                                                                                                                   | `-out output.json`                |
 | `db`       | `string` | Set the local vulnerability database file. It helps when you prefer to use your own vulnerability database. The format of the vulnerability database is shown below. If the cloud and local vulnerability databases are both set, the result of detection will merge both. | `-db db.json`                     |
 | `progress` | `bool`   | Show the progress bar.                                                                                                                                                                                                                                                     | `-progress`                       |
+| `dedup`    | `bool`   | Same result deduplication                                                                                                                                                                                                                                                  | `-dedup`                          |
 
 ------
 
@@ -142,7 +143,7 @@ opensca-cli -db db.json -path ${project_path}
 
 OpenSCA is an open source project, we appreciate your help!
 
-To contribute, please read our [Contributing Guideline](./docs/Contributing%20Guideline-en%20v1.0.md). 
+To contribute, please read our [Contributing Guideline](../docs/Contributing%20Guideline-en%20v1.0.md). 
 
 
 

README.md

@@ -84,6 +84,7 @@ opensca-cli -db db.json -path ${project_path}
 | `out`      | `string` | 将检测结果保存到指定文件，根据后缀生成不同格式的文件，默认为 `json` 格式                                                                          | `-out output.json`                |
 | `db`       | `string` | 指定本地漏洞库文件，希望使用自己漏洞库时可用，漏洞库文件为 `json` 格式，具体格式会在之后给出;若同时使用云端漏洞库与本地漏洞库，漏洞查询结果取并集 | `-db db.json`                     |
 | `progress` | `bool`   | 显示进度条                                                                                                                                        | `-progress`                       |
+| `dedup` | `bool`   | 相同组件去重                                                                                                                                        | `-dedup`                       |
 
 ---
 

analyzer/engine/archive.go

@@ -17,6 +17,7 @@ import (
 	"util/filter"
 	"util/logs"
 	"util/model"
+	"util/temp"
 
 	"github.com/axgle/mahonia"
 	"github.com/mholt/archiver"
@@ -39,7 +40,6 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) {
 	filepath = strings.ReplaceAll(filepath, `\`, `/`)
 	// 目录树根
 	root = model.NewDirTree()
-	root.Path = path.Base(filepath)
 	var walker archiver.Walker
 	if filter.Tar(filepath) {
 		walker = archiver.NewTar()
@@ -86,35 +86,27 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) {
 				// 支持解析的文件
 				root.AddFile(model.NewFileData(fileName, data))
 			} else if filter.AllPkg(fileName) {
-				// 支持检测的压缩包
-				rootPath, _ := os.Executable()
-				rootPath = path.Dir(strings.ReplaceAll(rootPath, `\`, `/`))
-				tempPath := path.Join(rootPath, ".temp_path")
-				// 创建临时文件夹
-				os.Mkdir(tempPath, os.ModeDir)
-				targetPath := path.Join(tempPath, path.Base(fileName))
 				// 将压缩包解压到本地
-				if out, err := os.Create(targetPath); err == nil {
-					_, err = out.Write(data)
-					out.Close()
-					if err != nil {
-						return errors.WithStack(err)
-					}
-					// 获取当前目录树
-					dir := root.GetDir(fileName)
-					name := path.Base(fileName)
-					if _, ok := dir.SubDir[name]; !ok {
-						// 将压缩包的内容添加到当前目录树
-						dir.DirList = append(dir.DirList, name)
-						dir.SubDir[name] = e.unArchiveFile(targetPath)
-					}
-					// 删除压缩包
-					if err = os.Remove(targetPath); err != nil {
+				temp.DoInTempDir(func(tempdir string) {
+					targetPath := path.Join(tempdir, path.Base(fileName))
+					if out, err := os.Create(targetPath); err == nil {
+						_, err = out.Write(data)
+						out.Close()
+						if err != nil {
+							logs.Error(err)
+						}
+						// 获取当前目录树
+						dir := root.GetDir(fileName)
+						name := path.Base(fileName)
+						if _, ok := dir.SubDir[name]; !ok {
+							// 将压缩包的内容添加到当前目录树
+							dir.DirList = append(dir.DirList, name)
+							dir.SubDir[name] = e.unArchiveFile(targetPath)
+						}
+					} else {
 						logs.Error(err)
 					}
-				} else {
-					logs.Error(err)
-				}
+				})
 			}
 		}
 		return nil

analyzer/engine/engine.go

@@ -24,6 +24,7 @@ import (
 	"analyzer/java"
 	"analyzer/javascript"
 	"analyzer/php"
+	"analyzer/python"
 	"analyzer/ruby"
 	"analyzer/rust"
 )
@@ -43,6 +44,9 @@ func NewEngine() Engine {
 			rust.New(),
 			golang.New(),
 			erlang.New(),
+			// 暂不解析groovy文件
+			// groovy.New(),
+			python.New(),
 		},
 	}
 }
@@ -97,7 +101,7 @@ func (e Engine) ParseFile(filepath string) (depRoot *model.DepTree, taskInfo rep
 	// 获取漏洞
 	taskInfo.Error = vuln.SearchVuln(depRoot)
 	// 是否仅保留漏洞组件
-	if args.OnlyVuln {
+	if args.Config.OnlyVuln {
 		root := model.NewDepTree(nil)
 		q := model.NewQueue()
 		q.Push(depRoot)

analyzer/engine/parse.go

@@ -7,6 +7,7 @@ package engine
 
 import (
 	"path"
+	"strings"
 	"util/filter"
 	"util/model"
 )
@@ -36,7 +37,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree)
 		for _, d := range analyzer.ParseFiles(files) {
 			depRoot.Children = append(depRoot.Children, d)
 			d.Parent = depRoot
-			if d.Name != "" && d.Version.Ok() {
+			if d.Name != "" && !strings.ContainsAny(d.Vendor+d.Name, "${}") && d.Version.Ok() {
 				d.Path = path.Join(d.Path, d.Dependency.String())
 			}
 			// 标识为直接依赖
@@ -73,7 +74,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree)
 	for len(q) > 0 {
 		n := q[0]
 		q = append(q[1:], n.Children...)
-		if n.Name == "" || !n.Version.Ok() {
+		if n.Name == "" || strings.ContainsAny(n.Vendor+n.Name, "${}") || !n.Version.Ok() {
 			n.Move(n.Parent)
 		}
 	}

analyzer/java/analyzer.go

@@ -35,7 +35,7 @@ func (Analyzer) GetLanguage() language.Type {
 
 // CheckFile Check if it is a parsable file
 func (Analyzer) CheckFile(filename string) bool {
-	return filter.JavaPom(filename)
+	return filter.JavaPom(filename) || filter.GroovyGradle(filename)
 }
 
 // pomTree pom文件树
@@ -182,9 +182,15 @@ func (a Analyzer) ParseFiles(files []*model.FileInfo) (deps []*model.DepTree) {
 			p.Path = f.Name
 			poms = append(poms, p)
 		}
+		if filter.GroovyGradle(f.Name) {
+			dep := model.NewDepTree(nil)
+			dep.Path = f.Name
+			parseGradle(dep, f)
+			deps = append(deps, dep)
+		}
 	}
 	// 构建jar树
-	deps = buildJarTree(jarMap)
+	deps = append(deps, buildJarTree(jarMap)...)
 	// 构建pom树
 	deps = append(deps, buildPomTree(poms).parsePomTree(jarMap)...)
 	return

analyzer/java/ext.go

@@ -18,17 +18,14 @@ import (
 	"util/enum/language"
 	"util/logs"
 	"util/model"
+	"util/temp"
 
 	"github.com/pkg/errors"
 )
 
 // MvnDepTree 调用mvn解析项目获取依赖树
 func MvnDepTree(path string, root *model.DepTree) {
-	pwd, err := os.Getwd()
-	if err != nil {
-		logs.Error(err)
-		return
-	}
+	pwd := temp.GetPwd()
 	os.Chdir(path)
 	cmd := exec.Command("mvn", "dependency:tree", "--fail-never")
 	out, _ := cmd.CombinedOutput()

analyzer/java/gradle.go

@@ -6,9 +6,12 @@ import (
 	"encoding/json"
 	"os"
 	"os/exec"
+	"regexp"
+	"strings"
 	"util/enum/language"
 	"util/logs"
 	"util/model"
+	"util/temp"
 )
 
 //go:embed oss.gradle
@@ -26,14 +29,10 @@ type gradleDep struct {
 
 // GradleDepTree 尝试获取 gradle 依赖树
 func GradleDepTree(dirpath string, root *model.DepTree) {
-	pwd, err := os.Getwd()
-	if err != nil {
-		logs.Error(err)
-		return
-	}
+	pwd := temp.GetPwd()
 	os.Chdir(dirpath)
 	// 复制 oss.gradle
-	if err = os.WriteFile("oss.gradle", ossGradle, 0444); err != nil {
+	if err := os.WriteFile("oss.gradle", ossGradle, 0444); err != nil {
 		logs.Warn(err)
 		return
 	}
@@ -52,7 +51,7 @@ func GradleDepTree(dirpath string, root *model.DepTree) {
 			data := out[startIndex+len(startTag) : endIndex]
 			out = out[endIndex+1:]
 			gdep := &gradleDep{MapDep: model.NewDepTree(root)}
-			err = json.Unmarshal(data, &gdep.Children)
+			err := json.Unmarshal(data, &gdep.Children)
 			if err != nil {
 				logs.Warn(err)
 			}
@@ -78,3 +77,35 @@ func GradleDepTree(dirpath string, root *model.DepTree) {
 	}
 	return
 }
+
+// parseGradle parse *.gradle or *.gradle.kts
+func parseGradle(root *model.DepTree, file *model.FileInfo) {
+	regexs := []*regexp.Regexp{
+		regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?name: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`),
+		regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?module: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`),
+		regexp.MustCompile(`['"]([^\s:'"]+):([^\s:'"]+):([^\s:'"]+)['"]`),
+	}
+	for _, line := range strings.Split(string(file.Data), "\n") {
+		for _, re := range regexs {
+			match := re.FindStringSubmatch(line)
+			// 有捕获内容
+			if len(match) == 4 &&
+				// 不以注释开头
+				!strings.HasPrefix(strings.TrimSpace(line), "/") &&
+				// 不是测试组件
+				!strings.Contains(strings.ToLower(line), "testimplementation") &&
+				// 去掉非组件内容
+				!strings.Contains(line, "//") {
+				ver := model.NewVersion(match[3])
+				// 版本号正常
+				if ver.Ok() {
+					dep := model.NewDepTree(root)
+					dep.Vendor = match[1]
+					dep.Name = match[2]
+					dep.Version = ver
+					break
+				}
+			}
+		}
+	}
+}

analyzer/java/pom.go

@@ -129,7 +129,7 @@ func (p *Pom) GetProperty(key string) string {
 		return p.Version
 	case "${project.groupId}", "${groupId}", "${pom.groupId}":
 		return p.GroupId
-	case "${project.artifactId}":
+	case "${project.artifactId}", "${artifactId}", "${pom.artifactId}":
 		return p.ArtifactId
 	case "${project.parent.version}", "${parent.version}":
 		return p.Parent.Version

analyzer/python/analyzer.go

@@ -0,0 +1,44 @@
+package python
+
+import (
+	"util/enum/language"
+	"util/filter"
+	"util/model"
+)
+
+type Analyzer struct {
+}
+
+func New() Analyzer {
+	return Analyzer{}
+}
+
+// GetLanguage get language of analyzer
+func (Analyzer) GetLanguage() language.Type {
+	return language.Python
+}
+
+// CheckFile check parsable file
+func (Analyzer) CheckFile(filename string) bool {
+	return filter.PythonSetup(filename) ||
+		filter.PythonPipfile(filename) ||
+		filter.PythonPipfileLock(filename)
+}
+
+// ParseFiles parse dependency from file
+func (Analyzer) ParseFiles(files []*model.FileInfo) []*model.DepTree {
+	deps := []*model.DepTree{}
+	for _, f := range files {
+		dep := model.NewDepTree(nil)
+		dep.Path = f.Name
+		if filter.PythonSetup(f.Name) {
+			parseSetup(dep, f)
+		} else if filter.PythonPipfile(f.Name) {
+			parsePipfile(dep, f)
+		} else if filter.PythonPipfileLock(f.Name) {
+			parsePipfileLock(dep, f)
+		}
+		deps = append(deps, dep)
+	}
+	return deps
+}

analyzer/python/oss.py

@@ -0,0 +1,34 @@
+import re
+import sys
+import json
+
+def parse_setup_py(setup_py_path):
+	"""解析setup.py文件"""
+	with open(setup_py_path, "r") as f:
+		pass_func = lambda **x: x
+		try:
+			import distutils
+			distutils.core.setup = pass_func
+		except Exception:
+			pass
+		try:
+			import setuptools
+			setuptools.setup = pass_func
+		except Exception:
+			pass
+		# 获取setup参数
+		args = {}
+		code = re.sub('(?<!\w)setup\(','args=setup(',f.read())
+		code = code.replace('__file__','"{}"'.format(setup_py_path))
+		exec(code, args)
+		if 'args' in args:
+			data = args['args']
+			info = {}
+			for k in ['name','version','license','packages','install_requires','requires']:
+				if k in data:
+					info[k] = data[k]
+			print('oss_start<<{}>>oss_end'.format(json.dumps(info)))
+
+if __name__ == "__main__":
+	if len(sys.argv) > 1:
+		parse_setup_py(sys.argv[1])