From 7b252e6fdcd6c2d84ea182f3b86188959ec94c30 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Mon, 30 May 2022 15:17:19 +0800 Subject: [PATCH 01/11] fix error of json report is empty --- util/report/format.go | 13 +++++++------ util/report/json.go | 3 +++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/util/report/format.go b/util/report/format.go index 34697311..641e2962 100644 --- a/util/report/format.go +++ b/util/report/format.go @@ -10,12 +10,13 @@ import ( // 任务检查信息 type TaskInfo struct { - AppName string `json:"app_name"` - Size int64 `json:"size"` - StartTime string `json:"start_time"` - EndTime string `json:"end_time"` - CostTime float64 `json:"cost_time"` - Error error `json:"error,omitempty"` + AppName string `json:"app_name"` + Size int64 `json:"size"` + StartTime string `json:"start_time"` + EndTime string `json:"end_time"` + CostTime float64 `json:"cost_time"` + Error error `json:"-"` + ErrorString string `json:"error,omitempty"` } // format 按照输出内容格式化(不可逆) diff --git a/util/report/json.go b/util/report/json.go index 7d092bfb..e92a317f 100644 --- a/util/report/json.go +++ b/util/report/json.go @@ -9,6 +9,9 @@ import ( // Json 获取json格式报告数据 func Json(dep *model.DepTree, taskInfo TaskInfo) []byte { format(dep) + if taskInfo.Error != nil { + taskInfo.ErrorString = taskInfo.Error.Error() + } if data, err := json.Marshal(struct { *model.DepTree TaskInfo TaskInfo `json:"task_info"` From 33ed3d6c85a34a0b58ab6445d1bc86e18c8b6193 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Mon, 30 May 2022 20:04:06 +0800 Subject: [PATCH 02/11] optimization args --- analyzer/engine/engine.go | 2 +- cli/main.go | 10 +++--- util/args/args.go | 72 ++++++++++++++++++++------------------- util/args/config.go | 63 ---------------------------------- util/bar/bar.go | 2 +- util/cache/cache.go | 4 +-- util/client/client.go | 8 ++--- util/vuln/local.go | 4 +-- util/vuln/vuln.go | 8 ++--- 9 files changed, 56 insertions(+), 117 deletions(-) delete mode 100644 util/args/config.go diff --git a/analyzer/engine/engine.go b/analyzer/engine/engine.go index ed67841b..d554c073 100644 --- a/analyzer/engine/engine.go +++ b/analyzer/engine/engine.go @@ -97,7 +97,7 @@ func (e Engine) ParseFile(filepath string) (depRoot *model.DepTree, taskInfo rep // 获取漏洞 taskInfo.Error = vuln.SearchVuln(depRoot) // 是否仅保留漏洞组件 - if args.OnlyVuln { + if args.Config.OnlyVuln { root := model.NewDepTree(nil) q := model.NewQueue() q.Push(depRoot) diff --git a/cli/main.go b/cli/main.go index 6d36bf58..06bdaeaf 100644 --- a/cli/main.go +++ b/cli/main.go @@ -17,8 +17,8 @@ import ( func main() { args.Parse() - if len(args.Filepath) > 0 { - output(engine.NewEngine().ParseFile(args.Filepath)) + if len(args.Config.Path) > 0 { + output(engine.NewEngine().ParseFile(args.Config.Path)) } else { flag.PrintDefaults() } @@ -30,7 +30,7 @@ func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { logs.Debug("\n" + depRoot.String()) // 输出结果 var reportFunc func(*model.DepTree, report.TaskInfo) []byte - switch path.Ext(args.Out) { + switch path.Ext(args.Config.Out) { case ".html": reportFunc = report.Html case ".json": @@ -38,8 +38,8 @@ func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { default: reportFunc = report.Json } - if args.Out != "" { - report.Save(reportFunc(depRoot, taskInfo), args.Out) + if args.Config.Out != "" { + report.Save(reportFunc(depRoot, taskInfo), args.Config.Out) } else { fmt.Println(string(reportFunc(depRoot, taskInfo))) } diff --git a/util/args/args.go b/util/args/args.go index e9ae45ff..513e1e5b 100644 --- a/util/args/args.go +++ b/util/args/args.go @@ -6,52 +6,54 @@ package args import ( + "encoding/json" "flag" + "fmt" + "io/ioutil" "strings" ) var ( - // 配置文件路径 - Config string - // 解析文件路径 - Filepath string - // 云服务地址 - Url string - IP string - // 云服务token - Token string - // 开启本地缓存 - Cache bool - // 输出文件 - Out string - // 仅展示有漏洞的组件 - OnlyVuln bool - // 本地漏洞库文件路径 - VulnDB string - // display progress bar - ProgressBar bool + ConfigPath string + Config = struct { + // detect option + Path string `json:"path"` + Out string `json:"out"` + Cache bool `json:"cache"` + Bar bool `json:"progress"` + OnlyVuln bool `json:"vuln"` + // remote vuldb + Url string `json:"url"` + Token string `json:"token"` + // local vuldb + VulnDB string `json:"db"` + }{} ) func init() { - // 设置参数信息 - flag.StringVar(&Config, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") - flag.StringVar(&Filepath, "path", "", "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") - flag.StringVar(&Url, "url", "", "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") - flag.StringVar(&IP, "ip", "", "(待废弃,删除)与url作用相同,兼容旧版本参数") - flag.StringVar(&Token, "token", "", "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") - flag.BoolVar(&Cache, "cache", false, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") - flag.BoolVar(&OnlyVuln, "vuln", false, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") - flag.StringVar(&Out, "out", "", "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") - flag.StringVar(&VulnDB, "db", "", "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") - flag.BoolVar(&ProgressBar, "progress", false, "(可选) 显示进度条") + flag.StringVar(&ConfigPath, "config", "config.json", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") + flag.StringVar(&Config.Path, "path", Config.Path, "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") + flag.StringVar(&Config.Url, "url", Config.Url, "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") + flag.StringVar(&Config.Token, "token", Config.Token, "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") + flag.BoolVar(&Config.Cache, "cache", Config.Cache, "(可选,建议开启) 缓存下载的文件(例如pom文件),重复检测相同组件时会节省时间,下载的文件会保存到工具所在目录的.cache目录下") + flag.BoolVar(&Config.OnlyVuln, "vuln", Config.OnlyVuln, "(可选) 结果仅保留有漏洞信息的组件,使用该参数不会保留组件层级结构") + flag.StringVar(&Config.Out, "out", Config.Out, "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") + flag.StringVar(&Config.VulnDB, "db", Config.VulnDB, "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") + flag.BoolVar(&Config.Bar, "progress", Config.Bar, "(可选) 显示进度条") } func Parse() { flag.Parse() - // 兼容旧版本，待废弃 - if Url == "" && IP != "" { - Url = IP + if ConfigPath != "" { + if data, err := ioutil.ReadFile(ConfigPath); err != nil { + fmt.Printf("load config file error: %s\n", err) + } else { + if err = json.Unmarshal(data, &Config); err != nil { + fmt.Printf("parse config file error: %s\n", err) + } + } + // 再次调用Parse, 优先使用cli参数 + flag.Parse() } - loadConfigFile() - Url = strings.TrimSuffix(Url, "/") + Config.Url = strings.TrimSuffix(Config.Url, "/") } diff --git a/util/args/config.go b/util/args/config.go deleted file mode 100644 index 4456d84b..00000000 --- a/util/args/config.go +++ /dev/null @@ -1,63 +0,0 @@ -package args - -import ( - "encoding/json" - "os" - "util/logs" -) - -// loadConfigFile 加载配置文件 -func loadConfigFile() bool { - configFilePath := Config - if configFilePath == "" { - return false - } - if _, err := os.Stat(configFilePath); err != nil { - logs.Error(err) - return false - } - if data, err := os.ReadFile(configFilePath); err != nil { - logs.Error(err) - return false - } else { - config := struct { - Path string `json:"path"` - DB string `json:"db"` - Url string `json:"url"` - Token string `json:"token"` - Out string `json:"out"` - Cache *bool `json:"cache"` - OnlyVuln *bool `json:"vuln"` - ProgressBar *bool `json:"progress"` - }{} - if err = json.Unmarshal(data, &config); err != nil { - logs.Error(err) - return false - } - if Filepath == "" && config.Path != "" { - Filepath = config.Path - } - if VulnDB == "" && config.DB != "" { - VulnDB = config.DB - } - if Url == "" && config.Url != "" { - Url = config.Url - } - if Token == "" && config.Token != "" { - Token = config.Token - } - if Out == "" && config.Out != "" { - Out = config.Out - } - if !Cache && config.Cache != nil { - Cache = *config.Cache - } - if !OnlyVuln && config.OnlyVuln != nil { - OnlyVuln = *config.OnlyVuln - } - if !ProgressBar && config.ProgressBar != nil { - ProgressBar = *config.ProgressBar - } - return true - } -} diff --git a/util/bar/bar.go b/util/bar/bar.go index e7627b01..9085e4ee 100644 --- a/util/bar/bar.go +++ b/util/bar/bar.go @@ -37,7 +37,7 @@ func newBar(text string) *Bar { // Add add progress func (b *Bar) Add(n int) { - if !args.ProgressBar { + if !args.Config.Bar { return } if b.id == -1 { diff --git a/util/cache/cache.go b/util/cache/cache.go index 82011341..7e44a4d0 100644 --- a/util/cache/cache.go +++ b/util/cache/cache.go @@ -33,7 +33,7 @@ func init() { // save save cache file func save(filepath string, data []byte) { - if args.Cache { + if args.Config.Cache { if err := os.MkdirAll(path.Join(cacheDir, path.Dir(filepath)), os.ModeDir); err == nil { if f, err := os.Create(path.Join(cacheDir, filepath)); err == nil { defer f.Close() @@ -45,7 +45,7 @@ func save(filepath string, data []byte) { // load load cache file func load(filepath string) []byte { - if args.Cache { + if args.Config.Cache { if data, err := ioutil.ReadFile(path.Join(cacheDir, filepath)); err == nil { return data } else { diff --git a/util/client/client.go b/util/client/client.go index 046eb0a0..d82b0665 100644 --- a/util/client/client.go +++ b/util/client/client.go @@ -109,11 +109,11 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // aes加密 ciphertext, tag := encrypt(reqbody, key, nonce) // 构建请求 - url := args.Url + "/oss-saas/api-v1/open-sca-client/detect" + url := args.Config.Url + "/oss-saas/api-v1/open-sca-client/detect" // 添加参数 param := DetectRequst{} param.ClientId = GetClientId() - param.Token = args.Token + param.Token = args.Config.Token param.Tag = base64.StdEncoding.EncodeToString(tag) param.Nonce = base64.StdEncoding.EncodeToString(nonce) // base64编码 @@ -168,14 +168,14 @@ func Detect(reqbody []byte) (repbody []byte, err error) { // getAesKey 获取aes-key func getAesKey() (key []byte, err error) { - u, err := url.Parse(args.Url + "/oss-saas/api-v1/open-sca-client/aes-key") + u, err := url.Parse(args.Config.Url + "/oss-saas/api-v1/open-sca-client/aes-key") if err != nil { return key, err } // 设置参数 param := url.Values{} param.Set("clientId", GetClientId()) - param.Set("ossToken", args.Token) + param.Set("ossToken", args.Config.Token) u.RawQuery = param.Encode() // 发送请求 rep, err := http.Get(u.String()) diff --git a/util/vuln/local.go b/util/vuln/local.go index 61a23a03..90d62f80 100644 --- a/util/vuln/local.go +++ b/util/vuln/local.go @@ -29,9 +29,9 @@ var vulnDB map[string]map[string][]vulnInfo // loadVulnDB 加载本地漏洞 func loadVulnDB() { vulnDB = map[string]map[string][]vulnInfo{} - if args.VulnDB != "" { + if args.Config.VulnDB != "" { // 读取本地漏洞数据 - if data, err := ioutil.ReadFile(args.VulnDB); err != nil { + if data, err := ioutil.ReadFile(args.Config.VulnDB); err != nil { logs.Error(err) } else { // 解析本地漏洞 diff --git a/util/vuln/vuln.go b/util/vuln/vuln.go index 92c5d40c..2a178327 100644 --- a/util/vuln/vuln.go +++ b/util/vuln/vuln.go @@ -29,14 +29,14 @@ func SearchVuln(root *model.DepTree) (err error) { for i, d := range deps { ds[i] = d.Dependency } - if args.VulnDB != "" { + if args.Config.VulnDB != "" { localVulns = GetLocalVulns(ds) } - if args.Url != "" && args.Token != "" { + if args.Config.Url != "" && args.Config.Token != "" { serverVulns, err = GetServerVuln(ds) - } else if args.VulnDB == "" && args.Url == "" && args.Token != "" { + } else if args.Config.VulnDB == "" && args.Config.Url == "" && args.Config.Token != "" { err = errors.New("url is null") - } else if args.VulnDB == "" && args.Url != "" && args.Token == "" { + } else if args.Config.VulnDB == "" && args.Config.Url != "" && args.Config.Token == "" { err = errors.New("token is null") } for i, dep := range deps { From 62d284c737dd421347076bf7d7339620b1fd3fde Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Mon, 30 May 2022 20:07:36 +0800 Subject: [PATCH 03/11] support python --- analyzer/engine/archive.go | 45 +++++++++----------- analyzer/engine/engine.go | 4 ++ analyzer/java/ext.go | 7 +--- analyzer/java/gradle.go | 11 ++--- analyzer/python/analyzer.go | 44 ++++++++++++++++++++ analyzer/python/oss.py | 27 ++++++++++++ analyzer/python/pipfile.go | 56 +++++++++++++++++++++++++ analyzer/python/setup.go | 75 ++++++++++++++++++++++++++++++++++ util/client/client.go | 51 +++++++++++------------ util/enum/language/language.go | 6 +++ util/filter/file.go | 7 ++++ util/model/version.go | 2 +- util/temp/temp.go | 37 +++++++++++++++++ 13 files changed, 306 insertions(+), 66 deletions(-) create mode 100644 analyzer/python/analyzer.go create mode 100644 analyzer/python/oss.py create mode 100644 analyzer/python/pipfile.go create mode 100644 analyzer/python/setup.go create mode 100644 util/temp/temp.go diff --git a/analyzer/engine/archive.go b/analyzer/engine/archive.go index 11da4c35..de58bcf7 100644 --- a/analyzer/engine/archive.go +++ b/analyzer/engine/archive.go @@ -17,6 +17,7 @@ import ( "util/filter" "util/logs" "util/model" + "util/temp" "github.com/axgle/mahonia" "github.com/mholt/archiver" @@ -86,35 +87,27 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { // 支持解析的文件 root.AddFile(model.NewFileData(fileName, data)) } else if filter.AllPkg(fileName) { - // 支持检测的压缩包 - rootPath, _ := os.Executable() - rootPath = path.Dir(strings.ReplaceAll(rootPath, `\`, `/`)) - tempPath := path.Join(rootPath, ".temp_path") - // 创建临时文件夹 - os.Mkdir(tempPath, os.ModeDir) - targetPath := path.Join(tempPath, path.Base(fileName)) // 将压缩包解压到本地 - if out, err := os.Create(targetPath); err == nil { - _, err = out.Write(data) - out.Close() - if err != nil { - return errors.WithStack(err) - } - // 获取当前目录树 - dir := root.GetDir(fileName) - name := path.Base(fileName) - if _, ok := dir.SubDir[name]; !ok { - // 将压缩包的内容添加到当前目录树 - dir.DirList = append(dir.DirList, name) - dir.SubDir[name] = e.unArchiveFile(targetPath) - } - // 删除压缩包 - if err = os.Remove(targetPath); err != nil { + temp.DoInTempDir(func(tempdir string) { + targetPath := path.Join(tempdir, path.Base(fileName)) + if out, err := os.Create(targetPath); err == nil { + _, err = out.Write(data) + out.Close() + if err != nil { + logs.Error(err) + } + // 获取当前目录树 + dir := root.GetDir(fileName) + name := path.Base(fileName) + if _, ok := dir.SubDir[name]; !ok { + // 将压缩包的内容添加到当前目录树 + dir.DirList = append(dir.DirList, name) + dir.SubDir[name] = e.unArchiveFile(targetPath) + } + } else { logs.Error(err) } - } else { - logs.Error(err) - } + }) } } return nil diff --git a/analyzer/engine/engine.go b/analyzer/engine/engine.go index d554c073..4a065050 100644 --- a/analyzer/engine/engine.go +++ b/analyzer/engine/engine.go @@ -24,6 +24,7 @@ import ( "analyzer/java" "analyzer/javascript" "analyzer/php" + "analyzer/python" "analyzer/ruby" "analyzer/rust" ) @@ -43,6 +44,9 @@ func NewEngine() Engine { rust.New(), golang.New(), erlang.New(), + // 暂不解析groovy文件 + // groovy.New(), + python.New(), }, } } diff --git a/analyzer/java/ext.go b/analyzer/java/ext.go index cc447808..2afd04fa 100644 --- a/analyzer/java/ext.go +++ b/analyzer/java/ext.go @@ -18,17 +18,14 @@ import ( "util/enum/language" "util/logs" "util/model" + "util/temp" "github.com/pkg/errors" ) // MvnDepTree 调用mvn解析项目获取依赖树 func MvnDepTree(path string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(path) cmd := exec.Command("mvn", "dependency:tree", "--fail-never") out, _ := cmd.CombinedOutput() diff --git a/analyzer/java/gradle.go b/analyzer/java/gradle.go index 51f5e571..3a951319 100644 --- a/analyzer/java/gradle.go +++ b/analyzer/java/gradle.go @@ -9,6 +9,7 @@ import ( "util/enum/language" "util/logs" "util/model" + "util/temp" ) //go:embed oss.gradle @@ -26,14 +27,10 @@ type gradleDep struct { // GradleDepTree 尝试获取 gradle 依赖树 func GradleDepTree(dirpath string, root *model.DepTree) { - pwd, err := os.Getwd() - if err != nil { - logs.Error(err) - return - } + pwd := temp.GetPwd() os.Chdir(dirpath) // 复制 oss.gradle - if err = os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { + if err := os.WriteFile("oss.gradle", ossGradle, 0444); err != nil { logs.Warn(err) return } @@ -52,7 +49,7 @@ func GradleDepTree(dirpath string, root *model.DepTree) { data := out[startIndex+len(startTag) : endIndex] out = out[endIndex+1:] gdep := &gradleDep{MapDep: model.NewDepTree(root)} - err = json.Unmarshal(data, &gdep.Children) + err := json.Unmarshal(data, &gdep.Children) if err != nil { logs.Warn(err) } diff --git a/analyzer/python/analyzer.go b/analyzer/python/analyzer.go new file mode 100644 index 00000000..047d3528 --- /dev/null +++ b/analyzer/python/analyzer.go @@ -0,0 +1,44 @@ +package python + +import ( + "util/enum/language" + "util/filter" + "util/model" +) + +type Analyzer struct { +} + +func New() Analyzer { + return Analyzer{} +} + +// GetLanguage get language of analyzer +func (Analyzer) GetLanguage() language.Type { + return language.Python +} + +// CheckFile check parsable file +func (Analyzer) CheckFile(filename string) bool { + return filter.PythonSetup(filename) || + filter.PythonPipfile(filename) || + filter.PythonPipfileLock(filename) +} + +// ParseFiles parse dependency from file +func (Analyzer) ParseFiles(files []*model.FileInfo) []*model.DepTree { + deps := []*model.DepTree{} + for _, f := range files { + dep := model.NewDepTree(nil) + dep.Path = f.Name + if filter.PythonSetup(f.Name) { + parseSetup(dep, f) + } else if filter.PythonPipfile(f.Name) { + parsePipfile(dep, f) + } else if filter.PythonPipfileLock(f.Name) { + parsePipfileLock(dep, f) + } + deps = append(deps, dep) + } + return deps +} diff --git a/analyzer/python/oss.py b/analyzer/python/oss.py new file mode 100644 index 00000000..8cd4957c --- /dev/null +++ b/analyzer/python/oss.py @@ -0,0 +1,27 @@ +import sys +import json + +def parse_setup_py(setup_py_path): + """解析setup.py文件""" + with open(setup_py_path, "r") as f: + pass_func = lambda **x: x + try: + import distutils + distutils.core.setup = pass_func + except Exception: + pass + try: + import setuptools + setuptools.setup = pass_func + except Exception: + pass + # 获取setup参数 + args = {} + exec(f.read().replace("setup(", "args=setup("), args) + if 'args' in args: + js = json.dumps(args["args"]) + print('oss_start<<{}>>oss_end'.format(js)) + +if __name__ == "__main__": + if len(sys.argv) > 1: + parse_setup_py(sys.argv[1]) \ No newline at end of file diff --git a/analyzer/python/pipfile.go b/analyzer/python/pipfile.go new file mode 100644 index 00000000..4b720c9c --- /dev/null +++ b/analyzer/python/pipfile.go @@ -0,0 +1,56 @@ +package python + +import ( + "encoding/json" + "util/logs" + "util/model" + + "github.com/BurntSushi/toml" +) + +// parsePipfile parse Pipfile file +func parsePipfile(root *model.DepTree, file *model.FileInfo) { + pip := struct { + DevPackages map[string]string `toml:"dev-packages"` + Packages map[string]string `toml:"packages"` + }{} + if err := toml.Unmarshal(file.Data, &pip); err != nil { + logs.Warn(err) + } + for name, version := range pip.Packages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } + for name, version := range pip.DevPackages { + dep := model.NewDepTree(root) + dep.Name = name + dep.Version = model.NewVersion(version) + } +} + +// parsePipfileLock parse pipfile.lock file +func parsePipfileLock(root *model.DepTree, file *model.FileInfo) { + lock := struct { + Default map[string]struct { + Version string `json:"version"` + } `json:"default"` + }{} + err := json.Unmarshal(file.Data, &lock) + if err != nil { + logs.Warn(err) + } + names := []string{} + for n := range lock.Default { + names = append(names, n) + } + for _, n := range names { + v := lock.Default[n].Version + if v != "" { + dep := model.NewDepTree(root) + dep.Name = n + dep.Version = model.NewVersion(v) + } + } + return +} diff --git a/analyzer/python/setup.go b/analyzer/python/setup.go new file mode 100644 index 00000000..5ce39505 --- /dev/null +++ b/analyzer/python/setup.go @@ -0,0 +1,75 @@ +package python + +import ( + _ "embed" + "encoding/json" + "os" + "os/exec" + "path" + "strings" + "util/logs" + "util/model" + "util/temp" +) + +//go:embed oss.py +var ossPy []byte + +// oss.py 脚本输出的依赖结构 +type setupDep struct { + Name string `json:"name"` + Version string `json:"version"` + License string `json:"license"` + Packages []string `json:"packages"` + InstallRequires []string `json:"install_requires"` + Requires []string `json:"requires"` +} + +// parseSetup 解析 setup.py 文件 +func parseSetup(root *model.DepTree, file *model.FileInfo) { + temp.DoInTempDir(func(tempdir string) { + ossfile := path.Join(tempdir, "oss.py") + setupfile := path.Join(tempdir, "setup.py") + // 创建 oss.py + if err := os.WriteFile(ossfile, ossPy, 0444); err != nil { + logs.Warn(err) + return + } + // 创建 setup.py + if err := os.WriteFile(setupfile, file.Data, 0444); err != nil { + logs.Warn(err) + return + } + // 解析 setup.py + cmd := exec.Command("python", ossfile, setupfile) + out, _ := cmd.CombinedOutput() + startTag, endTag := `oss_start<<`, `>>oss_end` + startIndex, endIndex := strings.Index(string(out), startTag), strings.Index(string(out), endTag) + if startIndex == -1 || endIndex == -1 { + return + } else { + out = out[startIndex+len(startTag) : endIndex] + } + // 获取解析结果 + var dep setupDep + if err := json.Unmarshal(out, &dep); err != nil { + logs.Warn(err) + } + root.Name = dep.Name + root.Version = model.NewVersion(dep.Version) + root.Licenses = append(root.Licenses, dep.License) + for _, pkg := range [][]string{dep.Packages, dep.InstallRequires, dep.Requires} { + for _, p := range pkg { + index := strings.IndexAny(p, "=<>") + sub := model.NewDepTree(root) + if index > -1 { + sub.Name = p[:index] + sub.Version = model.NewVersion(p[index:]) + } else { + sub.Name = p + } + } + } + }) + return +} diff --git a/util/client/client.go b/util/client/client.go index d82b0665..c5a76607 100644 --- a/util/client/client.go +++ b/util/client/client.go @@ -20,6 +20,7 @@ import ( "regexp" "util/args" "util/logs" + "util/temp" "github.com/pkg/errors" ) @@ -60,35 +61,31 @@ type DetectRequst struct { func GetClientId() string { // 默认id id := "XXXXXXXXXXXXXXXX" - if pwd, err := os.Getwd(); err != nil { - logs.Error(err) - } else { - // 尝试读取.key文件 - idFile := path.Join(pwd, ".key") - if _, err := os.Stat(idFile); err != nil { - // 文件不存在则生成随机ID并保存 - if f, err := os.Create(idFile); err != nil { - logs.Error(err) - } else { - defer f.Close() - const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - idbyte := []byte(id) - for i := range idbyte { - idbyte[i] = chars[mrand.Intn(26)] - } - f.Write(idbyte) - id = string(idbyte) - } + // 尝试读取.key文件 + idFile := path.Join(temp.GetPwd(), ".key") + if _, err := os.Stat(idFile); err != nil { + // 文件不存在则生成随机ID并保存 + if f, err := os.Create(idFile); err != nil { + logs.Error(err) } else { - // 文件存在则读取ID - idbyte, err := os.ReadFile(idFile) - if err != nil { - logs.Error(err) + defer f.Close() + const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + idbyte := []byte(id) + for i := range idbyte { + idbyte[i] = chars[mrand.Intn(26)] } - if len(idbyte) == 16 { - if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { - id = string(idbyte) - } + f.Write(idbyte) + id = string(idbyte) + } + } else { + // 文件存在则读取ID + idbyte, err := os.ReadFile(idFile) + if err != nil { + logs.Error(err) + } + if len(idbyte) == 16 { + if ok, err := regexp.Match(`[A-Z]{16}`, idbyte); ok && err == nil { + id = string(idbyte) } } } diff --git a/util/enum/language/language.go b/util/enum/language/language.go index e132168d..94c71755 100644 --- a/util/enum/language/language.go +++ b/util/enum/language/language.go @@ -21,6 +21,7 @@ const ( Golang Rust Erlang + Python ) // String 语言类型 @@ -42,6 +43,8 @@ func (l Type) String() string { return "Rust" case Erlang: return "Erlang" + case Python: + return "Python" default: return "None" } @@ -66,6 +69,8 @@ func (l Type) Vuln() string { return "rust" case Erlang: return "" + case Python: + return "python" default: return "" } @@ -84,6 +89,7 @@ func init() { lm[Golang] = []string{"golang", "go", "gomod"} lm[Rust] = []string{"rust", "cargo"} lm[Erlang] = []string{"erlang", "rebar"} + lm[Python] = []string{"python", "pip", "pipy"} for t, ls := range lm { for _, l := range ls { lanMap[l] = t diff --git a/util/filter/file.go b/util/filter/file.go index 331c1a94..e1d55690 100644 --- a/util/filter/file.go +++ b/util/filter/file.go @@ -84,3 +84,10 @@ var ( var ( GroovyFile = filterFunc(strings.HasSuffix, ".groovy") ) + +// python +var ( + PythonSetup = filterFunc(strings.HasSuffix, "setup.py") + PythonPipfile = filterFunc(strings.HasSuffix, "Pipfile") + PythonPipfileLock = filterFunc(strings.HasSuffix, "Pipfile.lock") +) diff --git a/util/model/version.go b/util/model/version.go index 1808e86f..01df4e2a 100644 --- a/util/model/version.go +++ b/util/model/version.go @@ -40,7 +40,7 @@ func (ver *Version) weight() (weight int) { func NewVersion(verStr string) *Version { verStr = strings.TrimSpace(verStr) ver := &Version{Nums: []int{}, Org: verStr} - verStr = strings.TrimLeft(verStr, "vV^") + verStr = strings.TrimLeft(verStr, "vV^~=<>") // 获取后缀 index := strings.Index(verStr, "-") if index != -1 { diff --git a/util/temp/temp.go b/util/temp/temp.go new file mode 100644 index 00000000..765a1874 --- /dev/null +++ b/util/temp/temp.go @@ -0,0 +1,37 @@ +package temp + +import ( + "os" + "path" + "strconv" + "strings" + "time" + "util/logs" +) + +const tempdir = ".temp" + +func init() { + os.RemoveAll(path.Join(GetPwd(), tempdir)) +} + +// GetPwd 获取当前目录 +func GetPwd() string { + filepath, err := os.Executable() + if err != nil { + logs.Error(err) + return "" + } + return path.Dir(strings.ReplaceAll(filepath, `\`, `/`)) +} + +// DoInTempDir 在临时目录中执行 +func DoInTempDir(do func(tempdir string)) { + dir := path.Join(GetPwd(), tempdir, strconv.FormatInt(time.Now().UnixNano(), 10)) + if err := os.MkdirAll(dir, os.ModePerm); err != nil { + logs.Warn(err) + } else { + defer os.RemoveAll(dir) + do(dir) + } +} From 203d5da7f38429d1b519b4c364a9b5e97d043514 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Mon, 30 May 2022 18:20:26 +0800 Subject: [PATCH 04/11] static parse gradle --- analyzer/java/analyzer.go | 9 +++++++-- analyzer/java/gradle.go | 27 +++++++++++++++++++++++++++ util/filter/file.go | 3 ++- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/analyzer/java/analyzer.go b/analyzer/java/analyzer.go index 6087b26d..dd5d4023 100644 --- a/analyzer/java/analyzer.go +++ b/analyzer/java/analyzer.go @@ -35,7 +35,7 @@ func (Analyzer) GetLanguage() language.Type { // CheckFile Check if it is a parsable file func (Analyzer) CheckFile(filename string) bool { - return filter.JavaPom(filename) + return filter.JavaPom(filename) || filter.GroovyGradle(filename) } // pomTree pom文件树 @@ -182,9 +182,14 @@ func (a Analyzer) ParseFiles(files []*model.FileInfo) (deps []*model.DepTree) { p.Path = f.Name poms = append(poms, p) } + if filter.GroovyGradle(f.Name) { + dep := model.NewDepTree(nil) + parseGradle(dep, f) + deps = append(deps, dep) + } } // 构建jar树 - deps = buildJarTree(jarMap) + deps = append(deps, buildJarTree(jarMap)...) // 构建pom树 deps = append(deps, buildPomTree(poms).parsePomTree(jarMap)...) return diff --git a/analyzer/java/gradle.go b/analyzer/java/gradle.go index 3a951319..6b098660 100644 --- a/analyzer/java/gradle.go +++ b/analyzer/java/gradle.go @@ -6,6 +6,8 @@ import ( "encoding/json" "os" "os/exec" + "regexp" + "strings" "util/enum/language" "util/logs" "util/model" @@ -75,3 +77,28 @@ func GradleDepTree(dirpath string, root *model.DepTree) { } return } + +// parseGradle parse *.gradle or *.gradle.kts +func parseGradle(root *model.DepTree, file *model.FileInfo) { + regexs := []*regexp.Regexp{ + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?name: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?module: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), + regexp.MustCompile(`['"]([^\s:]+):([^\s:]+):([^\s:]+)['"]`), + } + for _, line := range strings.Split(string(file.Data), "\n") { + for _, re := range regexs { + match := re.FindStringSubmatch(line) + // 有捕获内容且不以注释开头 + if len(match) == 4 && !strings.HasPrefix(strings.TrimSpace(line), "/") { + ver := model.NewVersion(match[3]) + if ver.Ok() { + dep := model.NewDepTree(root) + dep.Vendor = match[1] + dep.Name = match[2] + dep.Version = ver + break + } + } + } + } +} diff --git a/util/filter/file.go b/util/filter/file.go index e1d55690..81ffe09f 100644 --- a/util/filter/file.go +++ b/util/filter/file.go @@ -82,7 +82,8 @@ var ( // groovy var ( - GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyFile = filterFunc(strings.HasSuffix, ".groovy") + GroovyGradle = filterFunc(strings.HasSuffix, ".gradle", ".gradle.kts") ) // python From f9892658e6b979e6f2439f3121fbfa1b72e405e2 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Mon, 30 May 2022 21:12:32 +0800 Subject: [PATCH 05/11] result deduplication --- .github/README.md | 1 + README.md | 1 + analyzer/engine/archive.go | 1 - analyzer/java/analyzer.go | 1 + config.json | 3 ++- util/args/args.go | 14 ++++++++-- util/model/dependency.go | 4 ++- util/report/format.go | 55 +++++++++++++++++++++++++++++++------- 8 files changed, 65 insertions(+), 15 deletions(-) diff --git a/.github/README.md b/.github/README.md index c8b1046b..dd114aef 100644 --- a/.github/README.md +++ b/.github/README.md @@ -86,6 +86,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | Set the output file. The result defaults to json format. | `-out output.json` | | `db` | `string` | Set the local vulnerability database file. It helps when you prefer to use your own vulnerability database. The format of the vulnerability database is shown below. If the cloud and local vulnerability databases are both set, the result of detection will merge both. | `-db db.json` | | `progress` | `bool` | Show the progress bar. | `-progress` | +| `dedup` | `bool` | Same result deduplication | `-dedup` | ------ diff --git a/README.md b/README.md index bc274800..24a79fe1 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ opensca-cli -db db.json -path ${project_path} | `out` | `string` | 将检测结果保存到指定文件，根据后缀生成不同格式的文件，默认为 `json` 格式 | `-out output.json` | | `db` | `string` | 指定本地漏洞库文件，希望使用自己漏洞库时可用，漏洞库文件为 `json` 格式，具体格式会在之后给出;若同时使用云端漏洞库与本地漏洞库，漏洞查询结果取并集 | `-db db.json` | | `progress` | `bool` | 显示进度条 | `-progress` | +| `dedup` | `bool` | 相同组件去重 | `-dedup` | --- diff --git a/analyzer/engine/archive.go b/analyzer/engine/archive.go index de58bcf7..88d97a92 100644 --- a/analyzer/engine/archive.go +++ b/analyzer/engine/archive.go @@ -40,7 +40,6 @@ func (e Engine) unArchiveFile(filepath string) (root *model.DirTree) { filepath = strings.ReplaceAll(filepath, `\`, `/`) // 目录树根 root = model.NewDirTree() - root.Path = path.Base(filepath) var walker archiver.Walker if filter.Tar(filepath) { walker = archiver.NewTar() diff --git a/analyzer/java/analyzer.go b/analyzer/java/analyzer.go index dd5d4023..10c058e8 100644 --- a/analyzer/java/analyzer.go +++ b/analyzer/java/analyzer.go @@ -184,6 +184,7 @@ func (a Analyzer) ParseFiles(files []*model.FileInfo) (deps []*model.DepTree) { } if filter.GroovyGradle(f.Name) { dep := model.NewDepTree(nil) + dep.Path = f.Name parseGradle(dep, f) deps = append(deps, dep) } diff --git a/config.json b/config.json index 9381c40c..63fc6672 100644 --- a/config.json +++ b/config.json @@ -6,5 +6,6 @@ "out": "output.json", "cache": true, "vuln": false, - "progress": true + "progress": true, + "dedup": true } \ No newline at end of file diff --git a/util/args/args.go b/util/args/args.go index 513e1e5b..b7fd7327 100644 --- a/util/args/args.go +++ b/util/args/args.go @@ -10,7 +10,9 @@ import ( "flag" "fmt" "io/ioutil" + "path" "strings" + "util/temp" ) var ( @@ -22,6 +24,7 @@ var ( Cache bool `json:"cache"` Bar bool `json:"progress"` OnlyVuln bool `json:"vuln"` + Dedup bool `json:"dedup"` // remote vuldb Url string `json:"url"` Token string `json:"token"` @@ -40,6 +43,7 @@ func init() { flag.StringVar(&Config.Out, "out", Config.Out, "(可选) 将检测结果保存到指定文件,根据后缀生成不同格式的文件,默认为json格式,例: -out output.json") flag.StringVar(&Config.VulnDB, "db", Config.VulnDB, "(可选) 指定本地漏洞库文件,希望使用自己漏洞库时可用,漏洞库文件为json格式,具体格式会在开源项目文档中给出;若同时使用云端漏洞库与本地漏洞库,漏洞查询结果取并集,例: -db db.json") flag.BoolVar(&Config.Bar, "progress", Config.Bar, "(可选) 显示进度条") + flag.BoolVar(&Config.Dedup, "dedup", Config.Dedup, "(可选) 相同组件去重") } func Parse() { @@ -52,8 +56,14 @@ func Parse() { fmt.Printf("parse config file error: %s\n", err) } } - // 再次调用Parse, 优先使用cli参数 - flag.Parse() + } else { + // 默认读取目录下的config.json文件 + if data, err := ioutil.ReadFile(path.Join(temp.GetPwd(), "config.json")); err == nil { + // 不处理错误 + json.Unmarshal(data, &Config) + } } + // 再次调用Parse, 优先使用cli参数 + flag.Parse() Config.Url = strings.TrimSuffix(Config.Url, "/") } diff --git a/util/model/dependency.go b/util/model/dependency.go index f6c5ba72..cfcfede6 100644 --- a/util/model/dependency.go +++ b/util/model/dependency.go @@ -75,7 +75,8 @@ type DepTree struct { // 是否为直接依赖 Direct bool `json:"direct"` // 依赖路径 - Path string `json:"path,omitempty"` + Path string `json:"-"` + Paths []string `json:"paths,omitempty"` // 唯一的组件id，用来标识不同组件 ID int64 `json:"-"` // 父组件 @@ -97,6 +98,7 @@ func NewDepTree(parent *DepTree) *DepTree { Dependency: NewDependency(), Vulnerabilities: []*Vuln{}, Path: "", + Paths: nil, Parent: parent, Children: []*DepTree{}, licenseMap: map[string]struct{}{}, diff --git a/util/report/format.go b/util/report/format.go index 641e2962..fbd2163f 100644 --- a/util/report/format.go +++ b/util/report/format.go @@ -1,8 +1,9 @@ package report import ( + "fmt" "os" - "strings" + "util/args" "util/enum/language" "util/logs" "util/model" @@ -22,18 +23,52 @@ type TaskInfo struct { // format 按照输出内容格式化(不可逆) func format(dep *model.DepTree) { q := []*model.DepTree{dep} + // 保留要导出的数据 for len(q) > 0 { - node := q[0] - q = append(q[1:], node.Children...) - if node.Language != language.None { - node.LanguageStr = node.Language.String() + n := q[0] + q = append(q[1:], n.Children...) + if n.Language != language.None { + n.LanguageStr = n.Language.String() } - if node.Version != nil { - node.VersionStr = node.Version.Org + if n.Version != nil { + n.VersionStr = n.Version.Org + } + if n.Path != "" { + n.Paths = []string{n.Path} + } + n.Language = language.None + n.Version = nil + } + // 去重 + if args.Config.Dedup { + q = []*model.DepTree{dep} + dm := map[string]*model.DepTree{} + for len(q) > 0 { + n := q[0] + q = append(q[1:], n.Children...) + // 去重 + k := fmt.Sprintf("%s:%s@%s#%s", n.Vendor, n.Name, n.VersionStr, n.LanguageStr) + if d, ok := dm[k]; !ok { + dm[k] = n + } else { + // 已存在相同组件 + d.Paths = append(d.Paths, n.Path) + // 从父组件中移除当前组件 + if n.Parent != nil { + for i, c := range n.Parent.Children { + if c.ID == n.ID { + n.Parent.Children = append(n.Parent.Children[:i], n.Parent.Children[i+1:]...) + break + } + } + } + // 将当前组件的子组件转移到已存在组件的子依赖中 + d.Children = append(d.Children, n.Children...) + for _, c := range n.Children { + c.Parent = d + } + } } - node.Path = node.Path[strings.Index(node.Path, "/")+1:] - node.Language = language.None - node.Version = nil } } From 089d42701168da4f7698af0f456d34e3a0bfb0a2 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Tue, 31 May 2022 13:09:36 +0800 Subject: [PATCH 06/11] update html template --- analyzer/engine/parse.go | 5 +++-- analyzer/java/gradle.go | 2 +- analyzer/java/pom.go | 2 +- util/report/index.html | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/analyzer/engine/parse.go b/analyzer/engine/parse.go index 5f61b5dd..9049daa3 100644 --- a/analyzer/engine/parse.go +++ b/analyzer/engine/parse.go @@ -7,6 +7,7 @@ package engine import ( "path" + "strings" "util/filter" "util/model" ) @@ -36,7 +37,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for _, d := range analyzer.ParseFiles(files) { depRoot.Children = append(depRoot.Children, d) d.Parent = depRoot - if d.Name != "" && d.Version.Ok() { + if d.Name != "" && !strings.ContainsAny(d.Vendor+d.Name, "${}") && d.Version.Ok() { d.Path = path.Join(d.Path, d.Dependency.String()) } // 标识为直接依赖 @@ -73,7 +74,7 @@ func (e Engine) parseDependency(dirRoot *model.DirTree, depRoot *model.DepTree) for len(q) > 0 { n := q[0] q = append(q[1:], n.Children...) - if n.Name == "" || !n.Version.Ok() { + if n.Name == "" || strings.ContainsAny(n.Vendor+n.Name, "${}") || !n.Version.Ok() { n.Move(n.Parent) } } diff --git a/analyzer/java/gradle.go b/analyzer/java/gradle.go index 6b098660..ed460a1a 100644 --- a/analyzer/java/gradle.go +++ b/analyzer/java/gradle.go @@ -83,7 +83,7 @@ func parseGradle(root *model.DepTree, file *model.FileInfo) { regexs := []*regexp.Regexp{ regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?name: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), regexp.MustCompile(`group: ?['"]([^\s"']+)['"], ?module: ?['"]([^\s"']+)['"], ?version: ?['"]([^\s"']+)['"]`), - regexp.MustCompile(`['"]([^\s:]+):([^\s:]+):([^\s:]+)['"]`), + regexp.MustCompile(`['"]([^\s:'"]+):([^\s:'"]+):([^\s:'"]+)['"]`), } for _, line := range strings.Split(string(file.Data), "\n") { for _, re := range regexs { diff --git a/analyzer/java/pom.go b/analyzer/java/pom.go index ee4d686a..61ce5de0 100644 --- a/analyzer/java/pom.go +++ b/analyzer/java/pom.go @@ -129,7 +129,7 @@ func (p *Pom) GetProperty(key string) string { return p.Version case "${project.groupId}", "${groupId}", "${pom.groupId}": return p.GroupId - case "${project.artifactId}": + case "${project.artifactId}", "${artifactId}", "${pom.artifactId}": return p.ArtifactId case "${project.parent.version}", "${parent.version}": return p.Parent.Version diff --git a/util/report/index.html b/util/report/index.html index 76160e38..178ff1e7 100644 --- a/util/report/index.html +++ b/util/report/index.html @@ -1,2 +1,2 @@ -OpenSCA开源组件检测报告
\ No newline at end of file +OpenSCA开源组件检测报告
\ No newline at end of file From a43ecb21ac9c751003a2550e84066b257a66a2d9 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Mon, 13 Jun 2022 15:34:15 +0800 Subject: [PATCH 07/11] config default change to empty --- util/args/args.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/args/args.go b/util/args/args.go index b7fd7327..d0f54063 100644 --- a/util/args/args.go +++ b/util/args/args.go @@ -34,7 +34,7 @@ var ( ) func init() { - flag.StringVar(&ConfigPath, "config", "config.json", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") + flag.StringVar(&ConfigPath, "config", "", "(可选) 指定配置文件路径,指定后启动程序时将默认使用配置参数，配置参数与命令行输入参数冲突时优先使用输入参数") flag.StringVar(&Config.Path, "path", Config.Path, "(必须) 指定要检测的文件或目录路径,例: -path ./foo 或 -path ./foo.zip") flag.StringVar(&Config.Url, "url", Config.Url, "(可选,与token需一起使用) 从云漏洞库查询漏洞,指定要连接云服务的地址,例:-url https://opensca.xmirror.cn") flag.StringVar(&Config.Token, "token", Config.Token, "(可选,与url需一起使用) 云服务验证token,需要在云服务平台申请") From 62c1166431c6edbd73a4f5d1ea3f0c3c7e6629d8 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Mon, 13 Jun 2022 18:02:55 +0800 Subject: [PATCH 08/11] fix gradle --- analyzer/java/gradle.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/analyzer/java/gradle.go b/analyzer/java/gradle.go index ed460a1a..673cae18 100644 --- a/analyzer/java/gradle.go +++ b/analyzer/java/gradle.go @@ -88,9 +88,16 @@ func parseGradle(root *model.DepTree, file *model.FileInfo) { for _, line := range strings.Split(string(file.Data), "\n") { for _, re := range regexs { match := re.FindStringSubmatch(line) - // 有捕获内容且不以注释开头 - if len(match) == 4 && !strings.HasPrefix(strings.TrimSpace(line), "/") { + // 有捕获内容 + if len(match) == 4 && + // 不以注释开头 + !strings.HasPrefix(strings.TrimSpace(line), "/") && + // 不是测试组件 + !strings.Contains(strings.ToLower(line), "testimplementation") && + // 去掉非组件内容 + !strings.Contains(line, "//") { ver := model.NewVersion(match[3]) + // 版本号正常 if ver.Ok() { dep := model.NewDepTree(root) dep.Vendor = match[1] From 2ae561cf4f2b77fa6bed283cce1a87b699d1c6cd Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Thu, 16 Jun 2022 10:54:44 +0800 Subject: [PATCH 09/11] fix readme --- .github/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/README.md b/.github/README.md index dd114aef..9548a15f 100644 --- a/.github/README.md +++ b/.github/README.md @@ -143,7 +143,7 @@ opensca-cli -db db.json -path ${project_path} OpenSCA is an open source project, we appreciate your help! -To contribute, please read our [Contributing Guideline](./docs/Contributing%20Guideline-en%20v1.0.md). +To contribute, please read our [Contributing Guideline](../docs/Contributing%20Guideline-en%20v1.0.md). From 2ead78b82da8b1640977f8d05de28eb235923ea5 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Thu, 16 Jun 2022 14:36:44 +0800 Subject: [PATCH 10/11] update html version --- cli/main.go | 3 +++ util/report/format.go | 1 + util/report/index.html | 4 ++-- util/report/json.go | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cli/main.go b/cli/main.go index 06bdaeaf..796942c5 100644 --- a/cli/main.go +++ b/cli/main.go @@ -15,6 +15,8 @@ import ( "util/report" ) +var version string + func main() { args.Parse() if len(args.Config.Path) > 0 { @@ -26,6 +28,7 @@ func main() { // output 输出结果 func output(depRoot *model.DepTree, taskInfo report.TaskInfo) { + taskInfo.ToolVersion = version // 记录依赖 logs.Debug("\n" + depRoot.String()) // 输出结果 diff --git a/util/report/format.go b/util/report/format.go index fbd2163f..4da0940e 100644 --- a/util/report/format.go +++ b/util/report/format.go @@ -11,6 +11,7 @@ import ( // 任务检查信息 type TaskInfo struct { + ToolVersion string `json:"tool_version"` AppName string `json:"app_name"` Size int64 `json:"size"` StartTime string `json:"start_time"` diff --git a/util/report/index.html b/util/report/index.html index 178ff1e7..90a4ce35 100644 --- a/util/report/index.html +++ b/util/report/index.html @@ -1,2 +1,2 @@ -OpenSCA开源组件检测报告
\ No newline at end of file +OpenSCA开源组件检测报告
\ No newline at end of file diff --git a/util/report/json.go b/util/report/json.go index e92a317f..addc7022 100644 --- a/util/report/json.go +++ b/util/report/json.go @@ -13,11 +13,11 @@ func Json(dep *model.DepTree, taskInfo TaskInfo) []byte { taskInfo.ErrorString = taskInfo.Error.Error() } if data, err := json.Marshal(struct { - *model.DepTree TaskInfo TaskInfo `json:"task_info"` + *model.DepTree }{ - DepTree: dep, TaskInfo: taskInfo, + DepTree: dep, }); err != nil { logs.Error(err) } else { From 9646503129611a6e41229ef0412f42e85a7ae075 Mon Sep 17 00:00:00 2001 From: luotianqi777 Date: Thu, 16 Jun 2022 17:11:23 +0800 Subject: [PATCH 11/11] fix oss.py --- analyzer/python/oss.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/analyzer/python/oss.py b/analyzer/python/oss.py index 8cd4957c..4179e668 100644 --- a/analyzer/python/oss.py +++ b/analyzer/python/oss.py @@ -1,3 +1,4 @@ +import re import sys import json @@ -17,10 +18,16 @@ def parse_setup_py(setup_py_path): pass # 获取setup参数 args = {} - exec(f.read().replace("setup(", "args=setup("), args) + code = re.sub('(?>oss_end'.format(js)) + data = args['args'] + info = {} + for k in ['name','version','license','packages','install_requires','requires']: + if k in data: + info[k] = data[k] + print('oss_start<<{}>>oss_end'.format(json.dumps(info))) if __name__ == "__main__": if len(sys.argv) > 1: