From 8a4ed5b7aaa91e8d1aeff6adea32ce21761e01e9 Mon Sep 17 00:00:00 2001 From: xorhex Date: Sun, 1 Jan 2023 20:58:54 -0500 Subject: [PATCH 1/3] Added AssemblyLine as another source. Fixed some bugs around --readupdate flag. --- config.go | 37 +++++++++++--- download.go | 138 +++++++++++++++++++++++++++++++++++++++++++++++++- hashes.go | 22 ++++++++ mlget.go | 20 +++++--- mlget_test.go | 35 +++++++++++++ 5 files changed, 237 insertions(+), 15 deletions(-) diff --git a/config.go b/config.go index ad39b4a..30a20da 100644 --- a/config.go +++ b/config.go @@ -67,6 +67,7 @@ type MalwareRepoType int64 const ( NotSupported MalwareRepoType = iota //NotSupported must always be first, or other things won't work as expected + AssemblyLine AnyRun CapeSandbox FileScanIo @@ -161,6 +162,9 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h case URLScanIO: found, filename = urlscanio(mcr.Host, mcr.Api, hash) checkedRepo = URLScanIO + case AssemblyLine: + found, filename = assemblyline(mcr.Host, mcr.User, mcr.Api, mcr.IgnoreTLSErrors, hash) + checkedRepo = AssemblyLine //case AnyRun: // found, filename = anyrun(mcr.Host, hash) // checkedRepo = AnyRun @@ -196,6 +200,10 @@ func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool if repo.Host != "" { return true } + case AssemblyLine: + if repo.Host != "" && repo.Api != "" && repo.User != "" { + return true + } default: if repo.Host != "" && repo.Api != "" { return true @@ -207,6 +215,7 @@ func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { var host string var api string + var user string var default_url string @@ -259,12 +268,17 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { fmt.Println("Using the default url") host = default_url } + if malrepo == AssemblyLine { + fmt.Println("Enter User Name:") + fmt.Print(">> ") + fmt.Scanln(&user) + } if malrepo != MalwareBazaar && malrepo != ObjectiveSee && malrepo != AnyRun { fmt.Println("Enter API Key:") fmt.Print(">> ") fmt.Scanln(&api) } - return RepositoryConfigEntry{Host: host, Api: api, Type: malrepo.String()}, nil + return RepositoryConfigEntry{Host: host, User: user, Api: api, Type: malrepo.String()}, nil } func (malrepo MalwareRepoType) String() string { @@ -303,6 +317,8 @@ func (malrepo MalwareRepoType) String() string { return "URLScanIO" case AnyRun: return "AnyRun" + case AssemblyLine: + return "AssemblyLine" case UploadMWDB: return "UploadMWDB" @@ -356,7 +372,8 @@ func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract } else { valid, calculatedHash := hash.ValidateFile(filename) if !valid { - fmt.Printf(" [!] Downloaded file hash %s does not match searched for hash %s\nTrying another source.\n", calculatedHash, hash.Hash) + fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\nTrying another source.\n", calculatedHash, hash.Hash) + deleteInvalidFile(filename) continue } else { fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", hash.Hash) @@ -408,6 +425,8 @@ func getMalwareRepoByFlagName(name string) MalwareRepoType { return URLScanIO case strings.ToLower("ar"): return AnyRun + case strings.ToLower("al"): + return AssemblyLine } return NotSupported } @@ -448,6 +467,8 @@ func getMalwareRepoByName(name string) MalwareRepoType { return URLScanIO case strings.ToLower("AnyRun"): return AnyRun + case strings.ToLower("AssemblyLine"): + return AssemblyLine case strings.ToLower("UploadMWDB"): return UploadMWDB } @@ -465,11 +486,13 @@ func getConfigsByType(repoType MalwareRepoType, repos []RepositoryConfigEntry) [ } type RepositoryConfigEntry struct { - Type string `yaml:"type"` - Host string `yaml:"url"` - Api string `yaml:"api"` - QueryOrder int `yaml:"queryorder"` - Password string `yaml:"pwd"` + Type string `yaml:"type"` + Host string `yaml:"url"` + Api string `yaml:"api"` + QueryOrder int `yaml:"queryorder"` + Password string `yaml:"pwd"` + User string `yaml:"user"` + IgnoreTLSErrors bool `yaml:"ignoretlserrors"` } func LoadConfig(filename string) ([]RepositoryConfigEntry, error) { diff --git a/download.go b/download.go index 0049e2b..7586a47 100644 --- a/download.go +++ b/download.go @@ -3,6 +3,7 @@ package main import ( "bytes" "compress/gzip" + "crypto/tls" "encoding/base64" "encoding/json" "errors" @@ -55,6 +56,33 @@ type MalwareBazarQueryData struct { File_name string `json:"file_name"` } +type AssemblyLineQuery struct { + Error_message string `json:"api_error_message"` + Response *AssemblyLineQueryResponse `json:"api_response"` + Server_version string `json:"api_server_version"` + Status_Code int `json:"api_status_code"` +} + +type AssemblyLineQueryResponse struct { + AL *AssemblyLineQueryALResponse `json:"al"` +} + +type AssemblyLineQueryALResponse struct { + Error string `json:"error"` + Items []AssemblyLineQueryItem `json:"items"` +} + +type AssemblyLineQueryItem struct { + Classification string `json:"classification"` + Data *AssemblyLineQueryData `json:"data"` +} + +type AssemblyLineQueryData struct { + Md5 string `json:"md5"` + Sha1 string `json:"sha1` + Sha256 string `json:"sha256"` +} + type TriageQuery struct { Data []TriageQueryData `json:"data"` } @@ -1081,7 +1109,7 @@ func unpacmeDownload(uri string, api string, hash Hash) (bool, string) { } else if response.StatusCode == http.StatusForbidden { fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") return false, "" - } else if response.StatusCode == 401 { + } else if response.StatusCode == http.StatusUnauthorized { fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") return false, "" } @@ -1272,6 +1300,114 @@ func parseMalpediaJson(byteValue []byte) (bool, []MalpediaData) { return true, malpediaJsonItems } +func assemblyline(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) { + if api == "" { + fmt.Println(" [!] !! Missing Key !!") + return false, "" + } + if user == "" { + fmt.Println(" [!] !! Missing User !!") + return false, "" + } + + if hash.HashType != sha256 { + fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash) + + request, error := http.NewRequest("GET", uri+"/hash_search/"+url.PathEscape(hash.Hash)+"/", nil) + if error != nil { + fmt.Println(error) + return false, "" + } + + request.Header.Set("Content-Type", "application/json; charset=UTF-8") + request.Header.Set("x-user", user) + request.Header.Set("x-apikey", api) + + tr := &http.Transport{} + if ignoretlserrors { + fmt.Printf(" [!] Ignoring Certificate Errors.\n") + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + client := &http.Client{Transport: tr} + response, error := client.Do(request) + if error != nil { + fmt.Println(error) + return false, "" + } + defer response.Body.Close() + + if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized { + fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") + return false, "" + } + + byteValue, _ := ioutil.ReadAll(response.Body) + + var data = AssemblyLineQuery{} + error = json.Unmarshal(byteValue, &data) + + if error != nil { + fmt.Println(error) + return false, "" + } + + if data.Response.AL == nil { + return false, "" + } + hash.Hash = data.Response.AL.Items[0].Data.Sha256 + hash.HashType = sha256 + fmt.Printf(" [-] Using hash %s\n", hash.Hash) + } + + return assemblylineDownload(uri, user, api, ignoretlserrors, hash) + +} + +func assemblylineDownload(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) { + request, error := http.NewRequest("GET", uri+"/file/download/"+url.PathEscape(hash.Hash)+"/?encoding=raw", nil) + if error != nil { + fmt.Println(error) + return false, "" + } + + request.Header.Set("Content-Type", "application/json; charset=UTF-8") + request.Header.Set("x-user", user) + request.Header.Set("x-apikey", api) + + tr := &http.Transport{} + if ignoretlserrors { + tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + client := &http.Client{Transport: tr} + response, error := client.Do(request) + if error != nil { + fmt.Println(error) + return false, "" + } + + defer response.Body.Close() + + if response.StatusCode == http.StatusNotFound { + return false, "" + } else if response.StatusCode == http.StatusForbidden { + fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") + return false, "" + } else if response.StatusCode == http.StatusUnauthorized { + fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n") + return false, "" + } + + error = writeToFile(response.Body, hash.Hash) + if error != nil { + fmt.Println(error) + return false, "" + } + fmt.Printf(" [+] Downloaded %s\n", hash.Hash) + return true, hash.Hash +} + func extractGzip(hash string) error { r, err := os.Open(hash + ".gzip") if err != nil { diff --git a/hashes.go b/hashes.go index 0a32f3f..0e1f6a3 100644 --- a/hashes.go +++ b/hashes.go @@ -11,6 +11,8 @@ import ( "strings" ) +var alwaysDeleteInvalidFile = false + type Hashes struct { Hashes []Hash } @@ -146,6 +148,26 @@ func (h Hash) Validate(bytes []byte) (bool, string) { } } +func deleteInvalidFile(filename string) { + if !alwaysDeleteInvalidFile { + var delete_file string + fmt.Printf(" [?] Delete invalid file? [A/Y/n] Always delete/Yes, this time/No, not this time\n") + fmt.Scanln(&delete_file) + if strings.ToUpper(delete_file) == "Y" || delete_file == "" || strings.ToUpper(delete_file) == "A" { + os.Remove(filename) + fmt.Printf(" [!] Deleted invalid file\n") + if strings.ToUpper(delete_file) == "A" { + alwaysDeleteInvalidFile = true + } + } else { + fmt.Printf(" [!] Keeping invalid file\n") + } + } else { + os.Remove(filename) + fmt.Printf(" [!] Deleted invalid file\n") + } +} + func hashType(hash string) (HashTypeOption, error) { match, _ := regexp.MatchString("^[A-Fa-f0-9]{64}$", hash) if match { diff --git a/mlget.go b/mlget.go index 8a5c8f3..b4afd93 100644 --- a/mlget.go +++ b/mlget.go @@ -26,7 +26,7 @@ var commentsFlag []string var versionFlag bool var noValidationFlag bool -var version string = "3.0.1" +var version string = "3.1.0" func usage() { fmt.Println("mlget - A command line tool to download malware from a variety of sources") @@ -42,7 +42,7 @@ func usage() { } func init() { - flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - cp (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tg (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.") + flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - al (AssemblyLine)\n - cs (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tg (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.") flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)") flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (for later use with the --read flag)") flag.BoolVar(&helpFlag, "help", false, "Print the help message") @@ -102,6 +102,14 @@ func main() { args := flag.Args() + if apiFlag != "" { + flaggedRepo := getMalwareRepoByFlagName(apiFlag) + if flaggedRepo == NotSupported { + fmt.Printf("Invalid or unsupported malware repo type: %s\nCheck the help for the values to pass to the --from parameter\n", apiFlag) + return + } + } + if apiFlag != "" && downloadOnlyFlag { fmt.Printf(("Can't use both the --from flag and the --downloadonly flag together")) return @@ -174,10 +182,6 @@ func main() { if apiFlag != "" { flaggedRepo := getMalwareRepoByFlagName(apiFlag) - if flaggedRepo == NotSupported { - fmt.Printf("Invalid or unsupported malware repo type: %s\nCheck the help for the values to pass to the --from parameter\n", apiFlag) - continue - } fmt.Printf("Looking on %s\n", getMalwareRepoByFlagName(apiFlag)) @@ -196,7 +200,9 @@ func main() { } else { valid, calculatedHash := h.ValidateFile(filename) if !valid { - fmt.Printf(" [!] Downloaded file hash %s does not match searched for hash %s\n", calculatedHash, h.Hash) + fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\n", calculatedHash, h.Hash) + deleteInvalidFile(filename) + notFoundHashes, _ = addHash(notFoundHashes, h) continue } else { fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", h.Hash) diff --git a/mlget_test.go b/mlget_test.go index dd3c72c..99dfdee 100644 --- a/mlget_test.go +++ b/mlget_test.go @@ -625,6 +625,41 @@ func TestURLScanIo(t *testing.T) { } +func TestAssemblyLine(t *testing.T) { + home, _ := os.UserHomeDir() + cfg, err := LoadConfig(path.Join(home, ".mlget.yml")) + if err != nil { + log.Fatal() + t.Errorf("%v", err) + } + + scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name()) + if err != nil { + log.Fatal() + t.Errorf("%v", err) + } + + ht, _ := hashType(scfg.Hash) + hash := Hash{HashType: ht, Hash: scfg.Hash} + + var osq ObjectiveSeeQuery + result, filename, _ := AssemblyLine.QueryAndDownload(cfg, hash, false, osq) + + if !result { + t.Errorf("Assemblyline failed") + } else { + valid, errmsg := hash.ValidateFile(filename) + + if !valid { + os.Remove(filename) + t.Errorf(errmsg) + } else { + os.Remove(filename) + } + } + +} + /* func TestAnyRun(t *testing.T) { home, _ := os.UserHomeDir() From 615f8866c42e8a24fbd361bba1ae9282d25dd630 Mon Sep 17 00:00:00 2001 From: xorhex Date: Sun, 15 Oct 2023 19:45:24 -0400 Subject: [PATCH 2/3] Triage features, assembly line added, and Malpedia bug fix. --- .github/workflows/go.yml | 2 +- .github/workflows/release.yml | 2 - config.go | 7 +- download.go | 121 ++++++++++++++++++++++++++-------- go.mod | 2 +- hashes.go | 26 ++++++++ history.go | 36 +++++++++- mlget.go | 72 ++++++++++++-------- mlget_test.go | 5 +- 9 files changed, 208 insertions(+), 65 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index d45a667..a250c0c 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.19 + go-version: 1.21.3 - name: Build run: go get -u && go build -v ./... diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6397fcd..0c12486 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,5 @@ # .github/workflows/release.yaml -# .github/workflows/release.yaml - on: release: types: [created, draft] diff --git a/config.go b/config.go index 30a20da..2b20f8f 100644 --- a/config.go +++ b/config.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "os" "path" @@ -173,7 +172,7 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h checkedRepo = UploadMWDB } // So some repos we can't download from but we want to know that it exists at that service - // At the moment, this is just Any.Run but suspecct more will be added as time goes on + // At the moment, this is just Any.Run but suspect more will be added as time goes on if checkedRepo == AnyRun && found { continue } @@ -223,7 +222,7 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) { case NotSupported: return RepositoryConfigEntry{}, fmt.Errorf("malware repository rype, %s, is not supported", malrepo.String()) case MalwareBazaar: - default_url = "https://mb-api.abuse.ch/api/v1" + default_url = "https://mb-api.abuse.ch/api/v1/" case Malshare: default_url = "https://malshare.com" case MWDB: @@ -549,7 +548,7 @@ func parseFile(path string) (map[string]RepositoryConfigEntry, error) { return nil, err } - f, err := ioutil.ReadFile(path) + f, err := os.ReadFile(path) if err != nil { fmt.Printf("%v", err) return nil, err diff --git a/download.go b/download.go index 7586a47..48fa177 100644 --- a/download.go +++ b/download.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net/http" "net/url" "os" "regexp" + "strings" "github.com/yeka/zip" ) @@ -79,7 +79,7 @@ type AssemblyLineQueryItem struct { type AssemblyLineQueryData struct { Md5 string `json:"md5"` - Sha1 string `json:"sha1` + Sha1 string `json:"sha1"` Sha256 string `json:"sha256"` } @@ -125,7 +125,7 @@ func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) { defer response.Body.Close() if response.StatusCode == http.StatusOK { - byteValue, _ := ioutil.ReadAll(response.Body) + byteValue, _ := io.ReadAll(response.Body) var data = ObjectiveSeeQuery{} error = json.Unmarshal(byteValue, &data) @@ -239,7 +239,7 @@ func joesandbox(uri string, api string, hash Hash) (bool, string) { if response.StatusCode == http.StatusOK { - byteValue, error := ioutil.ReadAll(response.Body) + byteValue, error := io.ReadAll(response.Body) if error != nil { fmt.Println(error) return false, "" @@ -392,7 +392,7 @@ func inquestlabs(uri string, api string, hash Hash) (bool, string) { if response.StatusCode == http.StatusOK { - byteValue, _ := ioutil.ReadAll(response.Body) + byteValue, _ := io.ReadAll(response.Body) var data = InquestLabsQuery{} error = json.Unmarshal(byteValue, &data) @@ -642,7 +642,7 @@ func hybridAnlysis(uri string, api string, hash Hash, doNotExtract bool) (bool, fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") return false, "" } - byteValue, _ := ioutil.ReadAll(response.Body) + byteValue, _ := io.ReadAll(response.Body) var data = HybridAnalysisQuery{} error = json.Unmarshal(byteValue, &data) @@ -748,7 +748,7 @@ func triage(uri string, api string, hash Hash) (bool, string) { if response.StatusCode == http.StatusOK { - byteValue, error := ioutil.ReadAll(response.Body) + byteValue, error := io.ReadAll(response.Body) if error != nil { fmt.Println(error) return false, "" @@ -802,8 +802,54 @@ func traigeDownload(uri string, api string, sampleId string, hash Hash) (bool, s fmt.Println(error) return false, "" } - fmt.Printf(" [+] Downloaded %s\n", hash.Hash) - return true, hash.Hash + // Triage will download an archive file that contians the hash in question sometimes versus the actual sample being requested + hashMatch, _ := hash.ValidateFile(hash.Hash) + if !hashMatch { + files, err := extractPwdZip(hash.Hash, "", false, hash) + if err != nil { + fmt.Println(error) + return false, "" + } + + found := false + + fmt.Printf(" [-] The downloaded file appears to be a zip file in which the requested file should be located.\n") + for _, f := range files { + fmt.Printf(" [-] Checking file: %s\n", f.Name) + hashMatch, _ = hash.ValidateFile(f.Name) + if !hashMatch { + err = os.Remove(f.Name) + if err != nil { + fmt.Println(" [!] Error when deleting file: ", f.Name) + fmt.Println(err) + } + } else { + fmt.Printf(" [+] %s is hash %s\n", f.Name, hash.Hash) + err = os.Rename(f.Name, hash.Hash) + if err != nil { + fmt.Println(" [!] Error when renaming file: ", f.Name) + fmt.Println(err) + } else { + found = true + } + } + } + if !found { + fmt.Printf(" [!] Hash %s not found\n", hash.Hash) + err = os.Remove(hash.Hash) + if err != nil { + fmt.Println(" [!] Error when deleting file: ", hash.Hash) + fmt.Println(err) + } + return false, "" + } else { + fmt.Printf(" [+] Found %s\n", hash.Hash) + return true, hash.Hash + } + } else { + fmt.Printf(" [+] Downloaded %s\n", hash.Hash) + return true, hash.Hash + } } func malshare(url string, api string, hash Hash) (bool, string) { @@ -873,7 +919,7 @@ func malwareBazaar(uri string, hash Hash, doNotExtract bool, password string) (b return false, "" } - byteValue, _ := ioutil.ReadAll(response.Body) + byteValue, _ := io.ReadAll(response.Body) var data = MalwareBazarQuery{} error = json.Unmarshal(byteValue, &data) @@ -900,28 +946,39 @@ func malwareBazaar(uri string, hash Hash, doNotExtract bool, password string) (b func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password string) (bool, string) { query := "query=get_file&sha256_hash=" + hash.Hash - values, error := url.ParseQuery(query) - if error != nil { - fmt.Println(error) + values, err := url.ParseQuery(query) + if err != nil { + fmt.Println(err) return false, "" } client := &http.Client{} - response, error := client.PostForm(uri, values) - if error != nil { - fmt.Println(error) + + response, err := client.PostForm(uri, values) + if err != nil { + fmt.Println(err) return false, "" } defer response.Body.Close() if response.Header["Content-Type"][0] == "application/json" { + if response.StatusCode == http.StatusMethodNotAllowed { + if !strings.HasSuffix(uri, "/") { + fmt.Printf(" [!] Trying again with a trailing slash: %s/\n", uri) + return malwareBazaarDownload(uri+"/", hash, doNotExtract, password) + } else { + fmt.Printf(" [!] Normally the response code: %s means that the provided URL %s needs a trailing slash (to avoid the redirect), but this already has a trailing slash.\nPlease file a bug report at https://github.com/xorhex/mlget/issues\n", response.Status, uri) + } + } else { + fmt.Printf(" [!] %s\n", response.Status) + } return false, "" } - error = writeToFile(response.Body, hash.Hash+".zip") - if error != nil { - fmt.Println(error) + err = writeToFile(response.Body, hash.Hash+".zip") + if err != nil { + fmt.Println(err) return false, "" } @@ -930,7 +987,7 @@ func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password st return true, hash.Hash + ".zip" } else { fmt.Println(" [-] Extracting...") - files, err := extractPwdZip(hash.Hash, password) + files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) if err != nil { fmt.Println(err) return false, "" @@ -984,6 +1041,8 @@ func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool, pa return false, "" } else if response.StatusCode == http.StatusForbidden { fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n") + fmt.Printf(" [!] If you are sure this is correct, then test downloading a sample you've access to in their platform. It should work.\n") + fmt.Printf(" [!] Not sure why it does not just return a 404 instead; when the creds are correct but the file is not available.\n") return false, "" } @@ -998,7 +1057,7 @@ func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool, pa return true, hash.Hash + ".zip" } else { fmt.Println(" [-] Extracting...") - files, err := extractPwdZip(hash.Hash, password) + files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) if err != nil { fmt.Println(err) return false, "" @@ -1058,7 +1117,7 @@ func vxsharedownload(uri string, api string, hash Hash, doNotExtract bool, passw return true, hash.Hash + ".zip" } else { fmt.Println(" [-] Extracting...") - files, err := extractPwdZip(hash.Hash, password) + files, err := extractPwdZip(hash.Hash+".zip", password, true, hash) if err != nil { fmt.Println(err) return false, "" @@ -1253,7 +1312,7 @@ func malpediaDownload(uri string, api string, hash Hash) (bool, string) { return false, "" } - byteValue, _ := ioutil.ReadAll(response.Body) + byteValue, _ := io.ReadAll(response.Body) jsonParseSuccesful, mpData := parseMalpediaJson(byteValue) if jsonParseSuccesful { for _, item := range mpData { @@ -1342,7 +1401,7 @@ func assemblyline(uri string, user string, api string, ignoretlserrors bool, has return false, "" } - byteValue, _ := ioutil.ReadAll(response.Body) + byteValue, _ := io.ReadAll(response.Body) var data = AssemblyLineQuery{} error = json.Unmarshal(byteValue, &data) @@ -1424,9 +1483,9 @@ func extractGzip(hash string) error { return err } -func extractPwdZip(hash string, password string) ([]*zip.File, error) { +func extractPwdZip(file string, password string, renameFileAsHash bool, hash Hash) ([]*zip.File, error) { - r, err := zip.OpenReader(hash + ".zip") + r, err := zip.OpenReader(file) if err != nil { log.Fatal(err) } @@ -1444,7 +1503,15 @@ func extractPwdZip(hash string, password string) ([]*zip.File, error) { log.Fatal(err) } - out, error := os.Create(hash) + var name string + + if !renameFileAsHash { + name = f.Name + } else { + name = hash.Hash + } + + out, error := os.Create(name) if error != nil { return nil, error } diff --git a/go.mod b/go.mod index 42caa1a..1f02b97 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module xorhex.com/mlget -go 1.19 +go 1.21.3 require ( github.com/kr/pretty v0.1.0 diff --git a/hashes.go b/hashes.go index 0e1f6a3..8feedaa 100644 --- a/hashes.go +++ b/hashes.go @@ -183,3 +183,29 @@ func hashType(hash string) (HashTypeOption, error) { } return NotAValidHashType, errors.New("not a valid hash") } + +func extractHashes(text string) ([]string, error) { + hashes := make([]string, 0) + + re := regexp.MustCompile(`>\s*[A-Fa-f0-9]{64}\s*<`) + matches := re.FindAllStringSubmatch(text, 100) + for m := range matches { + hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) + } + re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{40}\s*<`) + matches = re.FindAllStringSubmatch(text, 100) + for m := range matches { + hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) + } + re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{32}\s*<`) + matches = re.FindAllStringSubmatch(text, 100) + for m := range matches { + hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1])) + } + + if len(hashes) > 0 { + return hashes, fmt.Errorf("no hashes found") + } + + return hashes, nil +} diff --git a/history.go b/history.go index 9faa38f..9171a22 100644 --- a/history.go +++ b/history.go @@ -138,6 +138,40 @@ func parseFileForHashEntries(filename string) ([]Hash, error) { pHash, err = parseFileHashEntry(hash, tags, comments) if err == nil { hashes = append(hashes, pHash) + } else { + // Try splitting on \t and check to see if any of the values match a hash + // This is useful for reading files from the web that list sample hashes + // This still assumes there is only one hash per line as it stops after the + // first hash is found on that line + s := func(c rune) bool { + return c == '\t' + } + + line := strings.FieldsFunc(strings.TrimSpace(text), s) + if len(line) > 0 { + for _, element := range line { + lHash := Hash{} + lHash, err := parseFileHashEntry(strings.TrimSpace(element), tags, comments) + if err == nil { + hashes = append(hashes, lHash) + break + } else { + + matches, err := extractHashes(strings.TrimSpace(element)) + if err != nil { + fmt.Println(err) + } + for m := range matches { + tags := []string{} + comments := []string{} + lHash, err = parseFileHashEntry(matches[m], tags, comments) + if err == nil { + hashes = append(hashes, lHash) + } + } + } + } + } } } } @@ -171,7 +205,7 @@ func parseFileHashEntry(hash string, tags []string, comments []string) (Hash, er fmt.Printf("\n Skipping %s because it's %s\n", hash, err) return Hash{}, err } - fmt.Printf("Hash found: %s\n", hash) // token in unicode-char + fmt.Printf("\nHash found: %s\n", hash) // token in unicode-char hashS := Hash{Hash: hash, HashType: ht} if len(tags) > 0 { hashS.Tags = tags diff --git a/mlget.go b/mlget.go index b4afd93..768ce5e 100644 --- a/mlget.go +++ b/mlget.go @@ -25,8 +25,11 @@ var tagsFlag []string var commentsFlag []string var versionFlag bool var noValidationFlag bool +var webserver bool +var ip string +var port int -var version string = "3.1.0" +var version string = "3.2.0" func usage() { fmt.Println("mlget - A command line tool to download malware from a variety of sources") @@ -42,7 +45,7 @@ func usage() { } func init() { - flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - al (AssemblyLine)\n - cs (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tg (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.") + flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - al (AssemblyLine)\n - cs (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tr (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.") flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)") flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (for later use with the --read flag)") flag.BoolVar(&helpFlag, "help", false, "Print the help message") @@ -57,6 +60,9 @@ func init() { flag.BoolVar(&downloadOnlyFlag, "downloadonly", false, "Download from any source, including your personal instance of MWDB.\nWhen this flag is set; it will NOT update any output file with the hashes not found.\nAnd it will not upload to any of the UploadMWDB instances.") flag.BoolVar(&versionFlag, "version", false, "Print the version number") flag.BoolVar(&noValidationFlag, "novalidation", false, "Turn off post download hash check verification") + //flag.BoolVar(&webserver, "webserver", false, "Run the webserver to handle request via the web browser.") + //flag.StringVar(&ip, "bind", "127.0.0.1", "Bind to IP. Default is localhost") + //flag.IntVar(&port, "port", 8080, "Bind to port. Default is 8080") } func main() { @@ -64,11 +70,6 @@ func main() { noSamplesRepoList := []MalwareRepoType{AnyRun} doNotValidatehash := []MalwareRepoType{ObjectiveSee} - if versionFlag { - fmt.Printf("Mlget version: %s\n", version) - return - } - homeDir, err := os.UserHomeDir() if err != nil { fmt.Println(err) @@ -85,6 +86,11 @@ func main() { flag.Parse() + if versionFlag { + fmt.Printf("Mlget version: %s\n", version) + return + } + if helpFlag { usage() return @@ -102,6 +108,36 @@ func main() { args := flag.Args() + if webserver { + runWebServer(ip, port) + } else { + downloadMalwareFromCLI(args, cfg, noSamplesRepoList, doNotValidatehash) + } + +} + +func parseArgHashes(hashes []string, tags []string, comments []string) Hashes { + parsedHashes := Hashes{} + for _, h := range hashes { + ht, err := hashType(h) + if err != nil { + fmt.Printf("\n Skipping %s because it's %s\n", h, err) + continue + } + fmt.Printf("Hash found: %s\n", h) // token in unicode-char + hash := Hash{Hash: h, HashType: ht} + if len(tags) > 0 { + hash.Tags = tags + } + if len(comments) > 0 { + hash.Comments = comments + } + parsedHashes, _ = addHash(parsedHashes, hash) + } + return parsedHashes +} + +func downloadMalwareFromCLI(args []string, cfg []RepositoryConfigEntry, noSamplesRepoList []MalwareRepoType, doNotValidatehash []MalwareRepoType) { if apiFlag != "" { flaggedRepo := getMalwareRepoByFlagName(apiFlag) if flaggedRepo == NotSupported { @@ -116,6 +152,7 @@ func main() { } var osq ObjectiveSeeQuery + var err error osConfigs := getConfigsByType(ObjectiveSee, cfg) // Can have multiple Objective-See configs but only the first one to load will be used for _, osc := range osConfigs { @@ -269,23 +306,6 @@ func main() { } } -func parseArgHashes(hashes []string, tags []string, comments []string) Hashes { - parsedHashes := Hashes{} - for _, h := range hashes { - ht, err := hashType(h) - if err != nil { - fmt.Printf("\n Skipping %s because it's %s\n", h, err) - continue - } - fmt.Printf("Hash found: %s\n", h) // token in unicode-char - hash := Hash{Hash: h, HashType: ht} - if len(tags) > 0 { - hash.Tags = tags - } - if len(comments) > 0 { - hash.Comments = comments - } - parsedHashes, _ = addHash(parsedHashes, hash) - } - return parsedHashes +func downloadMalwareFromWebServer(hashes Hashes) { + } diff --git a/mlget_test.go b/mlget_test.go index 99dfdee..34bb554 100644 --- a/mlget_test.go +++ b/mlget_test.go @@ -3,7 +3,6 @@ package main import ( "errors" "fmt" - "io/ioutil" "log" "os" "path" @@ -25,7 +24,7 @@ func parseTestConfig(path string, testName string) (TestConfigEntry, error) { return tce, err } - f, err := ioutil.ReadFile(path) + f, err := os.ReadFile(path) if err != nil { fmt.Printf("%v", err) return tce, err @@ -440,7 +439,7 @@ func TestMalwareBazaar(t *testing.T) { result, filename, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq) if !result { - t.Errorf("Malshare failed") + t.Errorf("MalwareBazaar failed") } else { valid, errmsg := hash.ValidateFile(filename) From e85f69677199dc17c729dcdda94036358807605f Mon Sep 17 00:00:00 2001 From: xorhex Date: Sun, 15 Oct 2023 19:52:39 -0400 Subject: [PATCH 3/3] Run as a webservice files added. Not functioning so path to code disabled. --- mlget-test-config/samples.yaml | 60 +++++++++++++++++++++++++++++++++ mlweb.go | 48 ++++++++++++++++++++++++++ web/scripts/script.js | 29 ++++++++++++++++ web/styles/style.css | 0 web/templates/index.html | 61 ++++++++++++++++++++++++++++++++++ 5 files changed, 198 insertions(+) create mode 100644 mlget-test-config/samples.yaml create mode 100644 mlweb.go create mode 100644 web/scripts/script.js create mode 100644 web/styles/style.css create mode 100644 web/templates/index.html diff --git a/mlget-test-config/samples.yaml b/mlget-test-config/samples.yaml new file mode 100644 index 0000000..bd01dd9 --- /dev/null +++ b/mlget-test-config/samples.yaml @@ -0,0 +1,60 @@ +test 1: + name: TestJoeSandbox + hash: e21ff9323365bca4131d8ec0a24b75521857776569316ffe3f7c97f327256d1b +test 2: + name: TestObjectiveSee + hash: 458a9ac086116fa011c1a7bd49ac15f386cd95e39eb6b7cd5c5125aef516c78c +test 3: + name: TestCapeSandbox + hash: 28eefc36104bebb595fb38cae21a7d0a +test 4: + name: TestInquestLabsLookUp + hash: b3f868fa1af24f270e3ecc0ecb79325e +test 5: + name: TestInquestLabsNoLookUp + hash: 6b425804d43bb369211bbec59808807730a908804ca9b8c09081139179bbc868 +test 6: + name: TestVirusTotal + hash: 21cc9c0ae5f97b66d69f1ff99a4fed264551edfe0a5ce8d5449942bf8f0aefb2 +test 7: + name: TestMWDB + hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 +test 8: + name: TestPolyswarm + hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 +test 9: + name: TestHybridAnalysis + hash: ed2f501408a7a6e1a854c29c4b0bc5648a6aa8612432df829008931b3e34bf56 +test 10: + name: TestTriage + hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 +test 11: + name: TestMalShare + hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18 +test 12: + name: TestMalwareBazaar + hash: 001bffcdd170c8328601006ad54a221d1073ba04fbdca556749cf1b041cfad97 +test 13: + name: TestMalpedia + hash: 78668c237097651d64c97b25fc86c74096bfe1ed53e1004445f118ea5feaa3ad +test 14: + name: TestUnpacme + hash: 0219a79a2f47da42601568ee4a41392aa429f62a1fb01080cb68540074449c92 +test 15: + name: TestVxShare + hash: 1c11c963a417674e1414bac05fdbfa5cfa09f92c7b0d9882aeb55ce2a058d668 +test 16: + name: TestFileScanIo + hash: 2799af2efd698da215afc9c88da3b1e84b00137433d9444a5c11d69092b3f80d +test 17: + name: TestURLScanIo + hash: 5b027ada26a610e97ab4ef9efb1118b377061712acec6db994d6aa1c78a332a8 +test 18: + name: TestAnyRun + hash: a78dbafaca4813307529cafbed554b53a622a639941f2e66520bbb92769ee960 +test 19: + name: TestAssemblyLine + hash: 7cbf6cb53214f11904e63bb7493999a3b2e88b62 +test 20: + name: TestTriageV2 + hash: 5eaaf8ac2d358c2d7065884b7994638fee3987f02474e54467f14b010a18d028 \ No newline at end of file diff --git a/mlweb.go b/mlweb.go new file mode 100644 index 0000000..293d554 --- /dev/null +++ b/mlweb.go @@ -0,0 +1,48 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "strings" + "text/template" +) + +type Page struct { + hashes []string + tags []string + comments []string +} + +func indexHandler(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" { + values := strings.Split(r.PostFormValue("hashes"), "\r\n") + tags := strings.Split(r.PostFormValue("tags"), "\r\n") + comments := strings.Split(r.PostFormValue("comments"), "\r\n") + go processMalwareDownloadRequest(values, tags, comments) + } + t, _ := template.ParseFiles("./web/templates/index.html") + t.Execute(w, nil) +} + +func processMalwareDownloadRequest(values []string, tags []string, comments []string) { + hashes := parseArgHashes(values, tags, comments) + downloadMalwareFromWebServer(hashes) +} + +func runWebServer(bind string, port int) { + + http.HandleFunc("/styles/style.css", func(response http.ResponseWriter, request *http.Request) { + http.ServeFile(response, request, "./web/styles/style.css") + }) + + http.HandleFunc("/scripts/script.js", func(response http.ResponseWriter, request *http.Request) { + http.ServeFile(response, request, "./web/scripts/script.js") + }) + + http.HandleFunc("/", indexHandler) + + //http.HandleFunc("/download", postDataHandler) + + log.Fatal(http.ListenAndServe(fmt.Sprint(bind, ":", port), nil)) +} diff --git a/web/scripts/script.js b/web/scripts/script.js new file mode 100644 index 0000000..1a3f4cc --- /dev/null +++ b/web/scripts/script.js @@ -0,0 +1,29 @@ +var table = $('#hashes').DataTable( { + serverSide: true, + ajax: '/data-source' +} ); + +// Attach a submit handler to the form +$( "#download" ).submit(function( event ) { + + // Stop form from submitting normally + event.preventDefault(); + + // Get some values from elements on the page: + var $form = $( this ), + term = $form.find( "input[name='hashes']" ).val(), + url = $form.attr( "action" ); + + // Send the data using post + var posting = $.post( url, { hashes: term } ); + + // Put the results in a div + posting.done(function( data ) { + table.ajax.reload( null, false ); // user paging is not reset on reload + }); +}); + + +setInterval( function () { + table.ajax.reload( null, false ); // user paging is not reset on reload +}, 30000 ); \ No newline at end of file diff --git a/web/styles/style.css b/web/styles/style.css new file mode 100644 index 0000000..e69de29 diff --git a/web/templates/index.html b/web/templates/index.html new file mode 100644 index 0000000..f46ef9d --- /dev/null +++ b/web/templates/index.html @@ -0,0 +1,61 @@ + + + + + + + + + + + + + + +
+
+
+

MLGET - Download Malware

+
+
+ +
+
+ +
+
+
+
+
+
+
+ +
+
+ + + + + + + + + + +
HashHash TypeFound On
+
+
\ No newline at end of file