diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index d45a667..a250c0c 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -16,7 +16,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v2
with:
- go-version: 1.19
+ go-version: 1.21.3
- name: Build
run: go get -u && go build -v ./...
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6397fcd..0c12486 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,7 +1,5 @@
# .github/workflows/release.yaml
-# .github/workflows/release.yaml
-
on:
release:
types: [created, draft]
diff --git a/config.go b/config.go
index ad39b4a..2b20f8f 100644
--- a/config.go
+++ b/config.go
@@ -4,7 +4,6 @@ import (
"errors"
"fmt"
"io"
- "io/ioutil"
"log"
"os"
"path"
@@ -67,6 +66,7 @@ type MalwareRepoType int64
const (
NotSupported MalwareRepoType = iota //NotSupported must always be first, or other things won't work as expected
+ AssemblyLine
AnyRun
CapeSandbox
FileScanIo
@@ -161,6 +161,9 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
case URLScanIO:
found, filename = urlscanio(mcr.Host, mcr.Api, hash)
checkedRepo = URLScanIO
+ case AssemblyLine:
+ found, filename = assemblyline(mcr.Host, mcr.User, mcr.Api, mcr.IgnoreTLSErrors, hash)
+ checkedRepo = AssemblyLine
//case AnyRun:
// found, filename = anyrun(mcr.Host, hash)
// checkedRepo = AnyRun
@@ -169,7 +172,7 @@ func (malrepo MalwareRepoType) QueryAndDownload(repos []RepositoryConfigEntry, h
checkedRepo = UploadMWDB
}
// So some repos we can't download from but we want to know that it exists at that service
- // At the moment, this is just Any.Run but suspecct more will be added as time goes on
+ // At the moment, this is just Any.Run but suspect more will be added as time goes on
if checkedRepo == AnyRun && found {
continue
}
@@ -196,6 +199,10 @@ func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool
if repo.Host != "" {
return true
}
+ case AssemblyLine:
+ if repo.Host != "" && repo.Api != "" && repo.User != "" {
+ return true
+ }
default:
if repo.Host != "" && repo.Api != "" {
return true
@@ -207,6 +214,7 @@ func (malrepo MalwareRepoType) VerifyRepoParams(repo RepositoryConfigEntry) bool
func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
var host string
var api string
+ var user string
var default_url string
@@ -214,7 +222,7 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
case NotSupported:
return RepositoryConfigEntry{}, fmt.Errorf("malware repository rype, %s, is not supported", malrepo.String())
case MalwareBazaar:
- default_url = "https://mb-api.abuse.ch/api/v1"
+ default_url = "https://mb-api.abuse.ch/api/v1/"
case Malshare:
default_url = "https://malshare.com"
case MWDB:
@@ -259,12 +267,17 @@ func (malrepo MalwareRepoType) CreateEntry() (RepositoryConfigEntry, error) {
fmt.Println("Using the default url")
host = default_url
}
+ if malrepo == AssemblyLine {
+ fmt.Println("Enter User Name:")
+ fmt.Print(">> ")
+ fmt.Scanln(&user)
+ }
if malrepo != MalwareBazaar && malrepo != ObjectiveSee && malrepo != AnyRun {
fmt.Println("Enter API Key:")
fmt.Print(">> ")
fmt.Scanln(&api)
}
- return RepositoryConfigEntry{Host: host, Api: api, Type: malrepo.String()}, nil
+ return RepositoryConfigEntry{Host: host, User: user, Api: api, Type: malrepo.String()}, nil
}
func (malrepo MalwareRepoType) String() string {
@@ -303,6 +316,8 @@ func (malrepo MalwareRepoType) String() string {
return "URLScanIO"
case AnyRun:
return "AnyRun"
+ case AssemblyLine:
+ return "AssemblyLine"
case UploadMWDB:
return "UploadMWDB"
@@ -356,7 +371,8 @@ func queryAndDownloadAll(repos []RepositoryConfigEntry, hash Hash, doNotExtract
} else {
valid, calculatedHash := hash.ValidateFile(filename)
if !valid {
- fmt.Printf(" [!] Downloaded file hash %s does not match searched for hash %s\nTrying another source.\n", calculatedHash, hash.Hash)
+ fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\nTrying another source.\n", calculatedHash, hash.Hash)
+ deleteInvalidFile(filename)
continue
} else {
fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", hash.Hash)
@@ -408,6 +424,8 @@ func getMalwareRepoByFlagName(name string) MalwareRepoType {
return URLScanIO
case strings.ToLower("ar"):
return AnyRun
+ case strings.ToLower("al"):
+ return AssemblyLine
}
return NotSupported
}
@@ -448,6 +466,8 @@ func getMalwareRepoByName(name string) MalwareRepoType {
return URLScanIO
case strings.ToLower("AnyRun"):
return AnyRun
+ case strings.ToLower("AssemblyLine"):
+ return AssemblyLine
case strings.ToLower("UploadMWDB"):
return UploadMWDB
}
@@ -465,11 +485,13 @@ func getConfigsByType(repoType MalwareRepoType, repos []RepositoryConfigEntry) [
}
type RepositoryConfigEntry struct {
- Type string `yaml:"type"`
- Host string `yaml:"url"`
- Api string `yaml:"api"`
- QueryOrder int `yaml:"queryorder"`
- Password string `yaml:"pwd"`
+ Type string `yaml:"type"`
+ Host string `yaml:"url"`
+ Api string `yaml:"api"`
+ QueryOrder int `yaml:"queryorder"`
+ Password string `yaml:"pwd"`
+ User string `yaml:"user"`
+ IgnoreTLSErrors bool `yaml:"ignoretlserrors"`
}
func LoadConfig(filename string) ([]RepositoryConfigEntry, error) {
@@ -526,7 +548,7 @@ func parseFile(path string) (map[string]RepositoryConfigEntry, error) {
return nil, err
}
- f, err := ioutil.ReadFile(path)
+ f, err := os.ReadFile(path)
if err != nil {
fmt.Printf("%v", err)
return nil, err
diff --git a/download.go b/download.go
index 0049e2b..48fa177 100644
--- a/download.go
+++ b/download.go
@@ -3,17 +3,18 @@ package main
import (
"bytes"
"compress/gzip"
+ "crypto/tls"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
- "io/ioutil"
"log"
"net/http"
"net/url"
"os"
"regexp"
+ "strings"
"github.com/yeka/zip"
)
@@ -55,6 +56,33 @@ type MalwareBazarQueryData struct {
File_name string `json:"file_name"`
}
+type AssemblyLineQuery struct {
+ Error_message string `json:"api_error_message"`
+ Response *AssemblyLineQueryResponse `json:"api_response"`
+ Server_version string `json:"api_server_version"`
+ Status_Code int `json:"api_status_code"`
+}
+
+type AssemblyLineQueryResponse struct {
+ AL *AssemblyLineQueryALResponse `json:"al"`
+}
+
+type AssemblyLineQueryALResponse struct {
+ Error string `json:"error"`
+ Items []AssemblyLineQueryItem `json:"items"`
+}
+
+type AssemblyLineQueryItem struct {
+ Classification string `json:"classification"`
+ Data *AssemblyLineQueryData `json:"data"`
+}
+
+type AssemblyLineQueryData struct {
+ Md5 string `json:"md5"`
+ Sha1 string `json:"sha1"`
+ Sha256 string `json:"sha256"`
+}
+
type TriageQuery struct {
Data []TriageQueryData `json:"data"`
}
@@ -97,7 +125,7 @@ func loadObjectiveSeeJson(uri string) (ObjectiveSeeQuery, error) {
defer response.Body.Close()
if response.StatusCode == http.StatusOK {
- byteValue, _ := ioutil.ReadAll(response.Body)
+ byteValue, _ := io.ReadAll(response.Body)
var data = ObjectiveSeeQuery{}
error = json.Unmarshal(byteValue, &data)
@@ -211,7 +239,7 @@ func joesandbox(uri string, api string, hash Hash) (bool, string) {
if response.StatusCode == http.StatusOK {
- byteValue, error := ioutil.ReadAll(response.Body)
+ byteValue, error := io.ReadAll(response.Body)
if error != nil {
fmt.Println(error)
return false, ""
@@ -364,7 +392,7 @@ func inquestlabs(uri string, api string, hash Hash) (bool, string) {
if response.StatusCode == http.StatusOK {
- byteValue, _ := ioutil.ReadAll(response.Body)
+ byteValue, _ := io.ReadAll(response.Body)
var data = InquestLabsQuery{}
error = json.Unmarshal(byteValue, &data)
@@ -614,7 +642,7 @@ func hybridAnlysis(uri string, api string, hash Hash, doNotExtract bool) (bool,
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
return false, ""
}
- byteValue, _ := ioutil.ReadAll(response.Body)
+ byteValue, _ := io.ReadAll(response.Body)
var data = HybridAnalysisQuery{}
error = json.Unmarshal(byteValue, &data)
@@ -720,7 +748,7 @@ func triage(uri string, api string, hash Hash) (bool, string) {
if response.StatusCode == http.StatusOK {
- byteValue, error := ioutil.ReadAll(response.Body)
+ byteValue, error := io.ReadAll(response.Body)
if error != nil {
fmt.Println(error)
return false, ""
@@ -774,8 +802,54 @@ func traigeDownload(uri string, api string, sampleId string, hash Hash) (bool, s
fmt.Println(error)
return false, ""
}
- fmt.Printf(" [+] Downloaded %s\n", hash.Hash)
- return true, hash.Hash
+ // Triage will download an archive file that contians the hash in question sometimes versus the actual sample being requested
+ hashMatch, _ := hash.ValidateFile(hash.Hash)
+ if !hashMatch {
+ files, err := extractPwdZip(hash.Hash, "", false, hash)
+ if err != nil {
+ fmt.Println(error)
+ return false, ""
+ }
+
+ found := false
+
+ fmt.Printf(" [-] The downloaded file appears to be a zip file in which the requested file should be located.\n")
+ for _, f := range files {
+ fmt.Printf(" [-] Checking file: %s\n", f.Name)
+ hashMatch, _ = hash.ValidateFile(f.Name)
+ if !hashMatch {
+ err = os.Remove(f.Name)
+ if err != nil {
+ fmt.Println(" [!] Error when deleting file: ", f.Name)
+ fmt.Println(err)
+ }
+ } else {
+ fmt.Printf(" [+] %s is hash %s\n", f.Name, hash.Hash)
+ err = os.Rename(f.Name, hash.Hash)
+ if err != nil {
+ fmt.Println(" [!] Error when renaming file: ", f.Name)
+ fmt.Println(err)
+ } else {
+ found = true
+ }
+ }
+ }
+ if !found {
+ fmt.Printf(" [!] Hash %s not found\n", hash.Hash)
+ err = os.Remove(hash.Hash)
+ if err != nil {
+ fmt.Println(" [!] Error when deleting file: ", hash.Hash)
+ fmt.Println(err)
+ }
+ return false, ""
+ } else {
+ fmt.Printf(" [+] Found %s\n", hash.Hash)
+ return true, hash.Hash
+ }
+ } else {
+ fmt.Printf(" [+] Downloaded %s\n", hash.Hash)
+ return true, hash.Hash
+ }
}
func malshare(url string, api string, hash Hash) (bool, string) {
@@ -845,7 +919,7 @@ func malwareBazaar(uri string, hash Hash, doNotExtract bool, password string) (b
return false, ""
}
- byteValue, _ := ioutil.ReadAll(response.Body)
+ byteValue, _ := io.ReadAll(response.Body)
var data = MalwareBazarQuery{}
error = json.Unmarshal(byteValue, &data)
@@ -872,28 +946,39 @@ func malwareBazaar(uri string, hash Hash, doNotExtract bool, password string) (b
func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password string) (bool, string) {
query := "query=get_file&sha256_hash=" + hash.Hash
- values, error := url.ParseQuery(query)
- if error != nil {
- fmt.Println(error)
+ values, err := url.ParseQuery(query)
+ if err != nil {
+ fmt.Println(err)
return false, ""
}
client := &http.Client{}
- response, error := client.PostForm(uri, values)
- if error != nil {
- fmt.Println(error)
+
+ response, err := client.PostForm(uri, values)
+ if err != nil {
+ fmt.Println(err)
return false, ""
}
defer response.Body.Close()
if response.Header["Content-Type"][0] == "application/json" {
+ if response.StatusCode == http.StatusMethodNotAllowed {
+ if !strings.HasSuffix(uri, "/") {
+ fmt.Printf(" [!] Trying again with a trailing slash: %s/\n", uri)
+ return malwareBazaarDownload(uri+"/", hash, doNotExtract, password)
+ } else {
+ fmt.Printf(" [!] Normally the response code: %s means that the provided URL %s needs a trailing slash (to avoid the redirect), but this already has a trailing slash.\nPlease file a bug report at https://github.com/xorhex/mlget/issues\n", response.Status, uri)
+ }
+ } else {
+ fmt.Printf(" [!] %s\n", response.Status)
+ }
return false, ""
}
- error = writeToFile(response.Body, hash.Hash+".zip")
- if error != nil {
- fmt.Println(error)
+ err = writeToFile(response.Body, hash.Hash+".zip")
+ if err != nil {
+ fmt.Println(err)
return false, ""
}
@@ -902,7 +987,7 @@ func malwareBazaarDownload(uri string, hash Hash, doNotExtract bool, password st
return true, hash.Hash + ".zip"
} else {
fmt.Println(" [-] Extracting...")
- files, err := extractPwdZip(hash.Hash, password)
+ files, err := extractPwdZip(hash.Hash+".zip", password, true, hash)
if err != nil {
fmt.Println(err)
return false, ""
@@ -956,6 +1041,8 @@ func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool, pa
return false, ""
} else if response.StatusCode == http.StatusForbidden {
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
+ fmt.Printf(" [!] If you are sure this is correct, then test downloading a sample you've access to in their platform. It should work.\n")
+ fmt.Printf(" [!] Not sure why it does not just return a 404 instead; when the creds are correct but the file is not available.\n")
return false, ""
}
@@ -970,7 +1057,7 @@ func filescaniodownload(uri string, api string, hash Hash, doNotExtract bool, pa
return true, hash.Hash + ".zip"
} else {
fmt.Println(" [-] Extracting...")
- files, err := extractPwdZip(hash.Hash, password)
+ files, err := extractPwdZip(hash.Hash+".zip", password, true, hash)
if err != nil {
fmt.Println(err)
return false, ""
@@ -1030,7 +1117,7 @@ func vxsharedownload(uri string, api string, hash Hash, doNotExtract bool, passw
return true, hash.Hash + ".zip"
} else {
fmt.Println(" [-] Extracting...")
- files, err := extractPwdZip(hash.Hash, password)
+ files, err := extractPwdZip(hash.Hash+".zip", password, true, hash)
if err != nil {
fmt.Println(err)
return false, ""
@@ -1081,7 +1168,7 @@ func unpacmeDownload(uri string, api string, hash Hash) (bool, string) {
} else if response.StatusCode == http.StatusForbidden {
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
return false, ""
- } else if response.StatusCode == 401 {
+ } else if response.StatusCode == http.StatusUnauthorized {
fmt.Printf(" [!] Not authorized. Check the URL and APIKey in the config.\n")
return false, ""
}
@@ -1225,7 +1312,7 @@ func malpediaDownload(uri string, api string, hash Hash) (bool, string) {
return false, ""
}
- byteValue, _ := ioutil.ReadAll(response.Body)
+ byteValue, _ := io.ReadAll(response.Body)
jsonParseSuccesful, mpData := parseMalpediaJson(byteValue)
if jsonParseSuccesful {
for _, item := range mpData {
@@ -1272,6 +1359,114 @@ func parseMalpediaJson(byteValue []byte) (bool, []MalpediaData) {
return true, malpediaJsonItems
}
+func assemblyline(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) {
+ if api == "" {
+ fmt.Println(" [!] !! Missing Key !!")
+ return false, ""
+ }
+ if user == "" {
+ fmt.Println(" [!] !! Missing User !!")
+ return false, ""
+ }
+
+ if hash.HashType != sha256 {
+ fmt.Printf(" [-] Looking up sha256 hash for %s\n", hash.Hash)
+
+ request, error := http.NewRequest("GET", uri+"/hash_search/"+url.PathEscape(hash.Hash)+"/", nil)
+ if error != nil {
+ fmt.Println(error)
+ return false, ""
+ }
+
+ request.Header.Set("Content-Type", "application/json; charset=UTF-8")
+ request.Header.Set("x-user", user)
+ request.Header.Set("x-apikey", api)
+
+ tr := &http.Transport{}
+ if ignoretlserrors {
+ fmt.Printf(" [!] Ignoring Certificate Errors.\n")
+ tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+ }
+
+ client := &http.Client{Transport: tr}
+ response, error := client.Do(request)
+ if error != nil {
+ fmt.Println(error)
+ return false, ""
+ }
+ defer response.Body.Close()
+
+ if response.StatusCode == http.StatusForbidden || response.StatusCode == http.StatusUnauthorized {
+ fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n")
+ return false, ""
+ }
+
+ byteValue, _ := io.ReadAll(response.Body)
+
+ var data = AssemblyLineQuery{}
+ error = json.Unmarshal(byteValue, &data)
+
+ if error != nil {
+ fmt.Println(error)
+ return false, ""
+ }
+
+ if data.Response.AL == nil {
+ return false, ""
+ }
+ hash.Hash = data.Response.AL.Items[0].Data.Sha256
+ hash.HashType = sha256
+ fmt.Printf(" [-] Using hash %s\n", hash.Hash)
+ }
+
+ return assemblylineDownload(uri, user, api, ignoretlserrors, hash)
+
+}
+
+func assemblylineDownload(uri string, user string, api string, ignoretlserrors bool, hash Hash) (bool, string) {
+ request, error := http.NewRequest("GET", uri+"/file/download/"+url.PathEscape(hash.Hash)+"/?encoding=raw", nil)
+ if error != nil {
+ fmt.Println(error)
+ return false, ""
+ }
+
+ request.Header.Set("Content-Type", "application/json; charset=UTF-8")
+ request.Header.Set("x-user", user)
+ request.Header.Set("x-apikey", api)
+
+ tr := &http.Transport{}
+ if ignoretlserrors {
+ tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
+ }
+
+ client := &http.Client{Transport: tr}
+ response, error := client.Do(request)
+ if error != nil {
+ fmt.Println(error)
+ return false, ""
+ }
+
+ defer response.Body.Close()
+
+ if response.StatusCode == http.StatusNotFound {
+ return false, ""
+ } else if response.StatusCode == http.StatusForbidden {
+ fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n")
+ return false, ""
+ } else if response.StatusCode == http.StatusUnauthorized {
+ fmt.Printf(" [!] Not authorized. Check the URL, User, and APIKey in the config.\n")
+ return false, ""
+ }
+
+ error = writeToFile(response.Body, hash.Hash)
+ if error != nil {
+ fmt.Println(error)
+ return false, ""
+ }
+ fmt.Printf(" [+] Downloaded %s\n", hash.Hash)
+ return true, hash.Hash
+}
+
func extractGzip(hash string) error {
r, err := os.Open(hash + ".gzip")
if err != nil {
@@ -1288,9 +1483,9 @@ func extractGzip(hash string) error {
return err
}
-func extractPwdZip(hash string, password string) ([]*zip.File, error) {
+func extractPwdZip(file string, password string, renameFileAsHash bool, hash Hash) ([]*zip.File, error) {
- r, err := zip.OpenReader(hash + ".zip")
+ r, err := zip.OpenReader(file)
if err != nil {
log.Fatal(err)
}
@@ -1308,7 +1503,15 @@ func extractPwdZip(hash string, password string) ([]*zip.File, error) {
log.Fatal(err)
}
- out, error := os.Create(hash)
+ var name string
+
+ if !renameFileAsHash {
+ name = f.Name
+ } else {
+ name = hash.Hash
+ }
+
+ out, error := os.Create(name)
if error != nil {
return nil, error
}
diff --git a/go.mod b/go.mod
index 42caa1a..1f02b97 100644
--- a/go.mod
+++ b/go.mod
@@ -1,6 +1,6 @@
module xorhex.com/mlget
-go 1.19
+go 1.21.3
require (
github.com/kr/pretty v0.1.0
diff --git a/hashes.go b/hashes.go
index 0a32f3f..8feedaa 100644
--- a/hashes.go
+++ b/hashes.go
@@ -11,6 +11,8 @@ import (
"strings"
)
+var alwaysDeleteInvalidFile = false
+
type Hashes struct {
Hashes []Hash
}
@@ -146,6 +148,26 @@ func (h Hash) Validate(bytes []byte) (bool, string) {
}
}
+func deleteInvalidFile(filename string) {
+ if !alwaysDeleteInvalidFile {
+ var delete_file string
+ fmt.Printf(" [?] Delete invalid file? [A/Y/n] Always delete/Yes, this time/No, not this time\n")
+ fmt.Scanln(&delete_file)
+ if strings.ToUpper(delete_file) == "Y" || delete_file == "" || strings.ToUpper(delete_file) == "A" {
+ os.Remove(filename)
+ fmt.Printf(" [!] Deleted invalid file\n")
+ if strings.ToUpper(delete_file) == "A" {
+ alwaysDeleteInvalidFile = true
+ }
+ } else {
+ fmt.Printf(" [!] Keeping invalid file\n")
+ }
+ } else {
+ os.Remove(filename)
+ fmt.Printf(" [!] Deleted invalid file\n")
+ }
+}
+
func hashType(hash string) (HashTypeOption, error) {
match, _ := regexp.MatchString("^[A-Fa-f0-9]{64}$", hash)
if match {
@@ -161,3 +183,29 @@ func hashType(hash string) (HashTypeOption, error) {
}
return NotAValidHashType, errors.New("not a valid hash")
}
+
+func extractHashes(text string) ([]string, error) {
+ hashes := make([]string, 0)
+
+ re := regexp.MustCompile(`>\s*[A-Fa-f0-9]{64}\s*<`)
+ matches := re.FindAllStringSubmatch(text, 100)
+ for m := range matches {
+ hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1]))
+ }
+ re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{40}\s*<`)
+ matches = re.FindAllStringSubmatch(text, 100)
+ for m := range matches {
+ hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1]))
+ }
+ re = regexp.MustCompile(`>\s*[A-Fa-f0-9]{32}\s*<`)
+ matches = re.FindAllStringSubmatch(text, 100)
+ for m := range matches {
+ hashes = append(hashes, strings.TrimSpace(matches[m][0][1:len(matches[m][0])-1]))
+ }
+
+ if len(hashes) > 0 {
+ return hashes, fmt.Errorf("no hashes found")
+ }
+
+ return hashes, nil
+}
diff --git a/history.go b/history.go
index 9faa38f..9171a22 100644
--- a/history.go
+++ b/history.go
@@ -138,6 +138,40 @@ func parseFileForHashEntries(filename string) ([]Hash, error) {
pHash, err = parseFileHashEntry(hash, tags, comments)
if err == nil {
hashes = append(hashes, pHash)
+ } else {
+ // Try splitting on \t and check to see if any of the values match a hash
+ // This is useful for reading files from the web that list sample hashes
+ // This still assumes there is only one hash per line as it stops after the
+ // first hash is found on that line
+ s := func(c rune) bool {
+ return c == '\t'
+ }
+
+ line := strings.FieldsFunc(strings.TrimSpace(text), s)
+ if len(line) > 0 {
+ for _, element := range line {
+ lHash := Hash{}
+ lHash, err := parseFileHashEntry(strings.TrimSpace(element), tags, comments)
+ if err == nil {
+ hashes = append(hashes, lHash)
+ break
+ } else {
+
+ matches, err := extractHashes(strings.TrimSpace(element))
+ if err != nil {
+ fmt.Println(err)
+ }
+ for m := range matches {
+ tags := []string{}
+ comments := []string{}
+ lHash, err = parseFileHashEntry(matches[m], tags, comments)
+ if err == nil {
+ hashes = append(hashes, lHash)
+ }
+ }
+ }
+ }
+ }
}
}
}
@@ -171,7 +205,7 @@ func parseFileHashEntry(hash string, tags []string, comments []string) (Hash, er
fmt.Printf("\n Skipping %s because it's %s\n", hash, err)
return Hash{}, err
}
- fmt.Printf("Hash found: %s\n", hash) // token in unicode-char
+ fmt.Printf("\nHash found: %s\n", hash) // token in unicode-char
hashS := Hash{Hash: hash, HashType: ht}
if len(tags) > 0 {
hashS.Tags = tags
diff --git a/mlget-test-config/samples.yaml b/mlget-test-config/samples.yaml
new file mode 100644
index 0000000..bd01dd9
--- /dev/null
+++ b/mlget-test-config/samples.yaml
@@ -0,0 +1,60 @@
+test 1:
+ name: TestJoeSandbox
+ hash: e21ff9323365bca4131d8ec0a24b75521857776569316ffe3f7c97f327256d1b
+test 2:
+ name: TestObjectiveSee
+ hash: 458a9ac086116fa011c1a7bd49ac15f386cd95e39eb6b7cd5c5125aef516c78c
+test 3:
+ name: TestCapeSandbox
+ hash: 28eefc36104bebb595fb38cae21a7d0a
+test 4:
+ name: TestInquestLabsLookUp
+ hash: b3f868fa1af24f270e3ecc0ecb79325e
+test 5:
+ name: TestInquestLabsNoLookUp
+ hash: 6b425804d43bb369211bbec59808807730a908804ca9b8c09081139179bbc868
+test 6:
+ name: TestVirusTotal
+ hash: 21cc9c0ae5f97b66d69f1ff99a4fed264551edfe0a5ce8d5449942bf8f0aefb2
+test 7:
+ name: TestMWDB
+ hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18
+test 8:
+ name: TestPolyswarm
+ hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18
+test 9:
+ name: TestHybridAnalysis
+ hash: ed2f501408a7a6e1a854c29c4b0bc5648a6aa8612432df829008931b3e34bf56
+test 10:
+ name: TestTriage
+ hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18
+test 11:
+ name: TestMalShare
+ hash: 75b2831d387a27b3ecfda6be6ff0523de50ec86e6ac3e7a2ce302690570b7d18
+test 12:
+ name: TestMalwareBazaar
+ hash: 001bffcdd170c8328601006ad54a221d1073ba04fbdca556749cf1b041cfad97
+test 13:
+ name: TestMalpedia
+ hash: 78668c237097651d64c97b25fc86c74096bfe1ed53e1004445f118ea5feaa3ad
+test 14:
+ name: TestUnpacme
+ hash: 0219a79a2f47da42601568ee4a41392aa429f62a1fb01080cb68540074449c92
+test 15:
+ name: TestVxShare
+ hash: 1c11c963a417674e1414bac05fdbfa5cfa09f92c7b0d9882aeb55ce2a058d668
+test 16:
+ name: TestFileScanIo
+ hash: 2799af2efd698da215afc9c88da3b1e84b00137433d9444a5c11d69092b3f80d
+test 17:
+ name: TestURLScanIo
+ hash: 5b027ada26a610e97ab4ef9efb1118b377061712acec6db994d6aa1c78a332a8
+test 18:
+ name: TestAnyRun
+ hash: a78dbafaca4813307529cafbed554b53a622a639941f2e66520bbb92769ee960
+test 19:
+ name: TestAssemblyLine
+ hash: 7cbf6cb53214f11904e63bb7493999a3b2e88b62
+test 20:
+ name: TestTriageV2
+ hash: 5eaaf8ac2d358c2d7065884b7994638fee3987f02474e54467f14b010a18d028
\ No newline at end of file
diff --git a/mlget.go b/mlget.go
index 8a5c8f3..768ce5e 100644
--- a/mlget.go
+++ b/mlget.go
@@ -25,8 +25,11 @@ var tagsFlag []string
var commentsFlag []string
var versionFlag bool
var noValidationFlag bool
+var webserver bool
+var ip string
+var port int
-var version string = "3.0.1"
+var version string = "3.2.0"
func usage() {
fmt.Println("mlget - A command line tool to download malware from a variety of sources")
@@ -42,7 +45,7 @@ func usage() {
}
func init() {
- flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - cp (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tg (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.")
+ flag.StringVar(&apiFlag, "from", "", "The service to download the malware from.\n Must be one of:\n - al (AssemblyLine)\n - cs (Cape Sandbox)\n - fs (FileScanIo)\n - ha (Hybird Anlysis)\n - iq (Inquest Labs)\n - js (Joe Sandbox)\n - mp (Malpedia)\n - ms (Malshare)\n - mb (Malware Bazaar)\n - mw (Malware Database)\n - os (Objective-See)\n - ps (PolySwarm)\n - tr (Triage)\n - um (UnpacMe)\n - us (URLScanIO)\n - vt (VirusTotal)\n - vx (VxShare)\nIf omitted, all services will be tried.")
flag.StringVar(&inputFileFlag, "read", "", "Read in a file of hashes (one per line)")
flag.BoolVar(&outputFileFlag, "output", false, "Write to a file the hashes not found (for later use with the --read flag)")
flag.BoolVar(&helpFlag, "help", false, "Print the help message")
@@ -57,6 +60,9 @@ func init() {
flag.BoolVar(&downloadOnlyFlag, "downloadonly", false, "Download from any source, including your personal instance of MWDB.\nWhen this flag is set; it will NOT update any output file with the hashes not found.\nAnd it will not upload to any of the UploadMWDB instances.")
flag.BoolVar(&versionFlag, "version", false, "Print the version number")
flag.BoolVar(&noValidationFlag, "novalidation", false, "Turn off post download hash check verification")
+ //flag.BoolVar(&webserver, "webserver", false, "Run the webserver to handle request via the web browser.")
+ //flag.StringVar(&ip, "bind", "127.0.0.1", "Bind to IP. Default is localhost")
+ //flag.IntVar(&port, "port", 8080, "Bind to port. Default is 8080")
}
func main() {
@@ -64,11 +70,6 @@ func main() {
noSamplesRepoList := []MalwareRepoType{AnyRun}
doNotValidatehash := []MalwareRepoType{ObjectiveSee}
- if versionFlag {
- fmt.Printf("Mlget version: %s\n", version)
- return
- }
-
homeDir, err := os.UserHomeDir()
if err != nil {
fmt.Println(err)
@@ -85,6 +86,11 @@ func main() {
flag.Parse()
+ if versionFlag {
+ fmt.Printf("Mlget version: %s\n", version)
+ return
+ }
+
if helpFlag {
usage()
return
@@ -102,12 +108,51 @@ func main() {
args := flag.Args()
+ if webserver {
+ runWebServer(ip, port)
+ } else {
+ downloadMalwareFromCLI(args, cfg, noSamplesRepoList, doNotValidatehash)
+ }
+
+}
+
+func parseArgHashes(hashes []string, tags []string, comments []string) Hashes {
+ parsedHashes := Hashes{}
+ for _, h := range hashes {
+ ht, err := hashType(h)
+ if err != nil {
+ fmt.Printf("\n Skipping %s because it's %s\n", h, err)
+ continue
+ }
+ fmt.Printf("Hash found: %s\n", h) // token in unicode-char
+ hash := Hash{Hash: h, HashType: ht}
+ if len(tags) > 0 {
+ hash.Tags = tags
+ }
+ if len(comments) > 0 {
+ hash.Comments = comments
+ }
+ parsedHashes, _ = addHash(parsedHashes, hash)
+ }
+ return parsedHashes
+}
+
+func downloadMalwareFromCLI(args []string, cfg []RepositoryConfigEntry, noSamplesRepoList []MalwareRepoType, doNotValidatehash []MalwareRepoType) {
+ if apiFlag != "" {
+ flaggedRepo := getMalwareRepoByFlagName(apiFlag)
+ if flaggedRepo == NotSupported {
+ fmt.Printf("Invalid or unsupported malware repo type: %s\nCheck the help for the values to pass to the --from parameter\n", apiFlag)
+ return
+ }
+ }
+
if apiFlag != "" && downloadOnlyFlag {
fmt.Printf(("Can't use both the --from flag and the --downloadonly flag together"))
return
}
var osq ObjectiveSeeQuery
+ var err error
osConfigs := getConfigsByType(ObjectiveSee, cfg)
// Can have multiple Objective-See configs but only the first one to load will be used
for _, osc := range osConfigs {
@@ -174,10 +219,6 @@ func main() {
if apiFlag != "" {
flaggedRepo := getMalwareRepoByFlagName(apiFlag)
- if flaggedRepo == NotSupported {
- fmt.Printf("Invalid or unsupported malware repo type: %s\nCheck the help for the values to pass to the --from parameter\n", apiFlag)
- continue
- }
fmt.Printf("Looking on %s\n", getMalwareRepoByFlagName(apiFlag))
@@ -196,7 +237,9 @@ func main() {
} else {
valid, calculatedHash := h.ValidateFile(filename)
if !valid {
- fmt.Printf(" [!] Downloaded file hash %s does not match searched for hash %s\n", calculatedHash, h.Hash)
+ fmt.Printf(" [!] Downloaded file hash %s\n does not match searched for hash %s\n", calculatedHash, h.Hash)
+ deleteInvalidFile(filename)
+ notFoundHashes, _ = addHash(notFoundHashes, h)
continue
} else {
fmt.Printf(" [+] Downloaded file %s validated as the requested hash\n", h.Hash)
@@ -263,23 +306,6 @@ func main() {
}
}
-func parseArgHashes(hashes []string, tags []string, comments []string) Hashes {
- parsedHashes := Hashes{}
- for _, h := range hashes {
- ht, err := hashType(h)
- if err != nil {
- fmt.Printf("\n Skipping %s because it's %s\n", h, err)
- continue
- }
- fmt.Printf("Hash found: %s\n", h) // token in unicode-char
- hash := Hash{Hash: h, HashType: ht}
- if len(tags) > 0 {
- hash.Tags = tags
- }
- if len(comments) > 0 {
- hash.Comments = comments
- }
- parsedHashes, _ = addHash(parsedHashes, hash)
- }
- return parsedHashes
+func downloadMalwareFromWebServer(hashes Hashes) {
+
}
diff --git a/mlget_test.go b/mlget_test.go
index dd3c72c..34bb554 100644
--- a/mlget_test.go
+++ b/mlget_test.go
@@ -3,7 +3,6 @@ package main
import (
"errors"
"fmt"
- "io/ioutil"
"log"
"os"
"path"
@@ -25,7 +24,7 @@ func parseTestConfig(path string, testName string) (TestConfigEntry, error) {
return tce, err
}
- f, err := ioutil.ReadFile(path)
+ f, err := os.ReadFile(path)
if err != nil {
fmt.Printf("%v", err)
return tce, err
@@ -440,7 +439,7 @@ func TestMalwareBazaar(t *testing.T) {
result, filename, _ := MalwareBazaar.QueryAndDownload(cfg, hash, false, osq)
if !result {
- t.Errorf("Malshare failed")
+ t.Errorf("MalwareBazaar failed")
} else {
valid, errmsg := hash.ValidateFile(filename)
@@ -625,6 +624,41 @@ func TestURLScanIo(t *testing.T) {
}
+func TestAssemblyLine(t *testing.T) {
+ home, _ := os.UserHomeDir()
+ cfg, err := LoadConfig(path.Join(home, ".mlget.yml"))
+ if err != nil {
+ log.Fatal()
+ t.Errorf("%v", err)
+ }
+
+ scfg, err := parseTestConfig("./mlget-test-config/samples.yaml", t.Name())
+ if err != nil {
+ log.Fatal()
+ t.Errorf("%v", err)
+ }
+
+ ht, _ := hashType(scfg.Hash)
+ hash := Hash{HashType: ht, Hash: scfg.Hash}
+
+ var osq ObjectiveSeeQuery
+ result, filename, _ := AssemblyLine.QueryAndDownload(cfg, hash, false, osq)
+
+ if !result {
+ t.Errorf("Assemblyline failed")
+ } else {
+ valid, errmsg := hash.ValidateFile(filename)
+
+ if !valid {
+ os.Remove(filename)
+ t.Errorf(errmsg)
+ } else {
+ os.Remove(filename)
+ }
+ }
+
+}
+
/*
func TestAnyRun(t *testing.T) {
home, _ := os.UserHomeDir()
diff --git a/mlweb.go b/mlweb.go
new file mode 100644
index 0000000..293d554
--- /dev/null
+++ b/mlweb.go
@@ -0,0 +1,48 @@
+package main
+
+import (
+ "fmt"
+ "log"
+ "net/http"
+ "strings"
+ "text/template"
+)
+
+type Page struct {
+ hashes []string
+ tags []string
+ comments []string
+}
+
+func indexHandler(w http.ResponseWriter, r *http.Request) {
+ if r.Method == "POST" {
+ values := strings.Split(r.PostFormValue("hashes"), "\r\n")
+ tags := strings.Split(r.PostFormValue("tags"), "\r\n")
+ comments := strings.Split(r.PostFormValue("comments"), "\r\n")
+ go processMalwareDownloadRequest(values, tags, comments)
+ }
+ t, _ := template.ParseFiles("./web/templates/index.html")
+ t.Execute(w, nil)
+}
+
+func processMalwareDownloadRequest(values []string, tags []string, comments []string) {
+ hashes := parseArgHashes(values, tags, comments)
+ downloadMalwareFromWebServer(hashes)
+}
+
+func runWebServer(bind string, port int) {
+
+ http.HandleFunc("/styles/style.css", func(response http.ResponseWriter, request *http.Request) {
+ http.ServeFile(response, request, "./web/styles/style.css")
+ })
+
+ http.HandleFunc("/scripts/script.js", func(response http.ResponseWriter, request *http.Request) {
+ http.ServeFile(response, request, "./web/scripts/script.js")
+ })
+
+ http.HandleFunc("/", indexHandler)
+
+ //http.HandleFunc("/download", postDataHandler)
+
+ log.Fatal(http.ListenAndServe(fmt.Sprint(bind, ":", port), nil))
+}
diff --git a/web/scripts/script.js b/web/scripts/script.js
new file mode 100644
index 0000000..1a3f4cc
--- /dev/null
+++ b/web/scripts/script.js
@@ -0,0 +1,29 @@
+var table = $('#hashes').DataTable( {
+ serverSide: true,
+ ajax: '/data-source'
+} );
+
+// Attach a submit handler to the form
+$( "#download" ).submit(function( event ) {
+
+ // Stop form from submitting normally
+ event.preventDefault();
+
+ // Get some values from elements on the page:
+ var $form = $( this ),
+ term = $form.find( "input[name='hashes']" ).val(),
+ url = $form.attr( "action" );
+
+ // Send the data using post
+ var posting = $.post( url, { hashes: term } );
+
+ // Put the results in a div
+ posting.done(function( data ) {
+ table.ajax.reload( null, false ); // user paging is not reset on reload
+ });
+});
+
+
+setInterval( function () {
+ table.ajax.reload( null, false ); // user paging is not reset on reload
+}, 30000 );
\ No newline at end of file
diff --git a/web/styles/style.css b/web/styles/style.css
new file mode 100644
index 0000000..e69de29
diff --git a/web/templates/index.html b/web/templates/index.html
new file mode 100644
index 0000000..f46ef9d
--- /dev/null
+++ b/web/templates/index.html
@@ -0,0 +1,61 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ | Hash |
+ Hash Type |
+ Found On |
+
+
+
+
+
+
+