Merge pull request #101 from PIGfaces/master

fix(filter): url fragment deduplication. 谢谢 @PIGfaces

.gitignore

@@ -0,0 +1,8 @@
+.vscode
+.idea
+bin
+upload
+result
+.DS_Store
+internal/.DS_Store
+pkg/.DS_Store

Makefile

@@ -0,0 +1,15 @@
+VERSION=$(shell git describe --tags --always)
+
+.PHONY: build_all
+# build
+build_all:
+	rm -rf bin && mkdir bin bin/linux-amd64 bin/linux-arm64 bin/darwin-amd64 bin/darwin-arm64 \
+	&& CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags "-X 'main.Version=$(VERSION)'" -o ./bin/darwin-arm64/ ./... \
+	&& CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "-X 'main.Version=$(VERSION)'" -o ./bin/darwin-amd64/ ./... \
+	&& CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "-X 'main.Version=$(VERSION)'" -o ./bin/linux-arm64/ ./... \
+	&& CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "-X 'main.Version=$(VERSION)'" -o ./bin/linux-amd64/ ./...
+
+.PHONY: build
+# build
+build:
+	rm -rf bin && mkdir bin && go build -ldflags "-X main.Version=$(VERSION)" -o ./bin/ ./...

README.md

@@ -28,9 +28,15 @@ crawlergo currently supports the following features:
 
 **Build**
 
+- compilation for current platform
+
+```shell
+make build
+```
+
+- compile for all platforms
 ```shell
-cd crawlergo/cmd/crawlergo
-go build crawlergo_cmd.go
+make build_all
 ```
 
 1. crawlergo relies only on the chrome environment to run, go to [download](https://www.chromium.org/getting-involved/download-chromium) for the new version of chromium.
@@ -45,14 +51,14 @@ go build crawlergo_cmd.go
 Assuming your chromium installation directory is `/tmp/chromium/`, set up 10 tabs open at the same time and crawl the `testphp.vulnweb.com`:
 
 ```shell
-./crawlergo -c /tmp/chromium/chrome -t 10 http://testphp.vulnweb.com/
+bin/crawlergo -c /tmp/chromium/chrome -t 10 http://testphp.vulnweb.com/
 ```
 
 
 ### Using Proxy
 
 ```shell
-./crawlergo -c /tmp/chromium/chrome -t 10 --request-proxy socks5://127.0.0.1:7891 http://testphp.vulnweb.com/
+bin/crawlergo -c /tmp/chromium/chrome -t 10 --request-proxy socks5://127.0.0.1:7891 http://testphp.vulnweb.com/
 ```
 
 
@@ -70,7 +76,7 @@ import subprocess
 
 def main():
     target = "http://testphp.vulnweb.com/"
-    cmd = ["./crawlergo", "-c", "/tmp/chromium/chrome", "-o", "json", target]
+    cmd = ["bin/crawlergo", "-c", "/tmp/chromium/chrome", "-o", "json", target]
     rsp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     output, error = rsp.communicate()
 	#  "--[Mission Complete]--"  is the end-of-task separator string

README_zh-cn.md

@@ -28,9 +28,15 @@ crawlergo 目前支持以下特性：
 
 **Build**
 
+- 编译适用于当前机器的文件
+
+```shell
+make build
+```
+
+- 交叉编译所有平台的文件
 ```shell
-cd crawlergo/cmd/crawlergo
-go build crawlergo_cmd.go
+make build_all
 ```
 
 1. crawlergo 只依赖chrome运行即可，前往[下载](https://www.chromium.org/getting-involved/download-chromium)新版本的chromium。
@@ -46,15 +52,15 @@ go build crawlergo_cmd.go
 假设你的chromium安装在 `/tmp/chromium/` ，开启最大10标签页，爬取AWVS靶场：
 
 ```shell
-./crawlergo -c /tmp/chromium/chrome -t 10 http://testphp.vulnweb.com/
+bin/crawlergo -c /tmp/chromium/chrome -t 10 http://testphp.vulnweb.com/
 ```
 
 
 
 ### 使用代理
 
 ```shell
-./crawlergo -c /tmp/chromium/chrome -t 10 --request-proxy socks5://127.0.0.1:7891 http://testphp.vulnweb.com/
+bin/crawlergo -c /tmp/chromium/chrome -t 10 --request-proxy socks5://127.0.0.1:7891 http://testphp.vulnweb.com/
 ```
 
 
@@ -73,7 +79,7 @@ import subprocess
 
 def main():
     target = "http://testphp.vulnweb.com/"
-    cmd = ["./crawlergo", "-c", "/tmp/chromium/chrome", "-o", "json", target]
+    cmd = ["bin/crawlergo", "-c", "/tmp/chromium/chrome", "-o", "json", target]
     rsp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     output, error = rsp.communicate()
 	#  "--[Mission Complete]--"  是任务结束的分隔字符串