Build rules dat files #57
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build rules dat files | |
on: | |
workflow_dispatch: | |
schedule: | |
- cron: "30 22 * * 2,4,6" # 6:30 AM UTC+8 | |
push: | |
branches: | |
- master | |
paths-ignore: | |
- "**/README.md" | |
jobs: | |
build: | |
runs-on: ubuntu-latest | |
steps: | |
- name: Set variables | |
run: | | |
echo "GOAMD64=v3" >> $GITHUB_ENV | |
echo "BUILDTIME=$(TZ=Asia/Shanghai date +'%Y-%m-%d %H:%M')" >> $GITHUB_ENV | |
echo "CHINA_DOMAINS_URL=https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/ChinaMax/ChinaMax_Domain.txt" >> $GITHUB_ENV | |
echo "GOOGLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" >> $GITHUB_ENV | |
echo "APPLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" >> $GITHUB_ENV | |
# echo "CUSTOM_DIRECT=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/cn.txt" >> $GITHUB_ENV | |
echo "CUSTOM_PROXY=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/geolocation-!cn.txt" >> $GITHUB_ENV | |
echo "WIN_SPY=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt" >> $GITHUB_ENV | |
echo "WIN_UPDATE=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/update.txt" >> $GITHUB_ENV | |
echo "WIN_EXTRA=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt" >> $GITHUB_ENV | |
shell: bash | |
- name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
repository: wzfdgh/geosite | |
- name: Checkout Loyalsoldier/domain-list-custom | |
uses: actions/checkout@v4 | |
with: | |
repository: Loyalsoldier/domain-list-custom | |
path: custom | |
- name: Checkout v2fly/domain-list-community | |
uses: actions/checkout@v4 | |
with: | |
repository: v2fly/domain-list-community | |
path: community | |
- name: Checkout cokebar/gfwlist2dnsmasq | |
uses: actions/checkout@v4 | |
with: | |
repository: cokebar/gfwlist2dnsmasq | |
path: gfwlist2dnsmasq | |
- name: Setup Go | |
uses: actions/setup-go@v5 | |
with: | |
go-version: '1.22' | |
cache-dependency-path: ./custom/go.sum | |
- name: Generate GFWList domains | |
run: | | |
cd gfwlist2dnsmasq || exit 1 | |
chmod +x ./gfwlist2dnsmasq.sh | |
./gfwlist2dnsmasq.sh -l -o ./temp-gfwlist.txt | |
- name: Get and add direct domains into temp-direct.txt file | |
run: | | |
curl -sSL ${CHINA_DOMAINS_URL} | sed '/^\s*#/d' | sed '/^[^\.]/ s/^/full:/' | sed 's/^\.\([^.]*\)/\1/' > temp-direct.txt | |
# curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-direct.txt | |
- name: Get and add proxy domains into temp-proxy.txt file | |
run: | | |
cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt | |
curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-proxy.txt | |
- name: Reserve `full`, `regexp` and `keyword` type of rules from custom lists to "reserve" files | |
run: | | |
# curl -sSL ${CUSTOM_DIRECT} | grep -v google | grep -v manhua | grep -v ooklaserver | grep -v "acg.rip" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > direct-reserve.txt | |
curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > proxy-reserve.txt | |
- name: Add proxy, direct and reject domains from "hidden" branch to appropriate temp files | |
run: | | |
cat ./resouces/proxy.txt >> temp-proxy.txt | |
cat ./resouces/direct.txt >> temp-direct.txt | |
# cat reject.txt >> temp-reject.txt | |
- name: Sort and generate redundant lists | |
run: | | |
cat temp-proxy.txt | sort --ignore-case -u > proxy-list-with-redundant | |
cat temp-direct.txt | sort --ignore-case -u > direct-list-with-redundant | |
# cat temp-reject.txt | sort --ignore-case -u > reject-list-with-redundant | |
- name: Remove redundant domains | |
run: | | |
chmod +x ./resouces/*.py | |
python ./resouces/findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort | |
python ./resouces/findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort | |
[ ! -f "direct-list-deleted-unsort" ] && touch direct-list-deleted-unsort | |
[ ! -f "proxy-list-deleted-unsort" ] && touch proxy-list-deleted-unsort | |
sort ./direct-list-deleted-unsort > ./direct-list-deleted-sort | |
sort ./proxy-list-deleted-unsort > ./proxy-list-deleted-sort | |
python ./resouces/removeFrom.py -remove ./direct-list-deleted-sort -from ./direct-list-with-redundant -out direct-list-without-redundant | |
python ./resouces/removeFrom.py -remove ./proxy-list-deleted-sort -from ./proxy-list-with-redundant -out proxy-list-without-redundant | |
- name: Remove domains from "need-to-remove" lists in "hidden" branch | |
run: | | |
python ./resouces/removeFrom.py -remove ./resouces/direct-need-to-remove.txt -from ./direct-list-without-redundant -out ./temp-cn.txt | |
python ./resouces/removeFrom.py -remove ./resouces/proxy-need-to-remove.txt -from ./proxy-list-without-redundant -out ./temp-geolocation-\!cn.txt | |
- name: Remove domains end with ".cn" in "temp-geolocation-!cn.txt" and write lists to data directory | |
run: | | |
cat temp-cn.txt | grep -v google | grep -v manhua | grep -v ooklaserver | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/cn | |
cat temp-cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > direct-tld-list.txt | |
cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' | perl -ne 'print if not /\.cn$/' > ./community/data/geolocation-\!cn | |
cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > proxy-tld-list.txt | |
- name: Add `full`, `regexp` and `keyword` type of rules back into "cn", and "geolocation-!cn" list | |
run: | | |
[ -f "direct-reserve.txt" ] && cat direct-reserve.txt >> ./community/data/cn | |
[ -f "proxy-reserve.txt" ] && cat proxy-reserve.txt >> ./community/data/geolocation-\!cn | |
cp ./community/data/cn direct-list.txt | |
cp ./community/data/geolocation-\!cn proxy-list.txt | |
- name: Create `google-cn`、`apple-cn`、`gfw` lists | |
run: | | |
curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/google-cn | |
curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/apple-cn | |
cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> ./community/data/gfw | |
curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-spy | |
curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-update | |
curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-extra | |
- name: merge 'ios_rule_script/SteamCN' | |
env: | |
SED: sed '/^\s*#/d' | sed 's/DOMAIN,//g' | sed 's/DOMAIN-SUFFIX,//g' | sed 's/DOMAIN-KEYWORD,/keyword:/g' | |
run: | | |
curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/SteamCN/SteamCN.list | ${{ env.SED }} > steamcn.txt | |
while read line; do grep -q "$line @cn" ./community/data/steam || sed -i "/$line/ s/$/ @cn/" ./community/data/steam; done < steamcn.txt | |
cat ./community/data/steam | |
- name: Add data for `meta` | |
env: | |
SED: sed '/^\s*#/d' | sed 's/^PROCESS-NAME,//' | sed 's/DOMAIN,/full:/g' | sed 's/DOMAIN-SUFFIX,//g' | sed 's/DOMAIN-KEYWORD,/keyword:/g' | sed '/^\s*IP-CIDR/d' | |
run: | | |
curl -sSL https://raw.githubusercontent.com/xishang0128/rules/main/biliintl.list > ./community/data/biliintl | |
curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/OneDrive/OneDrive.list | ${{ env.SED }} > ./community/data/onedrive | |
echo "sharepoint.cn" >> ./community/data/onedrive | |
curl -sSL https://raw.githubusercontent.com/xishang0128/rules/main/sharepoint.list > ./community/data/sharepoint | |
# curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/PrivateTracker/PrivateTracker.list | ${{ env.SED }} > ./community/data/tracker | |
# curl -sSL https://gitea.com/XIU2/TrackersListCollection/raw/branch/master/all.txt | grep -i "\.[A-Z]" | grep -v tracker | sed 's/^.*\/\///g' | sed 's/:.*\/.*//g' >> ./community/data/tracker | |
# curl -sSL https://github.com/blackmatrix7/ios_rule_script/raw/master/rule/Clash/BlockHttpDNS/BlockHttpDNS.list | ${{ env.SED }} > ./community/data/httpdns | |
# echo sentry.io >> ../community/data/openai | |
echo ipleak.net >> ./community/data/geolocation-\!cn && echo browserleaks.org >> ./community/data/geolocation-\!cn | |
echo xn--ngstr-cn-8za9o.com >> ./community/data/google | |
# - name: Build db and metadb file | |
# env: | |
# NO_SKIP: true | |
# run: | | |
# go install -trimpath -ldflags="-s -w -buildid=" github.com/metacubex/geo/cmd/geo@master | |
# geo convert site -i v2ray -o sing -f geosite.db ./custom/publish/geosite.dat | |
# geo convert site -i v2ray -o sing -f geosite-lite.db ./community/geosite-lite.dat | |
# geo convert ip -i v2ray -o sing -f geoip.db ./geoip.dat | |
# geo convert ip -i v2ray -o meta -f geoip.metadb ./geoip.dat | |
# geo convert ip -i v2ray -o sing -f geoip-lite.db ./geoip-lite.dat | |
# geo convert ip -i v2ray -o meta -f geoip-lite.metadb ./geoip-lite.dat | |
# - name: Convert geo to sing-rule-set | |
# env: | |
# NO_SKIP: true | |
# run: | | |
# mkdir -p ./sing-rule/geo | |
# cp ./geoip.dat ./geosite.db ./resouces/convert.sh ./sing-rule/geo/ | |
# cd ./sing-rule/geo | |
# wget https://github.com/SagerNet/sing-box/releases/download/v1.8.0-alpha.1/sing-box-1.8.0-alpha.1-linux-amd64.tar.gz -O sing-box.tar.gz && tar zxvf sing-box.tar.gz && mv sing-box-1.8.0-alpha.1-linux-amd64/sing-box ./ && rm -r sing-box-1.8.0-alpha.1-linux-amd64 | |
# wget https://github.com/IrineSistiana/mosdns/releases/download/v4.5.3/mosdns-linux-amd64.zip -O mosdns.zip && unzip mosdns.zip | |
# chmod 755 mosdns sing-box convert.sh && ./convert.sh | |
# rm mosdns* sing-box* convert.sh geoip.dat geosite.db LICENSE README.md config.yaml | |
- name: Build temp geosite.dat | |
run: | | |
cd custom || exit 1 | |
go run ./ --datapath=../community/data | |
- name: Convert geosite to metadata | |
env: | |
NO_SKIP: true | |
run: | | |
mkdir -p ./community/tmp | |
cp ./custom/publish/geosite.dat ./community/tmp | |
cd ./community/tmp | |
wget https://github.com/IrineSistiana/mosdns/releases/download/v4.5.3/mosdns-linux-amd64.zip -O mosdns.zip && unzip mosdns.zip | |
chmod 755 mosdns | |
mkdir -p geosite | |
./mosdns v2dat unpack-domain -o ./geosite/ geosite.dat | |
list=($(ls ./geosite | sed 's/geosite_//g' | sed 's/\.txt//g')) | |
for ((i=0;i<${#list[@]};i++)); do | |
mv ./geosite/geosite_${list[i]}.txt ./geosite/${list[i]}.list | |
done | |
rm mosdns* *.dat LICENSE README.md config.yaml | |
- name: Build geosite.dat file | |
env: | |
NO_SKIP: true | |
run: | | |
cd community || exit 1 | |
mkdir -p geo-data | |
code=(apple-cn apple bilibili biliintl cn ehentai geolocation-\!cn google-cn google line microsoft onedrive pixiv private sharepoint steam telegram twitter win-extra win-spy win-update youtube) | |
for((i=0;i<${#code[@]};i++));do | |
mkdir -p geosite | |
cp ./tmp/geosite/${code[i]}.list ./geo-data/${code[i]} | |
done | |
go run ./ --datapath=./geo-data --outputname geosite.dat | |
# - name: Convert bm7-rule-set to sing-rule-set | |
# env: | |
# NO_SKIP: true | |
# run: | | |
# mkdir -p ./sing-rule/bm7 | |
# cp ./resouces/bm7.sh ./sing-rule/bm7/ | |
# cd ./sing-rule/bm7 | |
# wget https://github.com/SagerNet/sing-box/releases/download/v1.8.0-alpha.1/sing-box-1.8.0-alpha.1-linux-amd64.tar.gz -O sing-box.tar.gz && tar zxvf sing-box.tar.gz && mv sing-box-1.8.0-alpha.1-linux-amd64/sing-box ./ && rm -r sing-box-1.8.0-alpha.1-linux-amd64 | |
# ls | |
# chmod 755 bm7.sh && ./bm7.sh | |
# rm -r bm7.sh sing-box* rule | |
- name: Move and zip files | |
run: | | |
mkdir -p ./publish/ | |
install -Dp ./community/geosite.dat ./publish/ | |
- name: Delete current release assets | |
uses: andreaswilli/[email protected] | |
with: | |
github_token: ${{ secrets.GITHUB_TOKEN }} | |
tag: latest | |
deleteOnlyFromDrafts: false | |
- name: Git push assets to "release" branch | |
run: | | |
cd publish || exit 1 | |
git init | |
git config --local user.name "wzfdgh" | |
git config --local user.email "[email protected]" | |
git checkout -b release | |
git add . | |
git commit -m release | |
git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" | |
git push -f -u origin release | |
- name: Purge jsdelivr CDN | |
run: | | |
cd publish || exit 1 | |
for file in $(ls); do | |
curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}" | |
done | |
- name: Upload files to GitHub release | |
id: upload_release | |
uses: svenstaro/upload-release-action@v2 | |
with: | |
repo_token: ${{ secrets.GITHUB_TOKEN }} | |
release_name: latest | |
tag: latest | |
file_glob: true | |
overwrite: true | |
file: ./publish/* | |
body: ${{ env.BUILDTIME }} | |
- name: Cleanup Workflow | |
uses: Mattraks/delete-workflow-runs@main | |
with: | |
retain_days: 0 | |
keep_minimum_runs: 1 |