Build rules dat files #285
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build rules dat files | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "30 22 * * *" # 6:30 AM UTC+8 | |
| push: | |
| branches: | |
| - master | |
| paths-ignore: | |
| - "**/README.md" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set variables | |
| run: | | |
| echo "GOAMD64=v3" >> $GITHUB_ENV | |
| echo "BUILDTIME=$(TZ=Asia/Shanghai date +'%Y-%m-%d %H:%M')" >> $GITHUB_ENV | |
| echo "CHINA_DOMAINS_URL=https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/ChinaMax/ChinaMax_Domain.txt" >> $GITHUB_ENV | |
| # echo "GOOGLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" >> $GITHUB_ENV | |
| # echo "APPLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" >> $GITHUB_ENV | |
| # echo "CUSTOM_DIRECT=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/cn.txt" >> $GITHUB_ENV | |
| # echo "CUSTOM_PROXY=https://raw.githubusercontent.com/Loyalsoldier/domain-list-custom/release/geolocation-!cn.txt" >> $GITHUB_ENV | |
| echo "WIN_SPY=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/spy.txt" >> $GITHUB_ENV | |
| echo "WIN_UPDATE=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/update.txt" >> $GITHUB_ENV | |
| echo "WIN_EXTRA=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt" >> $GITHUB_ENV | |
| shell: bash | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Checkout Loyalsoldier/domain-list-custom | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Loyalsoldier/domain-list-custom | |
| path: custom | |
| - name: Checkout v2fly/domain-list-community | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: v2fly/domain-list-community | |
| path: community | |
| - name: Checkout cokebar/gfwlist2dnsmasq | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: cokebar/gfwlist2dnsmasq | |
| path: gfwlist2dnsmasq | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| cache-dependency-path: ./custom/go.sum | |
| # - name: Generate GFWList domains | |
| # run: | | |
| # cd gfwlist2dnsmasq || exit 1 | |
| # chmod +x ./gfwlist2dnsmasq.sh | |
| # ./gfwlist2dnsmasq.sh -l -o ./temp-gfwlist.txt | |
| - name: Get and add direct domains into temp-direct.txt file | |
| run: | | |
| # curl -sSL ${CHINA_DOMAINS_URL} | sed '/^\s*#/d' | sed '/^[^\.]/ s/^/full:/' | sed 's/^\.\([^.]*\)/\1/' > temp-direct.txt | |
| # curl -sSL ${CUSTOM_DIRECT} | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-direct.txt | |
| cat ./community/data/cn >> temp-direct.txt | |
| - name: Get and add proxy domains into temp-proxy.txt file | |
| run: | | |
| # cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' > temp-proxy.txt | |
| # curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
| # curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "$1\n"' >> temp-proxy.txt | |
| # curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^(domain):([^:]+)(\n$|:@.+)/ && print "$2\n"' >> temp-proxy.txt | |
| cp ./community/data/geolocation-\!cn temp-proxy.txt | |
| # - name: Reserve `full`, `regexp` and `keyword` type of rules from custom lists to "reserve" files | |
| # run: | | |
| # curl -sSL ${CUSTOM_DIRECT} | grep -v google | grep -v manhua | grep -v ooklaserver | grep -v "acg.rip" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > direct-reserve.txt | |
| # curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > proxy-reserve.txt | |
| - name: Add proxy, direct and reject domains from "hidden" branch to appropriate temp files | |
| run: | | |
| cat ./resouces/proxy.txt >> temp-proxy.txt | |
| cat ./resouces/direct.txt >> temp-direct.txt | |
| # cat reject.txt >> temp-reject.txt | |
| - name: Sort and generate redundant lists | |
| run: | | |
| cat temp-proxy.txt | sort --ignore-case -u > proxy-list-with-redundant | |
| cat temp-direct.txt | sort --ignore-case -u > direct-list-with-redundant | |
| # cat temp-reject.txt | sort --ignore-case -u > reject-list-with-redundant | |
| - name: Remove redundant domains | |
| run: | | |
| chmod +x ./resouces/*.py | |
| python ./resouces/findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort | |
| python ./resouces/findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort | |
| [ ! -f "direct-list-deleted-unsort" ] && touch direct-list-deleted-unsort | |
| [ ! -f "proxy-list-deleted-unsort" ] && touch proxy-list-deleted-unsort | |
| sort ./direct-list-deleted-unsort > ./direct-list-deleted-sort | |
| sort ./proxy-list-deleted-unsort > ./proxy-list-deleted-sort | |
| python ./resouces/removeFrom.py -remove ./direct-list-deleted-sort -from ./direct-list-with-redundant -out direct-list-without-redundant | |
| python ./resouces/removeFrom.py -remove ./proxy-list-deleted-sort -from ./proxy-list-with-redundant -out proxy-list-without-redundant | |
| - name: Remove domains from "need-to-remove" lists in "resouces" folder | |
| run: | | |
| python ./resouces/removeFrom.py -remove ./resouces/direct-need-to-remove.txt -from ./direct-list-without-redundant -out ./temp-cn.txt | |
| python ./resouces/removeFrom.py -remove ./resouces/proxy-need-to-remove.txt -from ./proxy-list-without-redundant -out ./temp-geolocation-\!cn.txt | |
| - name: Write lists to data directory | |
| run: | | |
| cat temp-cn.txt | sort --ignore-case -u > ./community/data/cn | |
| # cat temp-cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > direct-tld-list.txt | |
| cat temp-geolocation-\!cn.txt | sort --ignore-case -u > ./community/data/geolocation-\!cn | |
| # cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > proxy-tld-list.txt | |
| # - name: Add `full`, `regexp` and `keyword` type of rules back into "cn", and "geolocation-!cn" list | |
| # run: | | |
| # [ -f "direct-reserve.txt" ] && cat direct-reserve.txt >> ./community/data/cn | |
| # [ -f "proxy-reserve.txt" ] && cat proxy-reserve.txt >> ./community/data/geolocation-\!cn | |
| # cp ./community/data/cn direct-list.txt | |
| # cp ./community/data/geolocation-\!cn proxy-list.txt | |
| - name: Create `win-spy` `win-update` `win-extra` lists | |
| run: | | |
| # curl -sSL $GOOGLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/google-cn | |
| # curl -sSL $APPLE_DOMAINS_URL | perl -ne '/^server=\/([^\/]+)\// && print "full:$1\n"' > ./community/data/apple-cn | |
| # cat ./gfwlist2dnsmasq/temp-gfwlist.txt | perl -ne '/^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/ && print "$1\n"' >> ./community/data/gfw | |
| curl -sSL $WIN_SPY | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-spy | |
| curl -sSL $WIN_UPDATE | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-update | |
| curl -sSL $WIN_EXTRA | grep "0.0.0.0" | awk '{print $2}' > ./community/data/win-extra | |
| - name: merge 'ios_rule_script/SteamCN' | |
| env: | |
| SED: sed '/^\s*#/d' | sed 's/DOMAIN,//g' | sed 's/DOMAIN-SUFFIX,//g' | sed 's/DOMAIN-KEYWORD,/keyword:/g' | |
| run: | | |
| curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/SteamCN/SteamCN.list | ${{ env.SED }} > steamcn.txt | |
| while read line; do grep -q "$line @cn" ./community/data/steam || sed -i "/$line/ s/$/ @cn/" ./community/data/steam; done < steamcn.txt | |
| cat ./community/data/steam | |
| - name: Add data for `meta` | |
| env: | |
| SED: sed '/^\s*#/d' | sed 's/^PROCESS-NAME,//' | sed 's/DOMAIN,/full:/g' | sed 's/DOMAIN-SUFFIX,//g' | sed 's/DOMAIN-KEYWORD,/keyword:/g' | sed '/^\s*IP-CIDR/d' | |
| run: | | |
| echo "auvpn.net" >> ./community/data/category-vpnservices | |
| echo "ausososo.click @cn" >> ./community/data/category-vpnservices | |
| echo "ausososo.club @cn" >> ./community/data/category-vpnservices | |
| echo "ausososo.online @cn" >> ./community/data/category-vpnservices | |
| echo "ausososo.xyz @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.autos @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.beauty @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.hair @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.lol @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.mom @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.pics @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.quest @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.shop @cn" >> ./community/data/category-vpnservices | |
| echo "ausu.skin @cn" >> ./community/data/category-vpnservices | |
| echo "ausvc.link @cn" >> ./community/data/category-vpnservices | |
| echo "rd.sbs @cn" >> ./community/data/category-vpnservices | |
| echo "full:api.inforun.work @cn" >> ./community/data/category-vpnservices | |
| echo "full:repo.t5.work @cn" >> ./community/data/category-vpnservices | |
| echo "splashtop.com @cn" >> ./community/data/category-vpnservices | |
| echo "include:geolocation-!cn@cn" >> ./community/data/cn | |
| echo "include:geolocation-cn@!cn" >> ./community/data/geolocation-\!cn | |
| curl -sSL https://raw.githubusercontent.com/wzfdgh/atri-rules/main/biliintl.list > ./community/data/biliintl | |
| curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/OneDrive/OneDrive.list | ${{ env.SED }} > ./community/data/onedrive | |
| echo "sharepoint.cn" >> ./community/data/onedrive | |
| curl -sSL https://raw.githubusercontent.com/xishang0128/rules/main/sharepoint.list > ./community/data/sharepoint | |
| curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/PrivateTracker/PrivateTracker.list |sed '/^\s*#/d' | sed 's/^PROCESS-NAME,//' | sed 's/DOMAIN,//g' | sed 's/DOMAIN-SUFFIX,//g' | sed 's/DOMAIN-KEYWORD,/keyword:/g' | sed '/^\s*IP-CIDR/d' > temp-tracker.txt | |
| curl -sSL https://raw.githubusercontent.com/XIU2/TrackersListCollection/master/all.txt | grep -i "\.[A-Z]" | sed 's/^.*\/\///g' | sed 's/:.*\/.*//g' >> temp-tracker.txt | |
| curl -sSL https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all.txt | grep -i "\.[A-Z]" | sed 's/^.*\/\///g' | sed 's/:.*\/.*//g' >> temp-tracker.txt | |
| cat temp-tracker.txt | sort | uniq > ./community/data/tracker | |
| # curl -sSL https://github.com/blackmatrix7/ios_rule_script/raw/master/rule/Clash/BlockHttpDNS/BlockHttpDNS.list | ${{ env.SED }} > ./community/data/httpdns | |
| echo ipleak.net >> ./community/data/geolocation-\!cn | |
| echo browserleaks.org >> ./community/data/geolocation-\!cn | |
| echo misskey.io >> ./community/data/geolocation-\!cn | |
| echo acgrip.com >> ./community/data/geolocation-\!cn | |
| echo xn--ngstr-cn-8za9o.com >> ./community/data/google | |
| - name: Build temp geosite.dat | |
| run: | | |
| cd custom || exit 1 | |
| go run ./ --datapath=../community/data | |
| - name: Convert geosite to metadata | |
| env: | |
| NO_SKIP: true | |
| run: | | |
| mkdir -p ./community/tmp | |
| cp ./custom/publish/geosite.dat ./community/tmp | |
| cd ./community/tmp | |
| wget https://github.com/MetaCubeX/geo/releases/download/v1.1/geo-linux-amd64 -O geo | |
| chmod +x geo | |
| mkdir geosite | |
| ./geo unpack site -d geosite geosite.dat | |
| rm geo | |
| - name: Build geosite.dat file | |
| env: | |
| NO_SKIP: true | |
| run: | | |
| cd community || exit 1 | |
| mkdir -p geo-data | |
| code=(apple bilibili biliintl cn ehentai geolocation-\!cn googlefcm google line microsoft onedrive pixiv private sharepoint steam telegram twitter win-update youtube) | |
| for((i=0;i<${#code[@]};i++));do | |
| cp ./tmp/geosite/${code[i]} ./geo-data/${code[i]} | |
| sed -i 's/regex:/regexp:/g; s/plain:/keyword:/g' ./geo-data/${code[i]} | |
| done | |
| go run ./ --datapath=./geo-data --outputname geosite.dat | |
| - name: Move and zip files | |
| run: | | |
| mkdir -p ./publish/ | |
| install -Dp ./community/geosite.dat ./publish/ | |
| - name: Delete current release assets | |
| uses: andreaswilli/[email protected] | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| tag: latest | |
| deleteOnlyFromDrafts: false | |
| - name: Git push assets to "release" branch | |
| run: | | |
| cd publish || exit 1 | |
| git init | |
| git config --local user.name "wzfdgh" | |
| git config --local user.email "[email protected]" | |
| git checkout -b release | |
| git add . | |
| git commit -m release | |
| git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" | |
| git push -f -u origin release | |
| - name: Upload files to GitHub release | |
| id: upload_release | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| release_name: latest | |
| tag: latest | |
| file_glob: true | |
| overwrite: true | |
| file: ./publish/* | |
| body: ${{ env.BUILDTIME }} | |
| - name: Purge jsdelivr CDN | |
| run: | | |
| cd publish || exit 1 | |
| for file in $(ls); do | |
| curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}" | |
| done |