isp_data_pollution.py

#!/usr/bin/env python3
#coding: utf-8
__author__ = 'stsmith'

# isp_data_pollution: bandwidth-limited ISP data pollution 

# Copyright 2017–2018 Steven T. Smith <steve dot t dot smith at gmail dot com>, GPL

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

__version__ = '2.0.1'

import argparse as ap, datetime as dt, importlib, numpy as np, numpy.random as npr, os, psutil, random, re, requests, signal, sys, tarfile, time, warnings as warn
import urllib.request, urllib.robotparser as robotparser, urllib.parse as uprs
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from io import BytesIO
import fake_useragent as fake_ua

# parse User-Agent for matching distribution
ua_parse_flag = True
try:
    # pip install user-agents
    import user_agents as ua
except ImportError:
    ua_parse_flag = False

# ensure pyopenssl exists to address SNI support
# https://stackoverflow.com/questions/18578439/using-requests-with-tls-doesnt-give-sni-support/18579484#18579484
if importlib.util.find_spec('OpenSSL') is None:
    msg = 'Use the pyopenssl package to enable SNI support for TLS-protected hosted domains.'
    print(msg)
    warn.warn(msg)

# headless Raspberry Pi
try:
    from pyvirtualdisplay import Display
    display = Display(visible=0, size=(1296,1018))
    display.start()
except ImportError:
    pass

# nice this process on UNIX
if hasattr(os,'nice'): os.nice(15)

gb_per_month = 100		# How many gigabytes to pollute per month
max_links_cached = 100000	# Maximum number of links to cache for download
max_links_per_page = 200	# Maximum number of links to add per page
max_links_per_domain = 400	# Maximum number of links to add per domain
wordsite_url = 'http://svnweb.freebsd.org/csrg/share/dict/words?view=co&content-type=text/plain'
timeout = 45
short_timeout = 10
browserdriver_rss_limit_mb = 1024  # Default maximum memory limit of browserdriver (chromedriver) processs (MB)
terminal_width = 80  # tty width, standard is 80 chars; add code to adapt later

blacklist_url = 'http://www.shallalist.de/Downloads/shallalist.tar.gz'
# Usage of the Shalla Blacklists:
# ===============================
#
# The Shalla Blacklists are property of Shalla Secure Services.
#
# This collection of url lists may be used for free for non
# commercial usage. This includes all kinds of private usage.
# The lists must not be given to any third party.

# property value distribution to match household
property_pvals = \
    {'DNT':  # Do Not Track HTTP header
        {True: 0.8, False: 0.2},
    'browser':
        {'Safari': 6, 'Firefox': 3, 'Chrome': 2, 'noneoftheabove': 1},
    'os':
        {r'Mac\s*OS': 3, r'iOS': 6, r'Linux': 1, r'Windows': 1, 'noneoftheabove': 1},
    'is_pc':
        {True: 4, False: 6},
    'is_pc':
        {True: 4, False: 6},
    'is_touch_capable':
        {True: 6, False: 4},
    }
# project to simplex
for tlf in property_pvals:
    tot = 0.
    for f in property_pvals[tlf]: tot += abs(property_pvals[tlf][f])
    for f in property_pvals[tlf]: property_pvals[tlf][f] = abs(property_pvals[tlf][f])/tot

# tell ISP that an iPad is being used
user_agent = 'Mozilla/5.0 (iPad; CPU OS 6_1 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10B141 Safari/8536.25'

# Tor browser size on Linux
window_size = (1296,1018)

# bias the content with non-random, diverse, link-heavy, popular content
seed_bias_links = ['http://my.xfinity.com/news',
                    'http://my.xfinity.com/entertainment',
                    'http://my.xfinity.com/shopping',
                    'http://www.cnbc.com/',
                    'https://news.google.com',
                    'https://news.yahoo.com',
                    'http://www.huffingtonpost.com',
                    'http://www.cnn.com',
                    'http://www.foxnews.com',
                    'http://www.nbcnews.com',
                    'http://www.usatoday.com',
                    'http://www.huffingtonpost.com',
                    'http://www.tmz.com',
                    'http://www.deadspin.com',
                    'http://www.dailycaller.com',
                    'http://www.sports.yahoo.com',
                    'http://www.espn.com',
                    'http://www.foxsports.com',
                    'http://www.finance.yahoo.com',
                    'http://www.money.msn.com',
                    'http://www.fool.com'
                    ]

# monkeypatch the read class method in RobotFileParser
# many sites will block access to robots.txt without a standard User-Agent header
class RobotFileParserUserAgent(robotparser.RobotFileParser):

    timeout = short_timeout  # short-term timeout

    def read(self):
        """Reads the robots.txt URL and feeds it to the parser."""
        try:
            headers = {'User-Agent': user_agent, }
            request = urllib.request.Request(self.url, None, headers)
            f = urllib.request.urlopen(request,timeout=self.timeout)
            # f = urllib.request.urlopen(self.url)   #! original code
        except urllib.error.HTTPError as err:
            if err.code in (401, 403):
                self.disallow_all = True
            elif err.code >= 400 and err.code < 500:
                self.allow_all = True
        else:
            raw = f.read()
            self.parse(raw.decode("utf-8").splitlines())

# Notes for the future:
# 1. The bandwidth usage is undoubtedly (much) smaller because gzip encoding is used
# 2. A lightweight proxy could be used for accurate bandwidth, and header editing

# Safe search options
class SafeWebSearch():
    """ Safe web search class with default Google parameters. 
Use unencrypted HTTP for ISP DPI.
"""
    def __init__(self,
            search_url='http://www.google.com/search',  # search engine
            query_parameter='q',                        # query parameter
            safe_parameter='safe=active',               # query parameter for safe searches
            css_selector='div.g',                       # css selector to harvest search results
            additional_parameters='',                   # additional parameters required to get results
            result_extraction=lambda x: x):             # function to extract the link
        self.search_url = search_url
        self.query_parameter = query_parameter
        self.safe_parameter = safe_parameter
        self.css_selector = css_selector
        self.additional_parameters = additional_parameters
        self.result_extraction = result_extraction

SafeGoogle = SafeWebSearch()
SafeBing = SafeWebSearch(search_url='http://www.bing.com/search',
                safe_parameter='adlt=strict',css_selector='li.b_algo')
yahoo_search_reprog = re.compile(r'/RU=(.+?)/R[A-Z]=')
SafeYahoo = SafeWebSearch(search_url='http://search.yahoo.com/search', query_parameter='p',
                safe_parameter='vm=r',css_selector='div.compTitle',
                result_extraction=lambda x: yahoo_search_reprog.findall(uprs.parse_qs(x)['_ylu'][0])[0])
SafeDuckDuckGo = SafeWebSearch(search_url='http://www.duckduckgo.com/',
                safe_parameter='kp=1',css_selector='div.result__body')

class ISPDataPollution:
    """
    Re: https://www.eff.org/deeplinks/2017/03/senate-puts-isp-profits-over-your-privacy
 
I pay my ISP a lot for data usage every month. I typically don't use
all the bandwidth that I pay for.  If my ISP is going to sell my
private browsing habits, then I'm going to pollute my browsing with
noise and use all the bandwidth that I pay for. This method
accomplishes this.

If everyone uses all the data they've paid for to pollute their
browsing history, then perhaps ISPs will reconsider the business model
of selling customer's private browsing history.

The alternative of using a VPN or Tor merely pushes the issue onto to
the choice of VPN provider, complicates networking, and adds the
real issue of navigating captchas when appearing as a Tor exit node.

The crawler uses the Python requests and lxml.html libraries, is hardcoded
to download html without javascript processing, will not download
images, and respects robots.txt, which all provide good security.
    """

    def __init__(self,gb_per_month=gb_per_month,
                 max_links_cached=max_links_cached,
                 max_links_per_page=max_links_per_page,
                 max_links_per_domain=max_links_per_domain,
                 property_pvals=property_pvals,
                 user_agent=user_agent,
                 blacklist_url=blacklist_url,
                 wordsite_url=wordsite_url,
                 seed_bias_links=seed_bias_links,
                 timeout=timeout, diurnal_flag=True,
                 quit_driver_every_call=False,
                 blacklist=True,verbose=True):
        print(f'This is ISP Data Pollution 🐙💨, Version {__version__}')
        self.max_links_cached = max_links_cached
        self.max_links_per_page = max_links_per_page
        self.max_links_per_domain = max_links_per_domain
        self.property_pvals = property_pvals
        self.user_agent = user_agent
        self.blacklist_url = blacklist_url
        self.wordsite_url = wordsite_url
        self.seed_bias_links = seed_bias_links
        self.blacklist = blacklist; self.verbose = verbose
        self.timeout = timeout
        self.diurnal_flag = diurnal_flag
        self.quit_driver_every_call = quit_driver_every_call
        # self.gb_per_month = gb_per_month  # set in parseArgs
        # self.debug = debug  # set in parseArgs
        self.args = self.args = self.parseArgs()
        # timeout configurable decorators
        self.chromedriver_timeout = self.block_timeout(self.chromedriver_hang_handler, \
            alarm_time=self.timeout+2,errors=(self.TimeoutError,), debug=self.debug)
        self.chromedriver_short_timeout = self.block_timeout(self.chromedriver_hang_handler, \
            alarm_time=short_timeout+2,errors=(self.TimeoutError,Exception), debug=self.debug)
        self.chromedriver_quit_timeout = self.block_timeout(self.chromedriver_quit_hang_handler, \
            alarm_time=short_timeout+2,errors=(self.TimeoutError,Exception), debug=self.debug)
        self.robots_timeout = self.block_timeout(self.robots_hang_handler, \
            alarm_time=short_timeout+2,errors=(self.TimeoutError,), debug=self.debug)
        self.check_chromedriver_version()
        self.get_useragents()
        self.hour_trigger = True
        self.twentyfour_hour_trigger = True
        self.domain_links = dict()
        self.start_time = time.time()
        self.data_usage = 0
        self.get_blacklist()
        self.get_random_words()
        self.pollute_forever()

    def parseArgs(self):
        parser = ap.ArgumentParser()
        parser.add_argument('-bw', '--gb_per_month', help="GB per month", type=int, default=gb_per_month)
        parser.add_argument('-mm', '--maxmemory',
            help="Maximum memory of chromedriver (MB); 0=>restart every link",
            type=int, default=1024)
        parser.add_argument('-P', '--chromedriver-binary-path', help="Path to chromedriver binary", type=str, default=None)
        parser.add_argument('-p', '--proxy', help="Proxy for chromedriver", type=str, default=None)
        parser.add_argument('-g', '--debug', help="Debug flag", action='store_true')
        args = parser.parse_args()
        for k in args.__dict__: setattr(self,k,getattr(args,k))
        self.sanity_check_arguments()
        return args

    def sanity_check_arguments(self):
        self.gb_per_month = min(2048,max(1,self.gb_per_month))  # min-max bandwidth limits
        if self.maxmemory == 0: self.quit_driver_every_call = True
        self.chromedriver_rss_limit_mb = min(4096,max(256,self.maxmemory))  # min-max bandwidth limits

    def check_chromedriver_version(self,recommended_version=(2,41)):
        self.open_driver()
        if self.debug:
            print("{} version is {}, chromedriver version is {}".format(self.driver.capabilities["browserName"],
                                                              self.driver.capabilities["version"],
                                                              self.driver.capabilities["chrome"]["chromedriverVersion"]))
        chromedriver_version = tuple(int(i) for i in
            re.sub(r'([\d.]+?) .*','\\1',self.driver.capabilities["chrome"]["chromedriverVersion"]).split('.'))
        if chromedriver_version < recommended_version:
            warn.warn("""{} version is {};
please upgrade to at least version {} from http://chromedriver.chromium.org/downloads.
""".format(self.driver.capabilities["browserName"],self.driver.capabilities["version"],
           '.'.join(str(i) for i in recommended_version)))
        self.quit_driver()

    def open_driver(self):
        self.quit_driver()
        if not hasattr(self, 'driver') or not isinstance(self.driver,webdriver.chrome.webdriver.WebDriver):
            # chromedriver
            chrome_options = webdriver.ChromeOptions()
            chrome_options.add_argument('headless')
            chrome_options.add_argument(f'user-agent={self.user_agent}')
            chrome_options.add_argument('window-size={:d},{:d}'.format(window_size[0],window_size[1]))
            # Disable image downloads; see https://stackoverflow.com/questions/18657976/disable-images-in-selenium-google-chromedriver
            chrome_options.add_argument('blink-settings=imagesEnabled=false')
            chrome_options.add_argument('mute-audio')
            if self.proxy is not None:
                chrome_options.add_argument(f'proxy-server={self.proxy}')
            if self.chromedriver_binary_path is None:
                driver = webdriver.Chrome(options=chrome_options)
            else:
                chrome_options.binary_location = self.chromedriver_binary_path
                driver = webdriver.Chrome(self.chromedriver_binary_path,chrome_options=chrome_options)
            driver.set_window_size(window_size[0],window_size[1])
            driver.implicitly_wait(self.timeout)
            driver.set_page_load_timeout(self.timeout)
            driver.set_script_timeout(self.timeout)
            self.driver = driver

    def quit_driver(self,hard_quit=False,pid=None,chromedriver_short_timeout_decorator=None):
        """
        close, kill -9, quit, del
        :param hard_quit: 
        :param pid: 
        :return: 
         """
        # Use original phantomjs code for chromedriver, even though chromedriver is likely far more robust
        # http://stackoverflow.com/questions/25110624/how-to-properly-stop-phantomjs-execution
        if chromedriver_short_timeout_decorator is None:
            chromedriver_short_timeout_decorator = self.chromedriver_short_timeout
        if hasattr(self,'driver'):
            if not hard_quit:
                @chromedriver_short_timeout_decorator
                def chromedriver_close(): self.driver.close()
                chromedriver_close()
            try:
                if pid is None:
                    @chromedriver_short_timeout_decorator
                    def chromedriver_process_pid(): return self.driver.service.process.pid
                    pid = chromedriver_process_pid()
                @chromedriver_short_timeout_decorator
                def chromedriver_send_signal():
                    # Google Chrome is a child process of chromedriver
                    for c in psutil.Process(pid).children(): c.send_signal(signal.SIGTERM)
                    self.driver.service.process.send_signal(signal.SIGTERM)
                chromedriver_send_signal()
            except Exception as e:
                if self.debug: print(f'.send_signal() exception:\n{e}')
                if isinstance(pid,int):
                    try:
                        # Google Chrome is a child process of chromedriver
                        for c in psutil.Process(pid).children(): os.kill(c.pid, signal.SIGTERM)
                        os.kill(pid, signal.SIGTERM)  # overkill (pun intended)
                    except Exception as e:
                        if self.debug: print(f'.kill() exception:\n{e}')
            try:
                @chromedriver_short_timeout_decorator
                def chromedriver_quit(): self.driver.quit()
                chromedriver_quit()
            except Exception as e:
                if self.debug: print(f'.quit() exception:\n{e}')
            del self.driver

    def clear_driver(self):
        # https://sqa.stackexchange.com/questions/10466/how-to-clear-localstorage-using-selenium-and-webdriver
        if hasattr(self, 'driver'):
            try:
                @self.chromedriver_short_timeout
                def chromedriver_delete_all_cookies(): self.driver.delete_all_cookies()
                chromedriver_delete_all_cookies()
            except Exception as e:
                if self.debug: print(f'.delete_all_cookies() exception:\n{e}')
            try:
                @self.chromedriver_short_timeout
                def chromedriver_clear():
                    self.driver.execute_script('window.localStorage.clear();')
                    self.driver.execute_script('window.sessionStorage.clear();')
                chromedriver_clear()
            except Exception as e:
                if self.debug: print(f'.execute_script() exception:\n{e}')

    def get_useragents(self):
        for attempt in range(5):
            try:
                self.fake_ua = fake_ua.UserAgent()
            except (fake_ua.errors.FakeUserAgentError,urllib.error.URLError) as e:
                if self.debug: print(f'.UserAgent exception #{attempt}:\n{e}')
            else:
                break
        else:
            print('Too many .UserAgent failures. Exiting.')
            sys.exit(1)

    def get_blacklist(self,update_flag=False):
        blacklist_domains = getattr(self,'blacklist_domains',set())
        blacklist_urls = getattr(self,'blacklist_urls',set())
        self.blacklist_domains = set()
        self.blacklist_urls = set()
        try:
            if self.blacklist:    # download the blacklist or not
                if self.verbose: print('Downloading the blacklists… ',end='',flush=True)
            else:
                raise Exception('Skip downloading the blacklist.')
            self.get_shalla_blacklist()
            if self.verbose: print('Shallalist done… ', end='', flush=True)
            self.get_easylist_blacklist()
            if self.verbose: print('EasyList done.', flush=True)
        except Exception as e:
            if self.verbose: print(e)
        # Make sure blacklists are not empty
        if self.blacklist:
            try: # no fully empty collection of blacklists
                assert (self.blacklist_domains != set() or self.blacklist_urls != set()) \
                    and (not update_flag or (blacklist_domains != set() or blacklist_urls != set()))
            except AssertionError as e:
                print(e)
                if update_flag:
                    self.blacklist_domains = blacklist_domains
                    self.blacklist_urls = blacklist_urls
                    warn.warn('Blacklists not updated; falling back on previous blacklist download.')
                else:
                    print('Empty blacklists! Exiting.')
                    sys.exit(1)
        # ignore problem urls
        self.blacklist_urls |= { 'about:blank' }

    def get_shalla_blacklist(self):
        # http://stackoverflow.com/questions/18623842/read-contents-tarfile-into-python-seeking-backwards-is-not-allowed
        tgzstream = urllib.request.urlopen(urllib.request.Request(self.blacklist_url, headers={'User-Agent': self.user_agent}))
        tmpfile = BytesIO()
        while True:
            s = tgzstream.read(16384)
            if not s: break
            tmpfile.write(s)
        tgzstream.close()
        tmpfile.seek(0)
        tgz = tarfile.open(fileobj=tmpfile, mode='r:gz')
        # bash$ ls BL
        # COPYRIGHT	education	isp		recreation	updatesites
        # adv		finance		jobsearch	redirector	urlshortener
        # aggressive	fortunetelling	library		religion	violence
        # alcohol		forum		military	remotecontrol	warez
        # anonvpn		gamble		models		ringtones	weapons
        # automobile	global_usage	movies		science		webmail
        # chat		government	music		searchengines	webphone
        # costtraps	hacking		news		sex		webradio
        # dating		hobby		podcasts	shopping	webtv
        # downloads	homestyle	politics	socialnet
        # drugs		hospitals	porn		spyware
        # dynamic		imagehosting	radiotv		tracker
        for member in [ 'downloads', 'drugs', 'hacking', 'gamble', 'porn', 'spyware', 'updatesites', 'urlshortener', 'violence', 'warez', 'weapons' ]:
            self.blacklist_domains |= set(tgz.extractfile(f'BL/{member}/domains').read().decode('utf-8').splitlines())
            self.blacklist_urls |= set(tgz.extractfile(f'BL/{member}/urls').read().decode('utf-8').splitlines())
        tgz.close()
        tmpfile.close()

    def get_easylist_blacklist(self):
        # Malware lists from open source AdBlock and spam404.com lists
        malwaredomains_full = 'https://easylist-downloads.adblockplus.org/malwaredomains_full.txt'
        spam404_com_adblock_list = 'https://raw.githubusercontent.com/Dawsey21/Lists/master/adblock-list.txt'
        spam404_com_main_blacklist = 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt'  # not EasyList format
        download_list = list(set([malwaredomains_full, spam404_com_adblock_list, spam404_com_main_blacklist]))
        download_parse = { malwaredomains_full: True, spam404_com_adblock_list: True, spam404_com_main_blacklist: False }

        for url in download_list:
            resp = urllib.request.urlopen(urllib.request.Request(url, headers={'User-Agent': self.user_agent}))
            for line in resp:
                line = line.decode('utf-8').rstrip()
                if download_parse[url]: self.parse_and_filter_rule_urls(line)
                else: self.blacklist_domains |= set([line])

    def get_random_words(self):
        try:
            reqsession = requests.Session()
            reqsession.headers.update({'User-Agent': self.user_agent})
            response = reqsession.get(self.wordsite_url,timeout=10)
            self.words = response.content.decode('utf-8').splitlines()
            reqsession.close()
        except Exception as e:
            if self.debug: print(f'requests exception:\n{e}')
            self.words = [ 'FUBAR' ]
        # if self.debug: print('There are {:d} words.'.format(len(self.words)))

    def pollute_forever(self):
        if self.verbose: print("""Display format:
Downloading: website.com; NNNNN links [in library], H(domain)= B bits [entropy]
Downloaded:  website.com: +LLL/NNNNN links [added], H(domain)= B bits [entropy]
""")
        self.open_driver()
        self.seed_links()
        self.clear_driver()
        if self.quit_driver_every_call: self.quit_driver()
        while True: # pollute forever, pausing only to meet the bandwidth requirement
            try:
                if (not self.diurnal_flag) or self.diurnal_cycle_test():
                    self.pollute()
                else:
                    time.sleep(self.chi2_mean_std(3.,1.))
                if npr.uniform() < 0.005: self.set_user_agent()  # reset the user agent occasionally
                self.elapsed_time = time.time() - self.start_time
                self.exceeded_bandwidth_tasks()
                self.random_interval_tasks()
                self.every_hour_tasks()
                time.sleep(self.chi2_mean_std(0.5,0.2))
            except Exception as e:
                if self.debug: print(f'.pollute() exception:\n{e}')

    def pollute(self):
        if not self.quit_driver_every_call: self.check_chromedriver_process()
        if self.link_count() < 2000:
            if self.quit_driver_every_call: self.open_driver()
            self.seed_links()
            self.clear_driver()
            if self.quit_driver_every_call: self.quit_driver()
        url = self.pop_link()
        if self.verbose: self.print_url(url)
        if self.quit_driver_every_call: self.open_driver()
        self.get_url(url)
        self.clear_driver()
        if self.quit_driver_every_call: self.quit_driver()

    def link_count(self):
        return int(np.array([len(self.domain_links[dmn]) for dmn in self.domain_links]).sum())

    def domain_entropy(self):
        result = 0.
        domain_count = np.array([(dmn, len(self.domain_links[dmn])) for dmn in self.domain_links])
        p = np.array([np.float(c) for d, c in domain_count])
        count_total = p.sum()
        if count_total > 0:
            p = p / p.sum()
            result = self.entropy(p)
        return result

    def entropy(self,p):
        return -np.fromiter((self.xlgx(x) for x in p.flatten()),dtype=p.dtype).sum()

    def xlgx(self,x):
        x = np.abs(x)
        y = 0.
        if not (x == 0. or x == 1.):
            y = x*np.log2(x)
        return y

    def seed_links(self):
        # bias with non-random seed links
        self.bias_links()
        if self.link_count() < self.max_links_cached:
            num_words = max(1,npr.poisson(1.33)+1)  # mean of 1.33 words per search
            if num_words == 1:
                word = ' '.join(random.sample(self.words,num_words))
            else:
                if npr.uniform() < 0.5:
                    word = ' '.join(random.sample(self.words,num_words))
                else:      # quote the first two words together
                    word = ' '.join(['"{}"'.format(' '.join(random.sample(self.words, 2))),
                                     ' '.join(random.sample(self.words, num_words-2))])
            if self.debug: print(f'Seeding with search for \'{word}\'…')
            self.get_websearch(word)

    def bias_links(self):
        for url in self.seed_bias_links: self.add_link(url)

    def diurnal_cycle_test(self):
        now = dt.datetime.now()
        tmhr = now.hour + now.minute/60.
        phase = npr.normal(14.,1.)
        exponent = min(0.667,self.chi2_mean_std(0.333,0.1))
        def cospow(x,e):  # flattened cosine with e < 1
            c = np.cos(x)
            return np.sign(c) * np.power(np.abs(c), e)
        diurn = max(0.,0.5*(1.+cospow((tmhr-phase)*(2.*np.pi/24.),exponent)))
        flr = min(0.1,self.chi2_mean_std(0.02,0.002))
        val = flr + (1.-flr)*diurn
        return npr.uniform() < val

    def chi2_mean_std(self,mean=1.,std=0.1):
        """
        Chi-squared random variable with given mean and standard deviation.
        :param mean: 
        :param std: 
        :return: 
        """
        scale = 2.*mean/std
        nu = mean*scale
        return npr.chisquare(nu)/scale

    def exceeded_bandwidth_tasks(self):
        if self.bandwidth_test():
            self.decimate_links(total_frac=0.81,decimate_frac=0.1)
            time.sleep(120)

    def random_interval_tasks(self,random_interval=None):
        if random_interval is None: random_interval = self.chi2_mean_std(2*3600.,3600.)
        def init_random_time():
            self.random_start_time = time.time()
            self.random_interval = self.random_start_time + random_interval
        if not hasattr(self,'random_interval'): init_random_time()
        if time.time() > self.random_interval:
            init_random_time()  # reinitialize random interval
            self.current_preferred_domain = self.draw_domain()

    def every_hour_tasks(self):
        if int(self.elapsed_time/60. % 60.) == 59:
            # reset user agent, clear out cookies, seed more links
            if self.hour_trigger:
                if hasattr(self,'driver'):
                    self.set_user_agent()
                    if True: pass
                    elif False:
                        # `set_user_agent` reopens chromedriver now
                        self.quit_driver()
                        self.open_driver()
                    else:
                        try:
                            @self.chromedriver_short_timeout
                            def chromedriver_delete_all_cookies(): self.driver.delete_all_cookies()
                            chromedriver_delete_all_cookies()
                        except Exception as e:
                            if self.debug: print(f'.delete_all_cookies() exception:\n{e}')
                    self.seed_links()
                else: self.open_driver()
                self.hour_trigger = False
        else:
            self.hour_trigger = True
        self.every_day_tasks()
        self.every_two_weeks_tasks()

    def every_day_tasks(self):
        if int(self.elapsed_time/3600. % 24.) == 23:
            # clear out cookies every day, decimate, and seed more links
            if self.twentyfour_hour_trigger:
                if hasattr(self,'driver'):
                    self.seed_links()
                    # restart the driver
                    self.quit_driver()
                    self.open_driver()
                else:
                    self.open_driver()
                    self.decimate_links(total_frac=0.667, decimate_frac=0.1)
                    self.seed_links()
                    if self.quit_driver_every_call: self.quit_driver()
                self.twentyfour_hour_trigger = False
        else:
            self.twentyfour_hour_trigger = True

    def every_two_weeks_tasks(self):
        if self.elapsed_time > 3600.*24*14:
            # reset bw stats and (really) decimate the stack every couple of weeks
            self.start_time = time.time()
            self.data_usage = 0
            self.decimate_links(total_frac=0.49, decimate_frac=0.333)
            self.get_blacklist(update_flag=True)  # reload the latest blacklists

    def decimate_links(self, total_frac=0.81, decimate_frac=0.1, log_sampling=False):
        """ Delete `decimate_frac` of links if the total exceeds `total_frac` of the maximum allowed. """
        if self.link_count() > int(np.ceil(total_frac * self.max_links_cached)):
            for url in self.draw_links(n=int(np.ceil(self.link_count()*decimate_frac)),log_sampling=log_sampling):
                self.remove_link(url)

    def set_user_agent(self):
        self.draw_user_agent()
        # chromedriver cannot reset the User-Agent in runtime, so it must be restarted with a new UA
        # https://stackoverflow.com/questions/50375628/how-to-change-useragent-string-in-runtime-chromedriver-selenium/50375914#50375914
        self.open_driver()

    def draw_user_agent(self,max_draws=10000):
        """Draw a random User-Agent either uniformly (mildly susceptible to ML), or from a matched distribution."""
        global ua_parse_flag, user_agent
        if not ua_parse_flag:
            self.user_agent = self.fake_ua.random if npr.random() < 0.95 else user_agent
            return
        # Draw User-Agent from pre-defined property distribution
        property_pvals = self.property_pvals
        k = 0
        while k < max_draws:
            uap = ua.parse(self.fake_ua.random)
            # print(uap.ua_string)
            p_browser = property_pvals['browser']['noneoftheabove']
            for ky in property_pvals['browser']:
                if bool(re.findall(ky, uap.browser.family, flags=re.IGNORECASE)):
                    p_browser = property_pvals['browser'][ky]
                    break
            p_os = property_pvals['os']['noneoftheabove']
            for ky in property_pvals['os']:
                if bool(re.findall(ky, uap.os.family, flags=re.IGNORECASE)):
                    p_os = property_pvals['os'][ky]
                    break
            p_pc = property_pvals['is_pc'][uap.is_pc]
            p_touch_capable = property_pvals['is_touch_capable'][uap.is_touch_capable]
            if npr.uniform() <= p_browser \
                    and npr.uniform() <= p_os \
                    and npr.uniform() <= p_pc \
                    and npr.uniform() <= p_touch_capable: break
            k += 1
        self.user_agent = uap.ua_string

    def draw_link(self,log_sampling=True):
        """ Draw a single, random link. """
        return self.draw_links(n=1,log_sampling=log_sampling)[0]

    def draw_links(self,n=1,log_sampling=False):
        """ Draw multiple random links. """
        urls = []
        domain_array = np.array([dmn for dmn in self.domain_links])
        domain_count = np.array([len(self.domain_links[domain_array[k]]) for k in range(domain_array.shape[0])])
        p = np.array([np.float(c) for c in domain_count])
        count_total = p.sum()
        if log_sampling:  # log-sampling [log(x+1)] to bias lower count domains
            p = np.fromiter((np.log1p(x) for x in p), dtype=p.dtype)
        if count_total > 0:
            p = p/p.sum()
            cnts = npr.multinomial(n, pvals=p)
            if n > 1:
                for k in range(cnts.shape[0]):
                    domain = domain_array[k]
                    cnt = min(cnts[k],domain_count[k])
                    for url in random.sample(self.domain_links[domain],cnt):
                        urls.append(url)
            else:
                k = int(np.nonzero(cnts)[0])
                domain = domain_array[k]
                url = random.sample(self.domain_links[domain],1)[0]
                urls.append(url)
        return urls

    def draw_domain(self,log_sampling=False):
        """ Draw a single, random domain. """
        domain = None
        domain_array = np.array([dmn for dmn in self.domain_links])
        domain_count = np.array([len(self.domain_links[domain_array[k]]) for k in range(domain_array.shape[0])])
        p = np.array([np.float(c) for c in domain_count])
        count_total = p.sum()
        if log_sampling:  # log-sampling [log(x+1)] to bias lower count domains
            p = np.fromiter((np.log1p(x) for x in p), dtype=p.dtype)
        if count_total > 0:
            p = p/p.sum()
            cnts = npr.multinomial(1, pvals=p)
            k = int(np.nonzero(cnts)[0])
            domain = domain_array[k]
        return domain

    def draw_link_from_domain(self,domain):
        """ Draw a single, random link from a specific domain. """
        domain_count = len(self.domain_links.get(domain,set()))
        url = random.sample(self.domain_links[domain],1)[0] if domain_count > 0 else None
        return url

    def pop_link(self,remove_link_fraction=0.95,current_preferred_domain_fraction=0.1):
        """ Pop a link from the collected list.
If `self.current_preferred_domain` is defined, then a link from this domain is drawn
a fraction of the time. """
        url = None
        if hasattr(self,'current_preferred_domain') and npr.uniform() < current_preferred_domain_fraction:
            while url is not None and len(self.domain_links) > 0:  # loop until `self.current_preferred_domain` has a url
                url = self.draw_link_from_domain(self.current_preferred_domain)
                if url is None: self.current_preferred_domain = self.draw_domain()
        if url is None: url = self.draw_link()
        if npr.uniform() < remove_link_fraction:  # 95% 1 GET, ~5% 2 GETs, .2% three GETs
            self.remove_link(url)  # pop a random item from the stack
        return url

    def add_link(self,url):
        result = False
        domain = self.domain_name(url)
        if self.link_count() < self.max_links_cached \
                and len(self.domain_links.get(domain,[])) < self.max_links_per_domain \
                and url not in self.domain_links.get(domain,set()):
            self.domain_links.setdefault(domain, set())
            self.domain_links[domain].add(url)
            result = True
            # if self.debug: print(f'\tAdded link \'{url}\'…')
        return result

    def remove_link(self,url):
        result = False
        domain = self.domain_name(url)
        if url in self.domain_links.get(domain,set()):
            self.domain_links[domain].remove(url)
            if len(self.domain_links[domain]) == 0:
                del self.domain_links[domain]
            result = True
        return result

    def domain_name(self,url):
        return '.'.join(uprs.urlparse(url).netloc.split('.')[-2:])

    def get_websearch(self,query):
        """
        HTTP GET of a websearch, then add any embedded links.
        :param query: 
        :return: 
        """
        self.select_random_search_engine()
        url = uprs.urlunparse(uprs.urlparse(self.SafeSearch.search_url)._replace(query='{}={}{}&{}'.format(
            self.SafeSearch.query_parameter,uprs.quote_plus(query),
            self.SafeSearch.additional_parameters,self.SafeSearch.safe_parameter)))
        if self.verbose: self.print_url(url)
        @self.chromedriver_timeout
        def chromedriver_get(): self.driver.get(url)  # selenium driver
        chromedriver_get()
        @self.chromedriver_short_timeout
        def chromedriver_page_source(): self.data_usage += len(self.driver.page_source)
        chromedriver_page_source()
        new_links = self.websearch_links()
        if self.link_count() < self.max_links_cached: self.add_url_links(new_links,url)

    def select_random_search_engine(self):
        self.SafeSearch = random.choice([SafeGoogle, SafeBing, SafeYahoo, SafeDuckDuckGo])
        return self.SafeSearch

    def websearch_links(self):
        """
        Webpage format for a popular search engine, <div class="g">.
        :return: 
        """
        # https://github.com/detro/ghostdriver/issues/169
        @self.chromedriver_short_timeout
        def chromedriver_find_elements_by_css_selector():
            return WebDriverWait(self.driver,short_timeout).until(lambda x: x.find_elements_by_css_selector(self.SafeSearch.css_selector))
        elements = chromedriver_find_elements_by_css_selector()
        # get links in random order until max. per page
        k = 0
        links = []
        try:
            for elt in sorted(elements,key=lambda k: random.random()):
                @self.chromedriver_short_timeout
                def chromedriver_find_element_by_tag_name(): return elt.find_element_by_tag_name('a')
                a_tag = chromedriver_find_element_by_tag_name()
                @self.chromedriver_short_timeout
                def chromedriver_get_attribute(): return a_tag.get_attribute('href')
                href = chromedriver_get_attribute()
                if href is not None:
                    href = self.SafeSearch.result_extraction(href)
                    links.append(href)
                k += 1
                if k > self.max_links_per_page or self.link_count() == self.max_links_cached: break
        except Exception as e:
            if self.debug: print(f'.find_element_by_tag_name.get_attribute() exception:\n{e}')
        return links

    def get_url(self,url):
        """
        HTTP GET of the url, and add any embedded links.
        :param url: 
        :return: 
        """
        if not self.check_robots(url): return  # bail out if robots.txt says to
        @self.chromedriver_timeout
        def chromedriver_get(): self.driver.get(url)  # selenium driver
        chromedriver_get()
        @self.chromedriver_short_timeout
        def chromedriver_page_source(): self.data_usage += len(self.driver.page_source)
        chromedriver_page_source()
        new_links = self.url_links()
        if self.link_count() < self.max_links_cached: self.add_url_links(new_links,url)

    def url_links(self):
        """Generic webpage link finder format."""
        # https://github.com/detro/ghostdriver/issues/169
        @self.chromedriver_short_timeout
        def chromedriver_find_elements_by_tag_name():
            return WebDriverWait(self.driver,short_timeout).until(lambda x: x.find_elements_by_tag_name('a'))
        elements = chromedriver_find_elements_by_tag_name()

        # get links in random order until max. per page
        k = 0
        links = []
        try:
            for a in sorted(elements,key=lambda k: random.random()):
                @self.chromedriver_short_timeout
                def chromedriver_get_attribute(): return a.get_attribute('href')
                href = chromedriver_get_attribute()
                if href is not None: links.append(href)
                k += 1
                if k > self.max_links_per_page or self.link_count() == self.max_links_cached: break
        except Exception as e:
            if self.debug: print(f'.get_attribute() exception:\n{e}')
        return links

    def check_robots(self,url):
        result = True
        url_robots = uprs.urlunparse(uprs.urlparse(url)._replace(scheme='https',
            path='/robots.txt', query='', params=''))
        @self.robots_timeout
        def robots_read():
            rp = RobotFileParserUserAgent()
            rp.set_url(url_robots)
            rp.read()
            result = rp.can_fetch(self.user_agent,url)
            del rp      # ensure self.close() in urllib
            return result
        result = robots_read()
        return result

    def add_url_links(self,links,url=''):
        k = 0
        for link in sorted(links,key=lambda k: random.random()):
            lp = uprs.urlparse(link)
            if (lp.scheme == 'http' or lp.scheme == 'https') and not self.blacklisted(link):
                if self.add_link(link): k += 1
                if k > self.max_links_per_page: break
        if self.verbose or self.debug:
            current_url = url  # default
            try:
                @self.chromedriver_short_timeout
                def chromedriver_current_url(): return self.driver.current_url
                current_url = chromedriver_current_url()
                # the current_url method breaks on a lot of sites, e.g.
                # python3 -c 'from selenium import webdriver; driver = webdriver.PhantomJS(); driver.get("https://github.com"); print(driver.title); print(driver.current_url); driver.quit()'
            except Exception as e:
                if self.debug: print(f'.current_url exception:\n{e}')
        if self.debug:
            print("{}: {:d} links added, {:d} total, {:.1f} bits domain entropy".format(current_url,k,self.link_count(),self.domain_entropy()))
        elif self.verbose:
            self.print_progress(current_url,num_links=k)

    def print_url(self,url):
        if self.debug: print(url + ' …')
        else: self.print_progress(url)

    def print_progress(self,url,num_links=None):
        if num_links is not None:
            text_suffix = ': +{:d}/{:d} links, H(domain)={:.1f} b'.format(num_links,self.link_count(),self.domain_entropy())
        else:
            text_suffix = '; {:d} links, H(domain)={:.1f} b …'.format(self.link_count(),self.domain_entropy())
        self.print_truncated_line(url,text_suffix)

    def print_truncated_line(self,url,text_suffix='',terminal_width=terminal_width):
        """
        Print truncated `url` + `text_suffix` to fill `terminal_width`
        :param url: 
        :param text_suffix: 
        :param terminal_width: 
        :return: 
        """
        chars_used = len(text_suffix)
        if text_suffix == '…':
            if len(url) >= terminal_width:
                url = url[:terminal_width-1]  # add '…' below
            elif len(url) < terminal_width-1:
                url += ' '  # add an extra space before the ellipsis
        else:
            if len(url) + chars_used > terminal_width:
                url = url[:terminal_width-chars_used-1] + '…'
        text = f"{url}{text_suffix}"  # added white space necessary
        text = text[:min(terminal_width,len(text))] + ' ' * max(0,terminal_width-len(text))
        print(text,end='',flush=True)
        time.sleep(0.01)
        print('\r',end='',flush=True)

    def blacklisted(self,link):
        return link in self.blacklist_urls or self.domain_name(link) in self.blacklist_domains

    def bandwidth_test(self):
        running_bandwidth = self.data_usage/(self.elapsed_time+900.)
        running_bandwidth = running_bandwidth/407.	# Convert to GB/month, 2**30/(3600*24*30.5)
        # if self.debug: print(f'Using {running_bandwidth} GB/month')
        return running_bandwidth > self.gb_per_month

    # handle chromedriver timeouts
    # configurable decorator to timeout chromedriver and robotparser calls
    # http://stackoverflow.com/questions/15572288/general-decorator-to-wrap-try-except-in-python
    # Syntax:
    # chromedriver_timeout = block_timeout(chromedriver_hang_handler)
    # @chromedriver_timeout
    # def chromedriver_block():
    #     # chromedriver stuff
    #     pass
    # chromedriver_block()

    def block_timeout(self,hang_handler, alarm_time=timeout, errors=(Exception,), debug=False):
        def decorator(func):
            def call_func(*args, **kwargs):
                signal.signal(signal.SIGALRM, hang_handler)  # register hang handler
                signal.alarm(alarm_time)  # set an alarm
                result = None
                try:
                    result = func(*args, **kwargs)
                except errors as e:
                    if debug: print(f'{func.__name__} exception:\n{e}')
                finally:
                    signal.alarm(0)  # cancel the alarm
                return result
            return call_func
        return decorator

    class TimeoutError(Exception):
        pass

    def chromedriver_hang_handler(self, signum, frame):
        # https://github.com/detro/ghostdriver/issues/334
        # http://stackoverflow.com/questions/492519/timeout-on-a-function-call
        if self.debug: print('Looks like chromedriver has hung.')
        try:
            self.quit_driver(chromedriver_short_timeout_decorator=self.chromedriver_quit_timeout)
        except Exception as e:
            if self.debug: print(e)
        self.open_driver()

    def chromedriver_quit_hang_handler(self, signum, frame):
        raise self.TimeoutError('chromedriver .quit method is taking too long')

    def robots_hang_handler(self, signum, frame):
        if self.debug: print('Looks like robotparser has hung.')
        raise self.TimeoutError('robotparser is taking too long')

    def check_chromedriver_process(self):
        """
        Check if chromedriver is running.
        :return: 
        """
        # Check rss and restart if too large, then check existence
        # http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid-in-python
        try:
            if not hasattr(self,'driver'): self.open_driver()
            pid, rss_mb = self.chromedriver_pid_and_memory()
            if rss_mb > self.chromedriver_rss_limit_mb:  # memory limit
                self.quit_driver(pid=pid)
                self.open_driver()
                pid, _ = self.chromedriver_pid_and_memory()
            # check existence
            os.kill(pid, 0)
        except (OSError,psutil.NoSuchProcess,Exception) as e:
            if self.debug: print(f'.chromedriver_pid_and_memory() exception:\n{e}')
            if issubclass(type(e),psutil.NoSuchProcess):
                raise Exception("There's a chromedriver zombie, and the thread shouldn't have reached this statement.")
            return False
        else:
            return True

    def chromedriver_pid_and_memory(self):
        """ Return the pid and memory (MB) of the chromedriver process,
        restart if it's a zombie, and exit if a restart isn't working
        after three attempts. """
        for k in range(3):    # three strikes
            try:
                @self.chromedriver_short_timeout
                def chromedriver_process_pid(): return self.driver.service.process.pid
                pid = chromedriver_process_pid()
                rss_mb = psutil.Process(pid).memory_info().rss / float(2 ** 20)
                break
            except (psutil.NoSuchProcess,Exception) as e:
                if self.debug: print(f'.service.process.pid exception:\n{e}')
                self.quit_driver(pid=pid)
                self.open_driver()
        else:  # throw in the towel and exit if no viable chromedriver process after multiple attempts
            print('No viable chromedriver process after multiple attempts!')
            sys.exit(1)
        return (pid, rss_mb)

    def parse_and_filter_rule_urls(self,line):
        """Convert EasyList domain anchor rule to domain or url."""
        line = line.rstrip()
        # filter out configuration, comment, exception lines, domain-specific, and selector rules
        if re_test(configuration_re, line) or re_test(comment_re, line) or re_test(exception_re, line) or re_test(
            domain_option_re, line) or re_test(selector_re, line): return
        if re_test(option_re, line):
            line = option_re.sub('\\1', line)  # delete all the options and continue
        # ignore these cases
        # blank url case: ignore
        if re_test(httpempty_re, line): return
        # blank line case: ignore
        if not bool(line): return
        # parse all remaining rules
        # treat each of the these cases separately
        # regex case: ignore
        if re_test(regex_re, line): return
        # now that regex's are handled, delete unnecessary wildcards, e.g. /.../*
        line = wildcard_begend_re.sub('\\1', line)
        # domain anchors, || or '|http://a.b' -> domain anchor 'a.b' for regex efficiency in JS
        if re_test(domain_anch_re, line) or re_test(scheme_anchor_re, line):
            # strip off initial || or |scheme://
            if re_test(domain_anch_re, line):
                line = domain_anch_re.sub('\\1', line)
            elif re_test(scheme_anchor_re, line):
                line = scheme_anchor_re.sub("", line)
            # host subcase
            if re_test(da_hostonly_re, line):
                line = da_hostonly_re.sub('\\1', line)
                if not re_test(wild_anch_sep_exc_re, line):  # exact subsubcase
                    if wildcard_ignore_test(line): return
                    self.blacklist_domains |= set([line])
                    return line
                else:
                    return  # regex subsubcase
            # hostpath subcase
            if re_test(da_hostpath_re, line):
                line = da_hostpath_re.sub('\\1', line)
                if not re_test(wild_sep_exc_noanch_re, line) and re_test(pathend_re, line):  # exact subsubcase
                    line = re.sub(r'[/|]$', '', line)  # strip EOL slashes and anchors
                    if wildcard_ignore_test(line): return
                    self.blacklist_urls |= set([line])
                    return line
                else:
                    return  # regex subsubcase
            # hostpathquery default case
            if wildcard_ignore_test(line): return
            self.blacklist_urls |= set([line])
            return line
        # all other non-regex patterns in for the path parts: ignore
        return


# EasyList regular expressions
# See https://github.com/essandess/easylist-pac-privoxy
comment_re = re.compile(r'^\s*?!')   # ! commment
configuration_re = re.compile(r'^\s*?\[[^]]*?\]')  # [Adblock Plus 2.0]
easylist_opts = r'~?\b(?:third\-party|domain|script|image|stylesheet|object(?!-subrequest)|object\-subrequest|xmlhttprequest|subdocument|ping|websocket|webrtc|document|elemhide|generichide|genericblock|other|sitekey|match-case|collapse|donottrack|popup|media|font)\b'
option_re = re.compile(r'^(.*?)\$(' + easylist_opts + r'.*?)$')
# regex's used to exclude options for specific cases
domain_option_re = re.compile(r'\$.*?(?:domain=)')  # discards rules specific to links from specific domains
selector_re = re.compile(r'^(.*?)#\@?#*?.*?$') # #@##div [should be #+?, but old style still used]
regex_re = re.compile(r'^\@{0,2}\/(.*?)\/$')
wildcard_begend_re = re.compile(r'^(?:\**?([^*]*?)\*+?|\*+?([^*]*?)\**?)$')
wild_anch_sep_exc_re = re.compile(r'[*|^@]')
wild_sep_exc_noanch_re = re.compile(r'(?:[*^@]|\|[\s\S])')
exception_re = re.compile(r'^@@(.*?)$')
httpempty_re = re.compile(r'^\|?https?://$')
pathend_re = re.compile(r'(?i)(?:[/|]$|\.(?:jsp?|php|xml|jpe?g|png|p?gif|img|swf|flv|[sp]?html?|f?cgi|pl?|aspx|ashx|css|jsonp?|asp|search|cfm|ico|act|act(?:ion)?|spy|do|stm|cms|txt|imu|dll|io|smjs|xhr|ount|bin|py|dyn|gne|mvc|lv|nap|jam|nhn))',re.IGNORECASE)

domain_anch_re = re.compile(r'^\|\|(.+?)$')
# omit scheme from start of rule -- this will also be done in JS for efficiency
scheme_anchor_re = re.compile(r'^(\|?(?:[\w*+-]{1,15})?://)');  # e.g. '|http://' at start

# (Almost) fully-qualified domain name extraction (with EasyList wildcards)
# Example case: banner.3ddownloads.com^
da_hostonly_re = re.compile(r'^((?:[\w*-]+\.)+[a-zA-Z0-9*-]{1,24}\.?)(?:$|[/^?])$')
da_hostpath_re = re.compile(r'^((?:[\w*-]+\.)+[a-zA-Z0-9*-]{1,24}\.?[\w~%./^*-]+?)\??$')

def re_test(regex,string):
    if isinstance(regex,str): regex = re.compile(regex)
    return bool(regex.search(string))

def wildcard_ignore_test(rule):
    return bool(wild_anch_sep_exc_re.search(rule))

if __name__ == "__main__":
    ISPDataPollution()