02-Generate_Attacks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import configparser\n",
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set to latest model version number\n",
    "\n",
    "def set_model_version_number():\n",
    "    version_number = []\n",
    "    global MODEL_VERSION\n",
    "    global MODEL_PATH\n",
    "\n",
    "    if os.path.exists(os.path.join(MODEL_SAVE_DIRECTORY,MODEL_NAME)):   \n",
    "        for entry in os.listdir(os.path.join(MODEL_SAVE_DIRECTORY,MODEL_NAME)):\n",
    "            version_number.append(entry)       \n",
    "        MODEL_VERSION = version_number[-1]\n",
    "        MODEL_PATH = os.path.join(MODEL_SAVE_DIRECTORY, MODEL_NAME, MODEL_VERSION)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "config = configparser.ConfigParser()\n",
    "config.read('config/main.conf')\n",
    "\n",
    "DATASET = 1\n",
    "MODEL_VERSION =  \"0001\"\n",
    "\n",
    "if DATASET == 1:\n",
    "    set_dataset = \"imdb\"\n",
    "if DATASET == 2:\n",
    "    set_dataset = \"s140\"\n",
    "\n",
    "DATASET_URL = (config[set_dataset]['DATASET_URL'])\n",
    "\n",
    "DATASET_FOLDER = config[set_dataset]['DATASET_FOLDER']\n",
    "DATASET_TAR_FILE_NAME = config[set_dataset]['DATASET_TAR_FILE_NAME']\n",
    "DATASET_NAME = config[set_dataset]['DATASET_NAME']\n",
    "\n",
    "MODEL_NAME = config[set_dataset]['MODEL_NAME']\n",
    "\n",
    "CLEAN_DATA_FILE = os.path.join(DATASET_FOLDER,\"normalized_dataset.csv\")\n",
    "TAR_FILE_PATH = os.path.join(DATASET_FOLDER,DATASET_TAR_FILE_NAME)\n",
    "DATA_SET_LOCATION = os.path.join(DATASET_FOLDER,DATASET_NAME)\n",
    "\n",
    "MODEL_SAVE_DIRECTORY = config[set_dataset]['MODEL_SAVE_DIRECTORY']\n",
    "# Create the model save directory\n",
    "if not os.path.exists(MODEL_SAVE_DIRECTORY):\n",
    "    os.makedirs(MODEL_SAVE_DIRECTORY)\n",
    "    \n",
    "IMAGE_SAVE_FOLDER = config[set_dataset]['IMAGE_SAVE_FOLDER']\n",
    "    \n",
    "GLOVE_EMBEDDINGS = config[set_dataset]['GLOVE_EMBEDDINGS']\n",
    "COUNTER_FITTED_VECTORS = config[set_dataset]['COUNTER_FITTED_VECTORS']\n",
    "\n",
    "GLOVE_EMBEDDINGS_MATRIX = config[set_dataset]['GLOVE_EMBEDDINGS_MATRIX']\n",
    "COUNTER_FITTED_EMBEDDINGS_MATRIX = config[set_dataset]['COUNTER_FITTED_EMBEDDINGS_MATRIX']\n",
    "\n",
    "####### files required to reconstruct the final trained model ##############################\n",
    "MODEL_PATH = os.path.join(MODEL_SAVE_DIRECTORY, MODEL_NAME, MODEL_VERSION)\n",
    "\n",
    "set_model_version_number()\n",
    "\n",
    "ASSESTS_FOLDER = os.path.join(MODEL_PATH,\"assets\")\n",
    "MODEL_ASSETS_VOCABULARY_FILE = os.path.join(ASSESTS_FOLDER,\"vocab\")\n",
    "MODEL_ASSETS_EMBEDDINGS_FILE = os.path.join(ASSESTS_FOLDER,\"imdb_glove_embeddings_matrix\")\n",
    "MODEL_ASSETS_COUNTER_EMBEDDINGS_FILE = os.path.join(ASSESTS_FOLDER,\"counter_embeddings_matrix\")\n",
    "MODEL_ASSETS_DISTANCE_MATRIX = os.path.join(ASSESTS_FOLDER,\"distance_matrix.npy\")\n",
    "MODEL_ASSETS_SAVE_BEST_WEIGHTS = os.path.join(ASSESTS_FOLDER, \"cp.ckpt\")\n",
    "MODEL_TRAINING_HISORTY_FILE = os.path.join(ASSESTS_FOLDER, \"training_history.csv\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 01) Load our pre trained sentiment model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7efc64141d30>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization # in Tensorflow 2.1 and above\n",
    "import pickle \n",
    "\n",
    "MAX_VOCABULARY_SIZE = 50000\n",
    "DIMENSION = 300\n",
    "LEARNING_RATE = 1e-4\n",
    "\n",
    "\n",
    "from manny_modules import tf_normalize_data as tfnd\n",
    "from manny_modules import return_model as rmodel\n",
    "\n",
    "saved_vocab = pickle.load(open(MODEL_ASSETS_VOCABULARY_FILE, 'rb'))\n",
    "saved_word_index = dict(zip(saved_vocab, range(len(saved_vocab))))\n",
    "\n",
    "saved_embeddings_matric = pickle.load(open(MODEL_ASSETS_EMBEDDINGS_FILE, 'rb'))\n",
    "\n",
    "\n",
    "vectorizer_layer = TextVectorization(\n",
    "    standardize=tfnd.normlize_data, \n",
    "    max_tokens=MAX_VOCABULARY_SIZE, \n",
    "    output_mode='int',\n",
    "    output_sequence_length=300)\n",
    "\n",
    "# build vocabulary, will also run the normalize_data() \n",
    "vectorizer_layer.set_vocabulary(saved_vocab)\n",
    "\n",
    "\n",
    "saved_model = rmodel.create_model(vectorizer_layer,\n",
    "                                  saved_embeddings_matric,\n",
    "                                  saved_vocab,\n",
    "                                  dimension=DIMENSION, \n",
    "                                  lrate=LEARNING_RATE)\n",
    "\n",
    "# load the weights\n",
    "saved_model.load_weights(MODEL_ASSETS_SAVE_BEST_WEIGHTS) # loads best weights saved during training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test model - check predictions for unseen data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive confidence:  [[0.01855881]]  Negative confidence:  [[0.9814412]]\n",
      "Positive confidence:  [[0.9553545]]  Negative confidence:  [[0.04464549]]\n"
     ]
    }
   ],
   "source": [
    "# check negative review\n",
    "p_2 = [[\"Seriously, don't bother if you're over 12. This looks like a kids show designed purely to sell merchandise, theme park rides, etc. No logic, holes all over the shop, no characters motivation and really crap acting to top it off... just rubbish, really\"]]\n",
    "prob_positive = saved_model.predict(p_2)\n",
    "print(\"Positive confidence: \",prob_positive, \" Negative confidence: \", (1 - prob_positive ))\n",
    "\n",
    "\n",
    "# check positive review\n",
    "p = [[\"It's one thing to bring back elements, characters, settings and stories, and to flash them in front of the audience to cash in on the nostalgia and/or recognisable memorabilia but without using it to further the plot and other to do exactly the opposite. It was about time that Star Wars directives understood that it is too unique a product to be lend to corporate filmmakers. Star Wars needs to be understood and its uniqueness has to be acknowledged in order to make the new stories feel like they belong. This may sound too obvious but if you ever wondered why the new SW movies are so controversial this may be the reason.Like with 'Spider-Man: Into the Spider-verse (2018)' and their comicbook-industry experts participation, the creators behind The Mandalorian were experts of the industry, connoisseurs of the Star Wars Universe and even long time fans. So they were able to not only recapture the aesthetic of the grimy, battered Star Wars but also build upon it taking the most 'subtle' things into account. Things like the predominancy of puppets and practical effects over CGI, settings you can feel and touch over green screens and the abundancy of not only known elements previously seen in Star Wars, but a whole batch of new creatures, designs and overall plot elements that felt like they belong to this universe and had always been there. Exceeding expectations are not only the visual aspects but the narrative too. It might be too late for some story elements now, but it is of great importance that from now on you try to watch the unraveling of the story unspoiled. I was lucky to have seen the premiere of the show before the 'memefication' of a certain 'element' that went viral and became one of the biggest highlights of the show. But for me I saw the reveal of this element unspoiled and I was pleasantly shocked, a memory I'll always carry with me. The ability of these creators to generate such shock value and deep moments it's often baffling to me. This is proof that the creators behind the narrative are fully aware of the complexities of the universe they are tampering with and like an experienced surgeon, they are able to tweak, traverse and call back any Star Wars element as they please and with astonishing results.\"]]\n",
    "prob_positive = saved_model.predict(p)\n",
    "print(\"Positive confidence: \",prob_positive, \" Negative confidence: \", (1 - prob_positive ))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load the distance matrix from disk (load this before running below tests)\n",
    "- This is a large file (~20GB), so will take time to load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "distance_matrix = np.load(MODEL_ASSETS_DISTANCE_MATRIX)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### test the distance matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_word =  saved_word_index['scotland']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "metadata": {},
   "outputs": [],
   "source": [
    "from manny_modules import nearest_neighbour as nn\n",
    "\n",
    "nearest_neighbour, distance_to_neighbour = nn.closest_neighbours(target_word, distance_matrix, number_of_words_to_return=5, max_distance=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Words closest to `scotland` are `['scot', 'scots', 'scottish', 'scotsman', 'scotch']` \n"
     ]
    }
   ],
   "source": [
    "closest_word = [saved_vocab[x] for x in nearest_neighbour]\n",
    "\n",
    "print(\"Words closest to `%s` are `%s` \" % (saved_vocab[target_word], closest_word))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 02) Load data sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load the saved sample dataset - use the same dataset for both BERT and distance matrix GA attacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 579,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import date\n",
    "\n",
    "dtypes = {'sentiment': 'int', 'text': 'str'}\n",
    "\n",
    "# if True then load saved sample dataset, else create a sample dataset\n",
    "SAVED_DATASET = True\n",
    "SAMPLE_SIZE = 1000\n",
    "\n",
    "def load_sample_dataset(): \n",
    "    global SAMPLE_SIZE\n",
    "    if SAVED_DATASET:\n",
    "        # load the same data-sample used in original attack\n",
    "        data_sample = pd.read_csv('imdb_dataset/sample_dataset.csv',dtype=dtypes).dropna()\n",
    "        print(\"Number of data items in sample: \",len(data_sample))\n",
    "        SAMPLE_SIZE = len(data_sample)\n",
    "    else:\n",
    "        #generate new sample from original data set\n",
    "        data_frame = pd.read_csv(CLEAN_DATA_FILE,dtype=dtypes)\n",
    "        data_sample = data_frame.sample(n = SAMPLE_SIZE).dropna()\n",
    "        \n",
    "        # if we generate a new sample dataset - save it with a date stamp\n",
    "        data_sample.to_csv('imdb_dataset/sample_dataset_'+ str(date.today()) + '_.csv', index = False)\n",
    "        SAMPLE_SIZE = len(data_sample)\n",
    "    return data_sample\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### add new columns to hold results after genetic attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 681,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_ga_columns(data_sample):\n",
    "\n",
    "    data_sample['probs'] = 0\n",
    "    data_sample['probs'] = data_sample['probs'].astype(float) # has to be of type float, to store probability values\n",
    "    \n",
    "    # copy current sentiment values to new columns, we can then later go through and compare any values that have been changed during the GA attack\n",
    "    data_sample['ga_sentiment'] = data_sample['sentiment']\n",
    "\n",
    "    # create new ga_text to hold perturbed text\n",
    "    data_sample['ga_text'] = \"\"\n",
    "    \n",
    "    # create ga_probs to hold new probability values after GA Attack, fill with current values\n",
    "    data_sample['ga_probs'] = 0.0\n",
    "    data_sample['ga_probs'] = data_sample['ga_probs'].astype(float)\n",
    "\n",
    "    # create ga_num_changes to hold the number of words changed\n",
    "    data_sample['ga_num_changes'] = 0\n",
    "    data_sample['ga_num_changes'] = data_sample['ga_num_changes'].astype(int)\n",
    "\n",
    "    # create ga_lev_ratio to hold the Levenshtein ratio\n",
    "    data_sample['ga_lev_ratio'] = 0.0\n",
    "    data_sample['ga_lev_ratio'] = data_sample['ga_lev_ratio'].astype(float)\n",
    "\n",
    "    # add field to indicate if sentiment was flipped on review text\n",
    "    data_sample['ga_flipped_sentiment'] = 'N'\n",
    "    data_sample['ga_flipped_sentiment'] = data_sample['ga_flipped_sentiment'].astype(str)\n",
    "\n",
    "\n",
    "    # percentage of words changed in sentence\n",
    "    data_sample['ga_percent_change'] = 0.0\n",
    "    data_sample['ga_percent_change'] = data_sample['ga_percent_change'].astype(float)\n",
    "\n",
    "    data_sample = data_sample.reset_index(drop=True) # reindex so we start from 0 in the sample data set\n",
    "    \n",
    "    return data_sample\n",
    "\n",
    "def add_ga_columns_stats(data_sample):\n",
    "\n",
    "    data_sample['probs'] = 0\n",
    "    data_sample['probs'] = data_sample['probs'].astype(float) # has to be of type float, to store probability values\n",
    "    \n",
    "    # copy current sentiment values to new columns, we can then later go through and compare any values that have been changed during the GA attack\n",
    "    data_sample['ga_sentiment'] = data_sample['sentiment']\n",
    "\n",
    "    # create new ga_text to hold perturbed text\n",
    "    data_sample['ga_text'] = data_sample['text']\n",
    "    \n",
    "    \n",
    "    # create ga_probs to hold new probability values after GA Attack, fill with current values\n",
    "    data_sample['ga_probs'] = 0.0\n",
    "    data_sample['ga_probs'] = data_sample['ga_probs'].astype(float)\n",
    "\n",
    "    # create ga_num_changes to hold the number of words changed\n",
    "    data_sample['ga_num_changes'] = 0\n",
    "    data_sample['ga_num_changes'] = data_sample['ga_num_changes'].astype(int)\n",
    "\n",
    "    # create ga_lev_ratio to hold the Levenshtein ratio\n",
    "    data_sample['ga_lev_ratio'] = 0.0\n",
    "    data_sample['ga_lev_ratio'] = data_sample['ga_lev_ratio'].astype(float)\n",
    "\n",
    "    # add field to indicate if sentiment was flipped on review text\n",
    "    data_sample['ga_flipped_sentiment'] = 'N'\n",
    "    data_sample['ga_flipped_sentiment'] = data_sample['ga_flipped_sentiment'].astype(str)\n",
    "\n",
    "\n",
    "    # percentage of words changed in sentence\n",
    "    data_sample['ga_percent_change'] = 0.0\n",
    "    data_sample['ga_percent_change'] = data_sample['ga_percent_change'].astype(float)\n",
    "    \n",
    "    # create ga_num_changes to hold the number of words changed\n",
    "    data_sample['ga_generations'] = 0\n",
    "    data_sample['ga_generations'] = data_sample['ga_generations'].astype(int)\n",
    "    \n",
    "    # create ga_lev_ratio to hold the Levenshtein ratio\n",
    "    data_sample['ga_time_taken'] = 0\n",
    "    data_sample['ga_time_taken'] = data_sample['ga_time_taken'].astype(int)\n",
    "\n",
    "    data_sample = data_sample.reset_index(drop=True) # reindex so we start from 0 in the sample data set\n",
    "    \n",
    "    return data_sample\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### now run each one against model and store current probabilities that text is a positive sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 581,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_current_probabilities(data_sample): \n",
    "    print(\"Calculating probabilities for sample dataset...\")\n",
    "    for i in data_sample.index:\n",
    "        p = saved_model.predict([data_sample.iloc[i]['text']])\n",
    "        data_sample.at[i,'probs']= p   \n",
    "        \n",
    "    # copy over current values to ga_probs column\n",
    "    data_sample['ga_probs'] = data_sample['probs']\n",
    "    return data_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check the predictions are correct, if not then drop row from data set\n",
    "### we only want to keep correctly classified data items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def check_predictions(data_sample):\n",
    "    \n",
    "    original_size = len(data_sample)\n",
    "    drop_indexes = []\n",
    "    print(\"Checking proabilities for sample dataset are correctly labelled...\")\n",
    "    for i in data_sample.index:\n",
    "        if data_sample.iloc[i]['sentiment'] == 1 and data_sample.iloc[i]['probs'] > 0.5:\n",
    "            continue\n",
    "        if data_sample.iloc[i]['sentiment'] == 0 and data_sample.iloc[i]['probs'] <= 0.5:\n",
    "            continue\n",
    "        else:\n",
    "            drop_indexes.append(i)\n",
    "    # drop data items that are not correctly classified by our model\n",
    "    # for our test sample we want to be sure all samples start out correctly classified\n",
    "    data_sample = data_sample.drop(drop_indexes)\n",
    "    data_sample = data_sample.reset_index(drop=True) # reindex dataframe to start from 0\n",
    "\n",
    "    # check how many rows we dropped due to incorrect classification\n",
    "    print(\"Number of data items dropped from sample: \",(original_size - len(data_sample)))\n",
    "    print(\"Number of data items kept in sample: \",(len(data_sample)))\n",
    "    \n",
    "    return data_sample\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 03) GA functions "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 595,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-large-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "from manny_modules import nearest_neighbour as nn\n",
    "from random import randrange\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from pytorch_pretrained_bert import BertTokenizer,BertForMaskedLM\n",
    "import torch\n",
    "from transformers import pipeline\n",
    "import math\n",
    "from operator import itemgetter\n",
    "\n",
    "stop_words = set(stopwords.words('english')) \n",
    "\n",
    "\n",
    "bertMaskedLM = BertForMaskedLM.from_pretrained('bert-large-uncased')\n",
    "bertMaskedLM.eval()\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')\n",
    "\n",
    "# create pipeline to process masked words\n",
    "# also re-try using bert-large-uncased\n",
    "# mask_word_pipeline = pipeline('fill-mask', model='bert-large-uncased') \n",
    "mask_word_pipeline = pipeline('fill-mask', model='bert-large-uncased')  \n",
    "\n",
    "\n",
    "\n",
    "# # @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n",
    "#################### START BERT functions #############################################################\n",
    "\n",
    "def return_masked_words(p, masked_sentence):\n",
    "    '''return list of possible masked words'''\n",
    "    return p(masked_sentence)\n",
    "\n",
    "def get_sentence_score(sentence):\n",
    "    ''' get a sentence score: the lower the score i.e. more valid the sentence'''\n",
    "    tokenized_input = tokenizer.tokenize(sentence)\n",
    "    tensor_input = torch.tensor([tokenizer.convert_tokens_to_ids(tokenized_input)])\n",
    "    predictions = bertMaskedLM(tensor_input)\n",
    "    loss_fct = torch.nn.CrossEntropyLoss()\n",
    "    loss = loss_fct(predictions.squeeze(),tensor_input.squeeze()).data\n",
    "    return math.exp(loss)\n",
    "\n",
    "#################### END BERT functions #############################################################\n",
    "# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n",
    "\n",
    "\n",
    "\n",
    "# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n",
    "#################### START GA functions #############################################################\n",
    "\n",
    "def fitness(population_list, model):\n",
    "    \n",
    "    # dataframe to store population text and probabilities\n",
    "    population_probs = pd.DataFrame(columns=['ga_text','ga_probs'])\n",
    "    \n",
    "    # make sure columns have the correct types\n",
    "    population_probs['ga_text'] = population_probs['ga_text'].astype(str)\n",
    "    population_probs['ga_probs'] = population_probs['ga_probs'].astype(float)\n",
    "    \n",
    "    for i in range(len(population_list)):\n",
    "        new_row = {'ga_text':population_list[i], 'ga_probs':model.predict([population_list[i]])}\n",
    "        #append row to the dataframe\n",
    "        population_probs = population_probs.append(new_row, ignore_index=True)\n",
    "    \n",
    "    # sort the array by probabilities column, i.e column 2 \n",
    "    return  population_probs.sort_values('ga_probs') # return sorted df, sorted by probability lowest to highest\n",
    "    \n",
    "\n",
    "def found_solution(target_label, population_df):\n",
    "        \n",
    "    if target_label == 1:\n",
    "        if population_df.iloc[-1]['ga_probs'] > 0.5:\n",
    "            return True\n",
    "    \n",
    "    if target_label == 0:\n",
    "        if population_df.iloc[0]['ga_probs'] <= 0.5:\n",
    "            return True \n",
    "        \n",
    "    return False\n",
    "\n",
    "\n",
    "def number_of_changes_made(review_before, review_after):\n",
    "    word_count = 0\n",
    "    r_before = review_before.split()\n",
    "    r_after = review_after.split()\n",
    "    for i in range(len(r_before)):\n",
    "        if r_before[i] == r_after[i]:\n",
    "            pass\n",
    "        else:\n",
    "            word_count += 1\n",
    "    return word_count\n",
    "\n",
    "\n",
    "def prediction_probability(model, text_review):\n",
    "    '''return probability of this string being a positive sentiment'''\n",
    "    return model.predict([text_review])\n",
    "\n",
    "\n",
    "def crossover(parent_one, parent_two):\n",
    "    p_one = parent_one.split()\n",
    "    p_two = parent_two.split()\n",
    "    \n",
    "    new_offspring = p_one.copy()\n",
    "    \n",
    "    text_len = min(len(new_offspring), len(p_two))\n",
    "    # use random uniform distribution to select which words to replace \n",
    "    # when creating the new offspring for our two parent strings\n",
    "    # Draw samples from a uniform distribution, i.e. each word is equally likely to be selected\n",
    "    # probability density function:  p(x) = 1 / (b - a)\n",
    "    for i in range(text_len):\n",
    "        if np.random.uniform() < 0.5:\n",
    "            new_offspring[i] = p_two[i]\n",
    "    return ' '.join(new_offspring)\n",
    "    \n",
    "    \n",
    "def mutation_distance(model, text_review, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab):\n",
    "    '''returns the string after swapping max_perturbations nearest neighbours\n",
    "    of each string'''\n",
    "    \n",
    "    # keep track of list index of the which word we have already changed\n",
    "    selected_index = []\n",
    "    \n",
    "    #split string so we can iterate over each word\n",
    "    t_split = text_review.split()\n",
    "    \n",
    "    # select a random index value\n",
    "    indx = randrange(len(t_split))\n",
    "    \n",
    "    for i in range(max_perturbations):\n",
    "        # get a random index number for word list\n",
    "        indx = randrange(len(t_split))\n",
    "        \n",
    "        found_in_vocab = False\n",
    "        # skip over all stop words and any indexes we have already selected\n",
    "        while t_split[indx] in stop_words or indx in selected_index  or not found_in_vocab:     \n",
    "            \n",
    "            indx = randrange(len(t_split))\n",
    "            # if word is not found in vocabulary, skip it and try next word\n",
    "            try:\n",
    "                target_word = saved_word_index[t_split[indx]]\n",
    "                found_in_vocab = True\n",
    "            except KeyError:\n",
    "                found_in_vocab = False\n",
    "        \n",
    "        # now we have a word that is not a stop word and has not already been selected\n",
    "        selected_index.append(indx) # add to our list\n",
    "        \n",
    "        # we want to now get a list of the closest max_neighbours synonyms\n",
    "        target_word = saved_word_index[t_split[indx]]\n",
    "     \n",
    "        nearest_neighbour, _ = nn.closest_neighbours(target_word, distance_matrix, number_of_words_to_return=max_neighbours)\n",
    "        \n",
    "        # create a list of the closest words returned\n",
    "        closest_word = [saved_vocab[x] for x in nearest_neighbour]\n",
    "        \n",
    "        # now we need to substitute each word and find the new probability after each substitution\n",
    "        # we need the original label and the target label we are aiming for\n",
    "       \n",
    "        original_word = t_split[indx]\n",
    "        word_prob_dict = dict()\n",
    "        word_prob_dict[original_word] = model.predict([' '.join(t_split)])\n",
    "        for w in closest_word:\n",
    "            if not w: # if we have an empty string then do nothing, we don't want to remove a word from the string\n",
    "                continue\n",
    "            t_split[indx] = w\n",
    "            word_prob_dict[w] = model.predict([' '.join(t_split)])\n",
    "            \n",
    "        # didn't find any suitable words\n",
    "        if len(word_prob_dict) == 0:\n",
    "            continue\n",
    "        \n",
    "        # the word we decided to substitute is based on the probability returned by the model\n",
    "        # if we have target_label == 1 then we are trying to go from negative to positive sentiment\n",
    "        # therefore we want to keep the highest probability returned\n",
    "        # NB the probability returned by our model is the probability that the review is positive, higher values == more positive sentiment\n",
    "        if target_label == 1:\n",
    "            sub_word = max(word_prob_dict, key=word_prob_dict.get)\n",
    "            t_split[indx] = sub_word\n",
    "            \n",
    "        # if our target_label == 0 i.e. negative, then we are going from positive to negative\n",
    "        # hence we want to keep only the lowest value\n",
    "        else:\n",
    "            sub_word = min(word_prob_dict, key=word_prob_dict.get)\n",
    "            t_split[indx] = sub_word\n",
    "        \n",
    "        \n",
    "        # add selected index to selected_index list\n",
    "        selected_index.append(indx)\n",
    "\n",
    "    return ' '.join(t_split)\n",
    "    \n",
    "    \n",
    "def mutation_bert(model, text_review, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab):\n",
    "    '''after picking word that best fits in the context of the partial sentence and select the one that takes us closest to out target label'''\n",
    "    \n",
    "    # keep track of list index of the which word we have already changed\n",
    "    selected_index = []\n",
    "    \n",
    "    #split string so we can iterate over each word\n",
    "    t_split = text_review.split()\n",
    "    \n",
    "    # select a random index value\n",
    "    indx = randrange(3, len(t_split) - 4)\n",
    "    \n",
    "    for i in range(max_perturbations):\n",
    "        # get a random index number for word list\n",
    "        # start at 4th word and upto 4th last word index\n",
    "        indx = randrange(3, len(t_split) - 4)\n",
    "        \n",
    "        found_in_vocab = False\n",
    "        # skip over all stop words and any indexes we have already selected\n",
    "        while t_split[indx] in stop_words or indx in selected_index  or not found_in_vocab:     \n",
    "            \n",
    "            indx = randrange(3, len(t_split) - 4)\n",
    "            # if word is not found in vocabulary, skip it and try next word\n",
    "            try:\n",
    "                target_word = saved_word_index[t_split[indx]]\n",
    "                found_in_vocab = True\n",
    "            except KeyError:\n",
    "                found_in_vocab = False\n",
    "        \n",
    "        # now we have a word that is not a stop word and has not already been selected\n",
    "        selected_index.append(indx) # add to our list     \n",
    "       \n",
    "        # we want to now get a list of the closest max_neighbours using BERT - these may NOT be synonyms\n",
    "        # we use the two words before and 2 after so we have context for chosen word\n",
    "        # and then use BERT to return a list of possible substitutions for our masked word\n",
    "        join_masked = [t_split[indx  - 3],t_split[indx  - 2],t_split[indx  - 1],'[MASK]',t_split[indx  + 1], t_split[indx  + 2],t_split[indx  + 3]]\n",
    "        masked_word_string = ' '.join(join_masked) \n",
    "        \n",
    "        # now we need to pass this string to BERT and get a list of possible substitutions back\n",
    "        possible_substitutions = return_masked_words(mask_word_pipeline, masked_word_string)\n",
    "        \n",
    "        # if one of the words returned is the same as the original word - we want to remove it from our list\n",
    "        # also remove any non alpha returns\n",
    "        # generate a list containing the closest words i.e. best substitutions\n",
    "        closest_word = []\n",
    "        for i in range(len(possible_substitutions)):\n",
    "            if possible_substitutions[i]['token_str'] == t_split[indx] or not possible_substitutions[i]['token_str'].isalpha():\n",
    "                continue\n",
    "            else:\n",
    "                closest_word.append(possible_substitutions[i]['token_str'])\n",
    "            \n",
    "        # substitute each word in turn and then test return from our model and add to dictionary\n",
    "        original_word = t_split[indx]\n",
    "        word_prob_dict = dict()\n",
    "        word_prob_dict[original_word] = model.predict([' '.join(t_split)])\n",
    "        for w in closest_word:\n",
    "            if not w: # if we have an empty string then do nothing, we don't want to remove a word from the string\n",
    "                continue\n",
    "            t_split[indx] = w\n",
    "            \n",
    "            # generate probabilities using each word and add to dict()\n",
    "            word_prob_dict[w] = model.predict([' '.join(t_split)])\n",
    "            \n",
    "        # didn't find any suitable words\n",
    "        if len(word_prob_dict) == 0:\n",
    "            continue\n",
    "        \n",
    "        # the word we decided to substitute is based on the probability returned by the model\n",
    "        # if we have target_label == 1 then we are trying to go from negative to positive sentiment\n",
    "        # therefore we want to keep the highest probability returned\n",
    "        # NB the probability returned by our model is the probability that the review is positive, higher values == more positive sentiment\n",
    "        if target_label == 1:\n",
    "            sub_word = max(word_prob_dict, key=word_prob_dict.get)\n",
    "            t_split[indx] = sub_word\n",
    "            \n",
    "        # if our target_label == 0 i.e. negative, then we are going from positive to negative\n",
    "        # hence we want to keep only the lowest value\n",
    "        else:\n",
    "            sub_word = min(word_prob_dict, key=word_prob_dict.get)\n",
    "            t_split[indx] = sub_word\n",
    "        \n",
    "        \n",
    "        # add selected index to selected_index list\n",
    "        selected_index.append(indx)\n",
    "\n",
    "    # return the string after substituting the best fit word, i.e the one that takes us towards our target label\n",
    "    return ' '.join(t_split)\n",
    "\n",
    "def mutation_bert_and_distance(model, text_review, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab):\n",
    "    '''returns the string after swapping the synonym that best fits and produces the most gramatically correct sentence'''\n",
    "    \n",
    "    # keep track of list index of the which word we have already changed\n",
    "    selected_index = []\n",
    "    \n",
    "    #split string so we can iterate over each word\n",
    "    t_split = text_review.split()\n",
    "    \n",
    "    # select a random index value make sure to leave 2 index values before and at the end\n",
    "    indx = randrange(3, len(t_split) - 4)\n",
    "    \n",
    "    for i in range(max_perturbations):\n",
    "        # get a random index number for word list\n",
    "        indx = randrange(3, len(t_split) - 4)\n",
    "        \n",
    "        found_in_vocab = False\n",
    "        # skip over all stop words and any indexes we have already selected\n",
    "        while t_split[indx] in stop_words or indx in selected_index  or not found_in_vocab:     \n",
    "            \n",
    "            indx = randrange(3, len(t_split) - 4)\n",
    "            # if word is not found in vocabulary, skip it and try next word\n",
    "            try:\n",
    "                target_word = saved_word_index[t_split[indx]]\n",
    "                found_in_vocab = True\n",
    "            except KeyError:\n",
    "                found_in_vocab = False\n",
    "        \n",
    "        # now we have a word that is not a stop word and has not already been selected\n",
    "        selected_index.append(indx) # add to our list\n",
    "        \n",
    "        # we want to now get a list of the closest max_neighbours synonyms\n",
    "        target_word = saved_word_index[t_split[indx]]\n",
    "     \n",
    "        nearest_neighbour, _ = nn.closest_neighbours(target_word, distance_matrix, number_of_words_to_return=max_neighbours)\n",
    "        \n",
    "        # create a list of the closest words returned\n",
    "        closest_word = [saved_vocab[x] for x in nearest_neighbour]\n",
    "        \n",
    "        \n",
    "        # now we need to substitute each word and find the new probability after each substitution\n",
    "        # we need the original label and the target label we are aiming for\n",
    "       \n",
    "        original_word = t_split[indx]\n",
    "        word_sentence_score = dict()\n",
    "\n",
    "        for w in closest_word:\n",
    "            if not w: # if we have an empty string then do nothing, we don't want to remove a word from the string\n",
    "                continue\n",
    "            t_split[indx] = w\n",
    "            # construct short sentence with 2 words before and 2 after our substituted word\n",
    "            sentence_list = [t_split[indx - 3],t_split[indx - 2], t_split[indx - 1],t_split[indx], t_split[indx + 1], t_split[indx + 2],t_split[indx + 3]]\n",
    "            sentence_string = ' '.join(sentence_list)\n",
    "            # now check the sentence score for each synonym and add the score to dict()\n",
    "            word_sentence_score[w] = get_sentence_score(sentence_string)\n",
    "        \n",
    "        # didn't find any suitable words\n",
    "        if len(word_sentence_score) == 0:\n",
    "            continue\n",
    "        \n",
    "        # find the word that gives the best score for sentence structure\n",
    "        # and replace our original word with it\n",
    "        sub_word = min(word_sentence_score, key=word_sentence_score.get)\n",
    "        t_split[indx] = sub_word\n",
    "            \n",
    "        # add selected index to selected_index list so we don't pick the same word again\n",
    "        selected_index.append(indx)\n",
    "\n",
    "    return ' '.join(t_split)\n",
    "    \n",
    "    \n",
    "\n",
    "def generate_population_bert(model, text_review, population_size, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab):  \n",
    "    '''return list of strings of size population_size'''       \n",
    "    return [mutation_bert(model, text_review, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab) for _ in range(population_size)]\n",
    "\n",
    "def generate_population_distance(model, text_review, population_size, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab):  \n",
    "    '''return list of strings of size population_size'''       \n",
    "    return [mutation_distance(model, text_review, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab) for _ in range(population_size)]\n",
    "\n",
    "def generate_population_bert_and_distance(model, text_review, population_size, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab):  \n",
    "    '''return list of strings of size population_size'''       \n",
    "    return [mutation_bert_and_distance(model, text_review, current_prediction, target_label, max_perturbations, max_neighbours, saved_vocab) for _ in range(population_size)]\n",
    "\n",
    "# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n",
    "#################### END GA functions ###############################################################\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GA attack utility functions, stats and load/save functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 669,
   "metadata": {},
   "outputs": [],
   "source": [
    "from manny_modules import normalize_dataset as nd\n",
    "import Levenshtein as lev\n",
    "\n",
    "# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n",
    "#################### START STATS functions ##########################################################\n",
    "\n",
    "def calculate_levenshtein_ratios(data_sample):\n",
    "    # calculate Levenshtein ratio for text and ga_text\n",
    "    # text == initial review text, ga_text == review text after GA Attack\n",
    "    # value closer to 1.0 indicates more similarity, i.e. less changes made to original text\n",
    "    for i in range (len(data_sample)):\n",
    "        data_sample.at[i,'ga_lev_ratio'] = lev.ratio(data_sample.at[i,'text'],data_sample.at[i,'ga_text'])\n",
    "        num_words_changes = number_of_changes_made(data_sample.at[i, \"text\"], data_sample.at[i, \"ga_text\"])\n",
    "        total_words_in_review = len(data_sample.at[i, \"text\"].split())\n",
    "        data_sample.at[i,'ga_percent_change'] = num_words_changes/total_words_in_review\n",
    "    \n",
    "    return data_sample\n",
    "\n",
    "def set_labels_changed(data_sample):\n",
    "\n",
    "    flipped_count = 0\n",
    "    failed_flipped_count = 0\n",
    "    for i in range(len(data_sample)):\n",
    "        if  data_sample.at[i,'sentiment'] != data_sample.at[i,'ga_sentiment'] and (data_sample.at[i,'ga_percent_change'] <= 0.2):\n",
    "            data_sample.at[i,'ga_flipped_sentiment'] = 'Y'\n",
    "            flipped_count += 1\n",
    "        else:\n",
    "            failed_flipped_count += 1\n",
    "            data_sample.at[i,'ga_flipped_sentiment'] = 'N'\n",
    "        \n",
    "    print(\"Percentage of reviews where sentiment was changed after attack:\", round(flipped_count/len(data_sample) * 100, 2),\"%\", \"changed sentiment. i.e.\", flipped_count, \" out of \",len(data_sample))\n",
    "    print(\"Percentage of reviews failed to change sentiment: \", round(failed_flipped_count/len(data_sample) * 100, 2), \"%\",\"did not change, i.e.\", failed_flipped_count, \" out of \",len(data_sample))\n",
    "    \n",
    "    return data_sample\n",
    "\n",
    "def deleted_words_from_text(data_sample):\n",
    "    '''\n",
    "    check to make sure no words were completely removed form reviews\n",
    "    '''\n",
    "    \n",
    "    len_diff_count = 0\n",
    "    for i in range(len(data_sample)):\n",
    "        if  len(data_sample.at[i,'text'].split()) != len(data_sample.at[i,'ga_text'].split()):\n",
    "            len_diff_count += 1\n",
    "            \n",
    "    return False if len_diff_count == 0 else True\n",
    "\n",
    "def set_percentage_modified(data_sample):\n",
    "\n",
    "    word_total_count = 0\n",
    "    percent_modified_total = 0.0\n",
    "    for i in range(len(data_sample)):\n",
    "        review_before = data_sample.at[i,'text'].split()\n",
    "        review_after = data_sample.at[i,'ga_text'].split()\n",
    "        word_count = 0\n",
    "        for j in range(len(review_before)):\n",
    "            if review_before[j] != review_after[j]:\n",
    "                word_count += 1\n",
    "        data_sample.at[i, 'ga_num_changes'] = word_count\n",
    "        percent_modified_total += (word_count / len(review_before))\n",
    "        data_sample.at[i, 'ga_percent_change'] = round(word_count / len(review_before), 2)\n",
    "        word_total_count += word_count\n",
    "            \n",
    "\n",
    "\n",
    "    print(\"Avg. num of words changed changes made: \", (int)(word_total_count / len(data_sample)))\n",
    "    print(\"Avg. percentage modified: \", (round(percent_modified_total / len(data_sample), 2) * 100),\"%\")\n",
    "    \n",
    "    return data_sample\n",
    "\n",
    "#################### END STATS functions ############################################################\n",
    "# @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n",
    "\n",
    "\n",
    "\n",
    "def load_raw_dataset(file_name):\n",
    "    # load raw dataset i.e. before we normalised it\n",
    "    dtypes = {'sentiment': 'int', \n",
    "              'text': 'str' \n",
    "             }\n",
    "\n",
    "    raw_data_sample = pd.read_csv('imdb_dataset/'+file_name, dtype=dtypes)\n",
    "    rds = raw_data_sample.copy()\n",
    "\n",
    "    # normalise the dataset\n",
    "    normalized_dataset = nd.clean_and_return(rds, 'text')\n",
    "    \n",
    "    return raw_data_sample, normalized_dataset\n",
    "\n",
    "def matching_indexes(data_sample,normalized_dataset):\n",
    "    '''match up index values from data_sample to raw data file'''\n",
    "    index_values_rawdata = []\n",
    "    for i in range(len(data_sample)):\n",
    "        for j in range(len(normalized_dataset)): \n",
    "            if normalized_dataset.at[j,'text'] == data_sample.at[i, 'text']:\n",
    "               # print(raw_data_sample.at[j,'text'],\"\\n\",data_sample.at[i, 'text'],\"\\n\\n\")\n",
    "                index_values_rawdata.append(j)\n",
    "                break\n",
    "    return index_values_rawdata\n",
    "\n",
    "\n",
    "def save_ga_attack_results(data_sample, file_name):\n",
    "    # save the final set of results\n",
    "    data_sample.to_csv('ga_attack_results/'+file_name, index = False)\n",
    "\n",
    "    # load saved file, so we can remove any results that were not processed\n",
    "    dtypes = {'sentiment': 'int', \n",
    "              'text': 'str', \n",
    "              'probs': 'float', \n",
    "              'ga_sentiment': 'int', \n",
    "              'ga_text': 'str',\n",
    "              'ga_probs': 'float', \n",
    "              'ga_num_changes': 'int', \n",
    "              'ga_lev_ratio': 'float', \n",
    "              'ga_flipped_sentiment': 'str',\n",
    "             'ga_percent_change': 'float'}\n",
    "\n",
    "    # make sure we can read and load the file\n",
    "    #data_sample = pd.read_csv('ga_attack_results/'+file_name)\n",
    "    data_sample = pd.read_csv('ga_attack_results/'+file_name, dtype=dtypes)\n",
    "\n",
    "    # drop any rows with nan value - i.e. data items not processed due to Jupyter notebook crash\n",
    "    # so we don't have to re-run the whole GA Attack again\n",
    "    data_sample = data_sample.dropna()\n",
    "\n",
    "    # save the final set of results\n",
    "    data_sample.to_csv('ga_attack_results/'+file_name, index = False)\n",
    "    print(\"Results saved to:\\t\",'ga_attack_results/'+file_name)\n",
    "    return\n",
    "\n",
    "\n",
    "def load_ga_attack_results(file_name):\n",
    "     # load saved file, so we can remove any results that were not processed\n",
    "    dtypes = {'sentiment': 'int', \n",
    "              'text': 'str', \n",
    "              'probs': 'float', \n",
    "              'ga_sentiment': 'int', \n",
    "              'ga_text': 'str',\n",
    "              'ga_probs': 'float', \n",
    "              'ga_num_changes': 'int', \n",
    "              'ga_lev_ratio': 'float', \n",
    "              'ga_flipped_sentiment': 'str',\n",
    "             'ga_percent_change': 'float'}\n",
    "\n",
    "    #return pd.read_csv('ga_attack_results/'+file_name)\n",
    "    return pd.read_csv('ga_attack_results/'+file_name, dtype=dtypes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GA Attack functions for BERT and DISTANCE MATRIX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 736,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "POPULATION_SIZE = 20 # max population size to create \n",
    "MAXIMUM_ITERATIONS = 20 # stopping condition for loop if we do not find an optimal solution increased to 300 for BERT attack\n",
    "MAX_PERTURBATIONS = 5 # maximum number of changes to make for each population member increase from 5 to 6 for BERT\n",
    "MAX_NEIGHBOURS = 4 # maximum number of neighbouring words to return and check against\n",
    "\n",
    "\n",
    "\n",
    "def GA_Attack(data_sample, type_of_attack=2):\n",
    "    data_sample_len = len(data_sample)\n",
    "\n",
    "    for i in range(data_sample_len):\n",
    "    \n",
    "        print(\"####### Data Item: \",i+1,\" #######\")\n",
    "        target_label = 0 if data_sample.iloc[i]['sentiment'] == 1 else 1\n",
    "        current_prediction = data_sample.iloc[i]['probs']\n",
    "        \n",
    "        # calculate how long it takes us to process each data item (in seconds)\n",
    "        t0 = time.time()\n",
    "        \n",
    "        # generate a new population of possible solutions\n",
    "        if type_of_attack == 1:\n",
    "            p = generate_population_bert(saved_model, data_sample.iloc[i]['text'], POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "        if type_of_attack == 2:\n",
    "            p = generate_population_distance(saved_model, data_sample.iloc[i]['text'], POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "        if type_of_attack == 3:\n",
    "            p = generate_population_bert_and_distance(saved_model, data_sample.iloc[i]['text'], POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "    \n",
    "        population_dataframe = fitness(p, saved_model)\n",
    "\n",
    "    \n",
    "        ## GA Attack START\n",
    "        # need to run through and do crossover and mutation, recheck the label and if it has flipped then we stop, other wise keep going\n",
    "        # also on each iteration keep updating the ga_text and ga_prob and if label has changed update the ga_sentiment column and stop\n",
    "        for j in range(MAXIMUM_ITERATIONS):\n",
    "        \n",
    "        \n",
    "            # first check if we have found a solution, if yes then we are done so save results and break\n",
    "            # and move onto next\n",
    "            if found_solution(target_label, population_dataframe):\n",
    "                #save solution\n",
    "                if target_label == 1:   \n",
    "                    data_sample.at[i, \"ga_text\"] = population_dataframe.iloc[-1]['ga_text']\n",
    "                    data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[-1]['ga_probs']\n",
    "                    data_sample.at[i, \"ga_sentiment\"] = 1\n",
    "                    break\n",
    "                if target_label == 0:\n",
    "                    data_sample.at[i, \"ga_text\"] = population_dataframe.iloc[0]['ga_text']\n",
    "                    data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[0]['ga_probs']\n",
    "                    data_sample.at[i, \"ga_sentiment\"] = 0\n",
    "                    break\n",
    "        \n",
    "            # select the two best parents from the population\n",
    "            if target_label == 1:\n",
    "                parent_one = population_dataframe.iloc[-1]['ga_text']\n",
    "                parent_two = population_dataframe.iloc[-2]['ga_text']\n",
    "            else: \n",
    "                parent_one = population_dataframe.iloc[0]['ga_text']\n",
    "                parent_two = population_dataframe.iloc[1]['ga_text']\n",
    "        \n",
    "            # save progress so far\n",
    "            if target_label == 1 and (current_prediction < population_dataframe.iloc[-1]['ga_probs']):\n",
    "                data_sample.at[i, \"ga_text\"] = population_dataframe.iloc[-1]['ga_text']\n",
    "                data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[-1]['ga_probs']\n",
    "            \n",
    "            if target_label == 0 and (current_prediction > population_dataframe.iloc[0]['ga_probs']):\n",
    "                data_sample.at[i, \"ga_text\"]  = population_dataframe.iloc[0]['ga_text']\n",
    "                data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[0]['ga_probs']\n",
    "        \n",
    "            # we didn't find a solution yet, so we do crossover and then generate a new population of possible solutions \n",
    "            x_over = crossover(parent_one, parent_two)\n",
    "            \n",
    "            # generate a new population of possible solutions\n",
    "            if type_of_attack == 1:\n",
    "                p = generate_population_bert(saved_model, x_over, POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "            if type_of_attack == 2:\n",
    "                p = generate_population_distance(saved_model, x_over, POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "            if type_of_attack == 3:\n",
    "                p = generate_population_bert_and_distance(saved_model, x_over, POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "                \n",
    "            population_dataframe = fitness(p, saved_model)\n",
    "        \n",
    "        num_words_changes = number_of_changes_made(data_sample.at[i, \"text\"], data_sample.at[i, \"ga_text\"])\n",
    "        total_words_in_review = len(data_sample.at[i, \"text\"].split())\n",
    "        \n",
    "        print(\"\\tElapsed time: \", round(time.time() - t0), \" seconds\")\n",
    "        print(\"\\tNumber of words in review: \",total_words_in_review )\n",
    "        print(\"\\tNumber of words swapped: \", num_words_changes)\n",
    "        print(\"\\tPercentage modified: \", round((num_words_changes / total_words_in_review ),2) * 100,\"%\")\n",
    "        print(\"\\tProb. before and after: \", data_sample.at[i, \"probs\"],\" : \", data_sample.at[i, \"ga_probs\"],\"\\n\")\n",
    "        \n",
    "    return data_sample\n",
    "\n",
    "\n",
    "\n",
    "        \n",
    "def GA_Attack_Stats(data_sample, type_of_attack=2):\n",
    "    data_sample_len = len(data_sample)\n",
    "\n",
    "    t0 = 0\n",
    "    generation = 0\n",
    "    for i in range(data_sample_len):\n",
    "    \n",
    "        print(\"####### Data Item: \",i+1,\" #######\")\n",
    "        target_label = 0 if data_sample.iloc[i]['sentiment'] == 1 else 1\n",
    "        current_prediction = data_sample.iloc[i]['probs']\n",
    "        \n",
    "        t0 = 0\n",
    "        generation = 0\n",
    "        t0 = time.time()\n",
    "        \n",
    "        # generate a new population of possible solutions\n",
    "        if type_of_attack == 1:\n",
    "            p = generate_population_bert(saved_model, data_sample.iloc[i]['text'], POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "        if type_of_attack == 2:\n",
    "            p = generate_population_distance(saved_model, data_sample.iloc[i]['text'], POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "        if type_of_attack == 3:\n",
    "            p = generate_population_bert_and_distance(saved_model, data_sample.iloc[i]['text'], POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "    \n",
    "        population_dataframe = fitness(p, saved_model)\n",
    "        \n",
    "        # calculate how long it takes us to process each data item (in seconds)\n",
    "        \n",
    "    \n",
    "        ## GA Attack START\n",
    "        # need to run through and do crossover and mutation, recheck the label and if it has flipped then we stop, other wise keep going\n",
    "        # also on each iteration keep updating the ga_text and ga_prob and if label has changed update the ga_sentiment column and stop\n",
    "        for j in range(MAXIMUM_ITERATIONS):\n",
    "            \n",
    "        \n",
    "            # first check if we have found a solution, if yes then we are done so save results and break\n",
    "            # and move onto next\n",
    "            if found_solution(target_label, population_dataframe):\n",
    "                generation = j + 1\n",
    "                #save solution\n",
    "                if target_label == 1:\n",
    "                    data_sample.at[i,'ga_generations'] = generation\n",
    "                    data_sample.at[i,'ga_time_taken'] = round(time.time() - t0)\n",
    "                    data_sample.at[i, \"ga_text\"] = population_dataframe.iloc[-1]['ga_text']\n",
    "                    data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[-1]['ga_probs']\n",
    "                    data_sample.at[i, \"ga_sentiment\"] = 1\n",
    "                    break\n",
    "                if target_label == 0:\n",
    "                    data_sample.at[i,'ga_generations'] = generation\n",
    "                    data_sample.at[i,'ga_time_taken'] = round(time.time() - t0)\n",
    "                    data_sample.at[i, \"ga_text\"] = population_dataframe.iloc[0]['ga_text']\n",
    "                    data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[0]['ga_probs']\n",
    "                    data_sample.at[i, \"ga_sentiment\"] = 0\n",
    "                    break\n",
    "        \n",
    "            # select the two best parents from the population\n",
    "            if target_label == 1:\n",
    "                parent_one = population_dataframe.iloc[-1]['ga_text']\n",
    "                parent_two = population_dataframe.iloc[-2]['ga_text']\n",
    "            else: \n",
    "                parent_one = population_dataframe.iloc[0]['ga_text']\n",
    "                parent_two = population_dataframe.iloc[1]['ga_text']\n",
    "        \n",
    "            # save progress so far\n",
    "            if target_label == 1 and (current_prediction < population_dataframe.iloc[-1]['ga_probs']):\n",
    "                data_sample.at[i, \"ga_text\"] = population_dataframe.iloc[-1]['ga_text']\n",
    "                data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[-1]['ga_probs']\n",
    "            \n",
    "            if target_label == 0 and (current_prediction > population_dataframe.iloc[0]['ga_probs']):\n",
    "                data_sample.at[i, \"ga_text\"]  = population_dataframe.iloc[0]['ga_text']\n",
    "                data_sample.at[i, \"ga_probs\"] = population_dataframe.iloc[0]['ga_probs']\n",
    "        \n",
    "            # we didn't find a solution yet, so we do crossover and then generate a new population of possible solutions \n",
    "            x_over = crossover(parent_one, parent_two)\n",
    "            \n",
    "            # generate a new population of possible solutions\n",
    "            if type_of_attack == 1:\n",
    "                p = generate_population_bert(saved_model, x_over, POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "            if type_of_attack == 2:\n",
    "                p = generate_population_distance(saved_model, x_over, POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "            if type_of_attack == 3:\n",
    "                p = generate_population_bert_and_distance(saved_model, x_over, POPULATION_SIZE, current_prediction, target_label, MAX_PERTURBATIONS, MAX_NEIGHBOURS, saved_vocab)\n",
    "                \n",
    "            population_dataframe = fitness(p, saved_model)\n",
    "        \n",
    "        num_words_changes = number_of_changes_made(data_sample.at[i, \"text\"], data_sample.at[i, \"ga_text\"])\n",
    "        total_words_in_review = len(data_sample.at[i, \"text\"].split())\n",
    "        \n",
    "        print(\"\\tElapsed time: \", round(time.time() - t0), \" seconds\")\n",
    "        print(\"\\tGeneration: \", generation)\n",
    "        print(\"\\tNumber of words in review: \",total_words_in_review )\n",
    "        print(\"\\tNumber of words swapped: \", num_words_changes)\n",
    "        print(\"\\tPercentage modified: \", round((num_words_changes / total_words_in_review ),2) * 100,\"%\")\n",
    "        print(\"\\tProb. before and after: \", data_sample.at[i, \"probs\"],\" : \", data_sample.at[i, \"ga_probs\"],\"\\n\")\n",
    "        \n",
    "    return data_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 04) Prepare GA attack dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 750,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of data items in sample:  1006\n",
      "Reduce sample size to:  250\n",
      "Calculating probabilities for sample dataset...\n",
      "Checking proabilities for sample dataset are correctly labelled...\n",
      "Number of data items dropped from sample:  0\n",
      "Number of data items kept in sample:  250\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "      <th>ga_generations</th>\n",
       "      <th>ga_time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>a group of cats look to find their way home af...</td>\n",
       "      <td>0.991575</td>\n",
       "      <td>1</td>\n",
       "      <td>a group of cats look to find their way home af...</td>\n",
       "      <td>0.991575</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>tu pa tam is one of the worst movies i've ever...</td>\n",
       "      <td>0.000875</td>\n",
       "      <td>0</td>\n",
       "      <td>tu pa tam is one of the worst movies i've ever...</td>\n",
       "      <td>0.000875</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>being an elvis fan  i can't understand how thi...</td>\n",
       "      <td>0.008624</td>\n",
       "      <td>0</td>\n",
       "      <td>being an elvis fan  i can't understand how thi...</td>\n",
       "      <td>0.008624</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>i agree with the previous comment in naming th...</td>\n",
       "      <td>0.761566</td>\n",
       "      <td>1</td>\n",
       "      <td>i agree with the previous comment in naming th...</td>\n",
       "      <td>0.761566</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>sometimes a premise starts out good  but becau...</td>\n",
       "      <td>0.002127</td>\n",
       "      <td>0</td>\n",
       "      <td>sometimes a premise starts out good  but becau...</td>\n",
       "      <td>0.002127</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          1  a group of cats look to find their way home af...  0.991575   \n",
       "1          0  tu pa tam is one of the worst movies i've ever...  0.000875   \n",
       "2          0  being an elvis fan  i can't understand how thi...  0.008624   \n",
       "3          1  i agree with the previous comment in naming th...  0.761566   \n",
       "4          0  sometimes a premise starts out good  but becau...  0.002127   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             1  a group of cats look to find their way home af...  0.991575   \n",
       "1             0  tu pa tam is one of the worst movies i've ever...  0.000875   \n",
       "2             0  being an elvis fan  i can't understand how thi...  0.008624   \n",
       "3             1  i agree with the previous comment in naming th...  0.761566   \n",
       "4             0  sometimes a premise starts out good  but becau...  0.002127   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \\\n",
       "0               0           0.0                    N                0.0   \n",
       "1               0           0.0                    N                0.0   \n",
       "2               0           0.0                    N                0.0   \n",
       "3               0           0.0                    N                0.0   \n",
       "4               0           0.0                    N                0.0   \n",
       "\n",
       "   ga_generations  ga_time_taken  \n",
       "0               0              0  \n",
       "1               0              0  \n",
       "2               0              0  \n",
       "3               0              0  \n",
       "4               0              0  "
      ]
     },
     "execution_count": 750,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# data_sample = load_sample_dataset()\n",
    "# data_sample = add_ga_columns(data_sample)\n",
    "# data_sample = generate_current_probabilities(data_sample)\n",
    "# data_sample = check_predictions(data_sample)\n",
    "\n",
    "\n",
    "########\n",
    "data_sample = load_sample_dataset()\n",
    "\n",
    "# take a sample of 250 items\n",
    "data_sample = data_sample.sample(n = 250) \n",
    "print(\"Reduce sample size to: \",len(data_sample))\n",
    "\n",
    "data_sample = add_ga_columns_stats(data_sample)\n",
    "data_sample = generate_current_probabilities(data_sample)\n",
    "data_sample = check_predictions(data_sample)\n",
    "\n",
    "data_sample.head()\n",
    "\n",
    "\n",
    "###########\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 05) start the GA Attack"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# generate a sample dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 765,
   "metadata": {},
   "outputs": [],
   "source": [
    "SAMPLE_SIZE = 250"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 769,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of data items in sample:  1006\n",
      "Reduce sample size to:  250\n",
      "Calculating probabilities for sample dataset...\n",
      "Checking proabilities for sample dataset are correctly labelled...\n",
      "Number of data items dropped from sample:  0\n",
      "Number of data items kept in sample:  250\n",
      "Results saved to:\t ga_attack_results/sample_stats_set_1006.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "      <th>ga_generations</th>\n",
       "      <th>ga_time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>fragmentaric movie about a couple of people in...</td>\n",
       "      <td>0.025485</td>\n",
       "      <td>0</td>\n",
       "      <td>fragmentaric movie about a couple of people in...</td>\n",
       "      <td>0.025485</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>i rented this movie today thinking it might be...</td>\n",
       "      <td>0.038891</td>\n",
       "      <td>0</td>\n",
       "      <td>i rented this movie today thinking it might be...</td>\n",
       "      <td>0.038891</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>this is a great compendium of interviews and e...</td>\n",
       "      <td>0.970709</td>\n",
       "      <td>1</td>\n",
       "      <td>this is a great compendium of interviews and e...</td>\n",
       "      <td>0.970709</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>with the badly injured tony in an induced coma...</td>\n",
       "      <td>0.785037</td>\n",
       "      <td>1</td>\n",
       "      <td>with the badly injured tony in an induced coma...</td>\n",
       "      <td>0.785037</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>a new and innovative show with a great cast th...</td>\n",
       "      <td>0.952263</td>\n",
       "      <td>1</td>\n",
       "      <td>a new and innovative show with a great cast th...</td>\n",
       "      <td>0.952263</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          0  fragmentaric movie about a couple of people in...  0.025485   \n",
       "1          0  i rented this movie today thinking it might be...  0.038891   \n",
       "2          1  this is a great compendium of interviews and e...  0.970709   \n",
       "3          1  with the badly injured tony in an induced coma...  0.785037   \n",
       "4          1  a new and innovative show with a great cast th...  0.952263   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             0  fragmentaric movie about a couple of people in...  0.025485   \n",
       "1             0  i rented this movie today thinking it might be...  0.038891   \n",
       "2             1  this is a great compendium of interviews and e...  0.970709   \n",
       "3             1  with the badly injured tony in an induced coma...  0.785037   \n",
       "4             1  a new and innovative show with a great cast th...  0.952263   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \\\n",
       "0               0           0.0                    N                0.0   \n",
       "1               0           0.0                    N                0.0   \n",
       "2               0           0.0                    N                0.0   \n",
       "3               0           0.0                    N                0.0   \n",
       "4               0           0.0                    N                0.0   \n",
       "\n",
       "   ga_generations  ga_time_taken  \n",
       "0               0              0  \n",
       "1               0              0  \n",
       "2               0              0  \n",
       "3               0              0  \n",
       "4               0              0  "
      ]
     },
     "execution_count": 769,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SAMPLE_SIZE = 250\n",
    "\n",
    "data_sample = load_sample_dataset()\n",
    "\n",
    "# take a sample of 250 items\n",
    "data_sample = data_sample.sample(n = 250)\n",
    "print(\"Reduce sample size to: \",len(data_sample))\n",
    "\n",
    "# add the additional columns to our datafile - these will be used to store our results stats\n",
    "data_sample = add_ga_columns_stats(data_sample)\n",
    "data_sample = generate_current_probabilities(data_sample)\n",
    "data_sample = check_predictions(data_sample)\n",
    "\n",
    "# save this sampple so we can re-use the same sample for all three methods\n",
    "save_ga_attack_results(data_sample, 'sample_stats_set_'+str(SAMPLE_SIZE)+'.csv')\n",
    "\n",
    "data_sample.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 770,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "      <th>ga_generations</th>\n",
       "      <th>ga_time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>fragmentaric movie about a couple of people in...</td>\n",
       "      <td>0.025485</td>\n",
       "      <td>0</td>\n",
       "      <td>fragmentaric movie about a couple of people in...</td>\n",
       "      <td>0.025485</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>i rented this movie today thinking it might be...</td>\n",
       "      <td>0.038891</td>\n",
       "      <td>0</td>\n",
       "      <td>i rented this movie today thinking it might be...</td>\n",
       "      <td>0.038891</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>this is a great compendium of interviews and e...</td>\n",
       "      <td>0.970709</td>\n",
       "      <td>1</td>\n",
       "      <td>this is a great compendium of interviews and e...</td>\n",
       "      <td>0.970709</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>with the badly injured tony in an induced coma...</td>\n",
       "      <td>0.785037</td>\n",
       "      <td>1</td>\n",
       "      <td>with the badly injured tony in an induced coma...</td>\n",
       "      <td>0.785037</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>a new and innovative show with a great cast th...</td>\n",
       "      <td>0.952263</td>\n",
       "      <td>1</td>\n",
       "      <td>a new and innovative show with a great cast th...</td>\n",
       "      <td>0.952263</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          0  fragmentaric movie about a couple of people in...  0.025485   \n",
       "1          0  i rented this movie today thinking it might be...  0.038891   \n",
       "2          1  this is a great compendium of interviews and e...  0.970709   \n",
       "3          1  with the badly injured tony in an induced coma...  0.785037   \n",
       "4          1  a new and innovative show with a great cast th...  0.952263   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             0  fragmentaric movie about a couple of people in...  0.025485   \n",
       "1             0  i rented this movie today thinking it might be...  0.038891   \n",
       "2             1  this is a great compendium of interviews and e...  0.970709   \n",
       "3             1  with the badly injured tony in an induced coma...  0.785037   \n",
       "4             1  a new and innovative show with a great cast th...  0.952263   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \\\n",
       "0               0           0.0                    N                0.0   \n",
       "1               0           0.0                    N                0.0   \n",
       "2               0           0.0                    N                0.0   \n",
       "3               0           0.0                    N                0.0   \n",
       "4               0           0.0                    N                0.0   \n",
       "\n",
       "   ga_generations  ga_time_taken  \n",
       "0               0              0  \n",
       "1               0              0  \n",
       "2               0              0  \n",
       "3               0              0  \n",
       "4               0              0  "
      ]
     },
     "execution_count": 770,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# load it back just to check it saved correctly\n",
    "data_sample = load_ga_attack_results('sample_stats_set_'+str(SAMPLE_SIZE)+'.csv')\n",
    "\n",
    "#data_sample = pd.read_csv('ga_attack_results/sample_stats_set_250.csv')\n",
    "\n",
    "data_sample.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### run the GA_Attack sample set (set ```TYPE_OF_ATTACK``` in turn for each method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 774,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "''' TYPE_OF_ATTACK:\n",
    "    1 == BERT i.e. using context to generate replacement words\n",
    "    2 == Distance Matrix i.e. using only nearest neighbour - synonyms to generate replacement words\n",
    "    3 == Distance Matric and BERT sentence score i.e. find synonyms and then use sentence structure to score partial sentence for each replacement word\n",
    "'''\n",
    "# change this as we run each method\n",
    "TYPE_OF_ATTACK = 1\n",
    "\n",
    "data_sample = load_ga_attack_results('sample_stats_set_250.csv')\n",
    "\n",
    "GA_Attack_Stats(data_sample, type_of_attack=TYPE_OF_ATTACK)\n",
    "\n",
    "data_sample.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# save stats sample data_Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 746,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results saved to:\t ga_attack_results/ga_results_DISTANCE_BERT_STATS.csv\n",
      "loaded file:  ga_results_DISTANCE_BERT.csv\n"
     ]
    }
   ],
   "source": [
    "ga_stats_results_BERT = \"ga_results_BERT_STATS.csv\"\n",
    "ga_stats_results_DISTANCE = \"ga_results_DISTANCE_STATS.csv\"\n",
    "ga_stats_results_DISTANCE_BERT = \"ga_results_DISTANCE_BERT_STATS.csv\"\n",
    "\n",
    "if TYPE_OF_ATTACK == 1:        \n",
    "    save_ga_attack_results(data_sample, ga_stats_results_BERT)\n",
    "if TYPE_OF_ATTACK == 2:\n",
    "    save_ga_attack_results(data_sample, ga_stats_results_DISTANCE)\n",
    "if TYPE_OF_ATTACK == 3:\n",
    "    save_ga_attack_results(data_sample, ga_stats_results_DISTANCE_BERT)\n",
    "    \n",
    "if TYPE_OF_ATTACK == 1:\n",
    "    data_sample = load_ga_attack_results(ga_stats_results_BERT)\n",
    "    print(\"loaded file: \", ga_stats_results_BERT)\n",
    "if TYPE_OF_ATTACK == 2:\n",
    "    data_sample = load_ga_attack_results(ga_stats_results_DISTANCE)\n",
    "    print(\"loaded file: \", ga_stats_results_DISTANCE)\n",
    "if TYPE_OF_ATTACK == 3:\n",
    "    data_sample = load_ga_attack_results(ga_stats_results_DISTANCE_BERT)\n",
    "    print(\"loaded file: \",ga_results_DISTANCE_BERT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 747,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "      <th>ga_generations</th>\n",
       "      <th>ga_time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>well i must say this is probably the worst fil...</td>\n",
       "      <td>0.025771</td>\n",
       "      <td>1</td>\n",
       "      <td>well i must say this is probably the meanest f...</td>\n",
       "      <td>0.768538</td>\n",
       "      <td>20</td>\n",
       "      <td>0.895522</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.13</td>\n",
       "      <td>5</td>\n",
       "      <td>264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>if christopher nolan had made memento before f...</td>\n",
       "      <td>0.769358</td>\n",
       "      <td>0</td>\n",
       "      <td>if christopher nolan had made memento before f...</td>\n",
       "      <td>0.395876</td>\n",
       "      <td>10</td>\n",
       "      <td>0.966031</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.03</td>\n",
       "      <td>2</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>i'm not saying anything new when i say that ra...</td>\n",
       "      <td>0.807952</td>\n",
       "      <td>0</td>\n",
       "      <td>i'm not saying somethings newer when i say tha...</td>\n",
       "      <td>0.426562</td>\n",
       "      <td>10</td>\n",
       "      <td>0.958418</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.05</td>\n",
       "      <td>2</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>i have been watching lost with my family since...</td>\n",
       "      <td>0.476023</td>\n",
       "      <td>1</td>\n",
       "      <td>i have been watching lost with my family since...</td>\n",
       "      <td>0.637239</td>\n",
       "      <td>5</td>\n",
       "      <td>0.978884</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>caught this flick as one of a fivefor deal fro...</td>\n",
       "      <td>0.942861</td>\n",
       "      <td>0</td>\n",
       "      <td>caught this flick as anyone of a fivefor addre...</td>\n",
       "      <td>0.294813</td>\n",
       "      <td>29</td>\n",
       "      <td>0.836257</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.19</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          0  well i must say this is probably the worst fil...  0.025771   \n",
       "1          1  if christopher nolan had made memento before f...  0.769358   \n",
       "2          1  i'm not saying anything new when i say that ra...  0.807952   \n",
       "3          0  i have been watching lost with my family since...  0.476023   \n",
       "4          1  caught this flick as one of a fivefor deal fro...  0.942861   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             1  well i must say this is probably the meanest f...  0.768538   \n",
       "1             0  if christopher nolan had made memento before f...  0.395876   \n",
       "2             0  i'm not saying somethings newer when i say tha...  0.426562   \n",
       "3             1  i have been watching lost with my family since...  0.637239   \n",
       "4             0  caught this flick as anyone of a fivefor addre...  0.294813   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \\\n",
       "0              20      0.895522                    Y               0.13   \n",
       "1              10      0.966031                    Y               0.03   \n",
       "2              10      0.958418                    Y               0.05   \n",
       "3               5      0.978884                    Y               0.01   \n",
       "4              29      0.836257                    Y               0.19   \n",
       "\n",
       "   ga_generations  ga_time_taken  \n",
       "0               5            264  \n",
       "1               2            107  \n",
       "2               2            106  \n",
       "3               1             53  \n",
       "4               5            256  "
      ]
     },
     "execution_count": 747,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load results file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 772,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded BERT results file\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>playwright sidney bruhl michael caine has had ...</td>\n",
       "      <td>0.978781</td>\n",
       "      <td>0</td>\n",
       "      <td>playwright sidney bruhl michael jordan has had...</td>\n",
       "      <td>0.149355</td>\n",
       "      <td>14</td>\n",
       "      <td>0.940671</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>yikes did this movie blow  the characters were...</td>\n",
       "      <td>0.007015</td>\n",
       "      <td>1</td>\n",
       "      <td>yikes did this movie develop the characters we...</td>\n",
       "      <td>0.767355</td>\n",
       "      <td>16</td>\n",
       "      <td>0.866159</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>this would have to be one of the worst  if not...</td>\n",
       "      <td>0.016789</td>\n",
       "      <td>1</td>\n",
       "      <td>this would have to be one of the best if not t...</td>\n",
       "      <td>0.663885</td>\n",
       "      <td>8</td>\n",
       "      <td>0.944520</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>a brilliant sherlock holmes adventure starring...</td>\n",
       "      <td>0.999339</td>\n",
       "      <td>0</td>\n",
       "      <td>a brilliant sherlock book club starring the br...</td>\n",
       "      <td>0.362426</td>\n",
       "      <td>22</td>\n",
       "      <td>0.968952</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>first  i must say that i don't speak spanish a...</td>\n",
       "      <td>0.906772</td>\n",
       "      <td>0</td>\n",
       "      <td>first i must say that i don't like spanish and...</td>\n",
       "      <td>0.249828</td>\n",
       "      <td>8</td>\n",
       "      <td>0.914607</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          1  playwright sidney bruhl michael caine has had ...  0.978781   \n",
       "1          0  yikes did this movie blow  the characters were...  0.007015   \n",
       "2          0  this would have to be one of the worst  if not...  0.016789   \n",
       "3          1  a brilliant sherlock holmes adventure starring...  0.999339   \n",
       "4          1  first  i must say that i don't speak spanish a...  0.906772   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             0  playwright sidney bruhl michael jordan has had...  0.149355   \n",
       "1             1  yikes did this movie develop the characters we...  0.767355   \n",
       "2             1  this would have to be one of the best if not t...  0.663885   \n",
       "3             0  a brilliant sherlock book club starring the br...  0.362426   \n",
       "4             0  first i must say that i don't like spanish and...  0.249828   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \n",
       "0              14      0.940671                    Y               0.06  \n",
       "1              16      0.866159                    Y               0.19  \n",
       "2               8      0.944520                    Y               0.06  \n",
       "3              22      0.968952                    Y               0.03  \n",
       "4               8      0.914607                    Y               0.10  "
      ]
     },
     "execution_count": 772,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "def load_results_datafile():\n",
    "    if TYPE_OF_ATTACK == 1:\n",
    "        data_sample = load_ga_attack_results(ga_results_BERT)\n",
    "        print(\"loaded BERT results file\")\n",
    "    if TYPE_OF_ATTACK == 2:\n",
    "        data_sample = load_ga_attack_results(ga_results_DISTANCE)\n",
    "        print(\"loaded DISTANCE MATRIX results file\")\n",
    "    if TYPE_OF_ATTACK == 3:\n",
    "        data_sample = load_ga_attack_results(ga_results_DISTANCE_BERT)\n",
    "        print(\"loaded DISTANCE MATRIX results file: \",ga_results_DISTANCE_BERT)\n",
    "    \n",
    "    return data_sample\n",
    "\n",
    "data_sample = load_results_datafile()\n",
    "data_sample.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 06)  Calculate stats after the attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 748,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "      <th>ga_generations</th>\n",
       "      <th>ga_time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>well i must say this is probably the worst fil...</td>\n",
       "      <td>0.025771</td>\n",
       "      <td>1</td>\n",
       "      <td>well i must say this is probably the meanest f...</td>\n",
       "      <td>0.768538</td>\n",
       "      <td>20</td>\n",
       "      <td>0.895522</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.13</td>\n",
       "      <td>5</td>\n",
       "      <td>264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>if christopher nolan had made memento before f...</td>\n",
       "      <td>0.769358</td>\n",
       "      <td>0</td>\n",
       "      <td>if christopher nolan had made memento before f...</td>\n",
       "      <td>0.395876</td>\n",
       "      <td>10</td>\n",
       "      <td>0.966031</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.03</td>\n",
       "      <td>2</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>i'm not saying anything new when i say that ra...</td>\n",
       "      <td>0.807952</td>\n",
       "      <td>0</td>\n",
       "      <td>i'm not saying somethings newer when i say tha...</td>\n",
       "      <td>0.426562</td>\n",
       "      <td>10</td>\n",
       "      <td>0.958418</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.05</td>\n",
       "      <td>2</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>i have been watching lost with my family since...</td>\n",
       "      <td>0.476023</td>\n",
       "      <td>1</td>\n",
       "      <td>i have been watching lost with my family since...</td>\n",
       "      <td>0.637239</td>\n",
       "      <td>5</td>\n",
       "      <td>0.978884</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.01</td>\n",
       "      <td>1</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>caught this flick as one of a fivefor deal fro...</td>\n",
       "      <td>0.942861</td>\n",
       "      <td>0</td>\n",
       "      <td>caught this flick as anyone of a fivefor addre...</td>\n",
       "      <td>0.294813</td>\n",
       "      <td>29</td>\n",
       "      <td>0.836257</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.19</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          0  well i must say this is probably the worst fil...  0.025771   \n",
       "1          1  if christopher nolan had made memento before f...  0.769358   \n",
       "2          1  i'm not saying anything new when i say that ra...  0.807952   \n",
       "3          0  i have been watching lost with my family since...  0.476023   \n",
       "4          1  caught this flick as one of a fivefor deal fro...  0.942861   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             1  well i must say this is probably the meanest f...  0.768538   \n",
       "1             0  if christopher nolan had made memento before f...  0.395876   \n",
       "2             0  i'm not saying somethings newer when i say tha...  0.426562   \n",
       "3             1  i have been watching lost with my family since...  0.637239   \n",
       "4             0  caught this flick as anyone of a fivefor addre...  0.294813   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \\\n",
       "0              20      0.895522                    Y               0.13   \n",
       "1              10      0.966031                    Y               0.03   \n",
       "2              10      0.958418                    Y               0.05   \n",
       "3               5      0.978884                    Y               0.01   \n",
       "4              29      0.836257                    Y               0.19   \n",
       "\n",
       "   ga_generations  ga_time_taken  \n",
       "0               5            264  \n",
       "1               2            107  \n",
       "2               2            106  \n",
       "3               1             53  \n",
       "4               5            256  "
      ]
     },
     "execution_count": 748,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample = calculate_levenshtein_ratios(data_sample)\n",
    "data_sample.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### number of test data items processed during GA Attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 742,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of data items processed in GA Attack:  250\n",
      "No words were removed from original text.\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of data items processed in GA Attack: \", len(data_sample))\n",
    "if deleted_words_from_text(data_sample):\n",
    "    print(\"Some words were removed from original text!\")\n",
    "else:\n",
    "    print(\"No words were removed from original text.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### check for which reviews we successfully flipped the sentiment and set ```ga_flipped_sentiment``` == 'Y' if we were successful"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 743,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of reviews where sentiment was changed after attack: 53.2 % changed sentiment. i.e. 133  out of  250\n",
      "Percentage of reviews failed to change sentiment:  46.8 % did not change, i.e. 117  out of  250\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "      <th>ga_generations</th>\n",
       "      <th>ga_time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>well i must say this is probably the worst fil...</td>\n",
       "      <td>0.025771</td>\n",
       "      <td>1</td>\n",
       "      <td>well i must say this is probably the meanest f...</td>\n",
       "      <td>0.768538</td>\n",
       "      <td>0</td>\n",
       "      <td>0.895522</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.129032</td>\n",
       "      <td>5</td>\n",
       "      <td>264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>if christopher nolan had made memento before f...</td>\n",
       "      <td>0.769358</td>\n",
       "      <td>0</td>\n",
       "      <td>if christopher nolan had made memento before f...</td>\n",
       "      <td>0.395876</td>\n",
       "      <td>0</td>\n",
       "      <td>0.966031</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.032895</td>\n",
       "      <td>2</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>i'm not saying anything new when i say that ra...</td>\n",
       "      <td>0.807952</td>\n",
       "      <td>0</td>\n",
       "      <td>i'm not saying somethings newer when i say tha...</td>\n",
       "      <td>0.426562</td>\n",
       "      <td>0</td>\n",
       "      <td>0.958418</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.053476</td>\n",
       "      <td>2</td>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>i have been watching lost with my family since...</td>\n",
       "      <td>0.476023</td>\n",
       "      <td>1</td>\n",
       "      <td>i have been watching lost with my family since...</td>\n",
       "      <td>0.637239</td>\n",
       "      <td>0</td>\n",
       "      <td>0.978884</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.013514</td>\n",
       "      <td>1</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>caught this flick as one of a fivefor deal fro...</td>\n",
       "      <td>0.942861</td>\n",
       "      <td>0</td>\n",
       "      <td>caught this flick as anyone of a fivefor addre...</td>\n",
       "      <td>0.294813</td>\n",
       "      <td>0</td>\n",
       "      <td>0.836257</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.189542</td>\n",
       "      <td>5</td>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          0  well i must say this is probably the worst fil...  0.025771   \n",
       "1          1  if christopher nolan had made memento before f...  0.769358   \n",
       "2          1  i'm not saying anything new when i say that ra...  0.807952   \n",
       "3          0  i have been watching lost with my family since...  0.476023   \n",
       "4          1  caught this flick as one of a fivefor deal fro...  0.942861   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             1  well i must say this is probably the meanest f...  0.768538   \n",
       "1             0  if christopher nolan had made memento before f...  0.395876   \n",
       "2             0  i'm not saying somethings newer when i say tha...  0.426562   \n",
       "3             1  i have been watching lost with my family since...  0.637239   \n",
       "4             0  caught this flick as anyone of a fivefor addre...  0.294813   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \\\n",
       "0               0      0.895522                    Y           0.129032   \n",
       "1               0      0.966031                    Y           0.032895   \n",
       "2               0      0.958418                    Y           0.053476   \n",
       "3               0      0.978884                    Y           0.013514   \n",
       "4               0      0.836257                    Y           0.189542   \n",
       "\n",
       "   ga_generations  ga_time_taken  \n",
       "0               5            264  \n",
       "1               2            107  \n",
       "2               2            106  \n",
       "3               1             53  \n",
       "4               5            256  "
      ]
     },
     "execution_count": 743,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample = set_labels_changed(data_sample)\n",
    "data_sample.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### count how many words were changed for each review, average number of word changed and percentage of change to review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 744,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Avg. num of words changed changes made:  35\n",
      "Avg. percentage modified:  18.0 %\n"
     ]
    }
   ],
   "source": [
    "data_sample = set_percentage_modified(data_sample)\n",
    "\n",
    "#save_all_results(data_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 773,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>probs</th>\n",
       "      <th>ga_sentiment</th>\n",
       "      <th>ga_text</th>\n",
       "      <th>ga_probs</th>\n",
       "      <th>ga_num_changes</th>\n",
       "      <th>ga_lev_ratio</th>\n",
       "      <th>ga_flipped_sentiment</th>\n",
       "      <th>ga_percent_change</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>playwright sidney bruhl michael caine has had ...</td>\n",
       "      <td>0.978781</td>\n",
       "      <td>0</td>\n",
       "      <td>playwright sidney bruhl michael jordan has had...</td>\n",
       "      <td>0.149355</td>\n",
       "      <td>14</td>\n",
       "      <td>0.940671</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>yikes did this movie blow  the characters were...</td>\n",
       "      <td>0.007015</td>\n",
       "      <td>1</td>\n",
       "      <td>yikes did this movie develop the characters we...</td>\n",
       "      <td>0.767355</td>\n",
       "      <td>16</td>\n",
       "      <td>0.866159</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>this would have to be one of the worst  if not...</td>\n",
       "      <td>0.016789</td>\n",
       "      <td>1</td>\n",
       "      <td>this would have to be one of the best if not t...</td>\n",
       "      <td>0.663885</td>\n",
       "      <td>8</td>\n",
       "      <td>0.944520</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>a brilliant sherlock holmes adventure starring...</td>\n",
       "      <td>0.999339</td>\n",
       "      <td>0</td>\n",
       "      <td>a brilliant sherlock book club starring the br...</td>\n",
       "      <td>0.362426</td>\n",
       "      <td>22</td>\n",
       "      <td>0.968952</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>first  i must say that i don't speak spanish a...</td>\n",
       "      <td>0.906772</td>\n",
       "      <td>0</td>\n",
       "      <td>first i must say that i don't like spanish and...</td>\n",
       "      <td>0.249828</td>\n",
       "      <td>8</td>\n",
       "      <td>0.914607</td>\n",
       "      <td>Y</td>\n",
       "      <td>0.10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                               text     probs  \\\n",
       "0          1  playwright sidney bruhl michael caine has had ...  0.978781   \n",
       "1          0  yikes did this movie blow  the characters were...  0.007015   \n",
       "2          0  this would have to be one of the worst  if not...  0.016789   \n",
       "3          1  a brilliant sherlock holmes adventure starring...  0.999339   \n",
       "4          1  first  i must say that i don't speak spanish a...  0.906772   \n",
       "\n",
       "   ga_sentiment                                            ga_text  ga_probs  \\\n",
       "0             0  playwright sidney bruhl michael jordan has had...  0.149355   \n",
       "1             1  yikes did this movie develop the characters we...  0.767355   \n",
       "2             1  this would have to be one of the best if not t...  0.663885   \n",
       "3             0  a brilliant sherlock book club starring the br...  0.362426   \n",
       "4             0  first i must say that i don't like spanish and...  0.249828   \n",
       "\n",
       "   ga_num_changes  ga_lev_ratio ga_flipped_sentiment  ga_percent_change  \n",
       "0              14      0.940671                    Y               0.06  \n",
       "1              16      0.866159                    Y               0.19  \n",
       "2               8      0.944520                    Y               0.06  \n",
       "3              22      0.968952                    Y               0.03  \n",
       "4               8      0.914607                    Y               0.10  "
      ]
     },
     "execution_count": 773,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_sample.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}