gen_test_vec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3bc7738f-aca9-47ab-95ad-aae54a24753a",
   "metadata": {},
   "source": [
    "# DAT480 Grp 1 pattern matching preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "90b19a46-aa3f-40dd-bacb-346926d7ff37",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt \n",
    "import subprocess\n",
    "import os\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import re\n",
    "import string\n",
    "import random\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8551b41-dd72-4355-88fd-1bfc0977ad13",
   "metadata": {},
   "source": [
    "## Initalize files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d2371466-a22c-4eb3-9d62-ecc9efb489bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"/home/jonathan/Documents/Chalmers/Year5/DAT480/Lab_Project/DAT480/Rules_div_by_length/\"\n",
    "r_hashed = open(\"/home/jonathan/Documents/Chalmers/Year5/DAT480/Lab_Project/pattern_match_snort3_content_hashed.txt\")\n",
    "r_length = open(\"/home/jonathan/Documents/Chalmers/Year5/DAT480/Lab_Project/pattern_match_snort3_content_length.txt\")\n",
    "r_text = open(\"/home/jonathan/Documents/Chalmers/Year5/DAT480/Lab_Project/pattern_match_snort3_content.txt\")\n",
    "t_hashed = r_hashed.read().splitlines(False)\n",
    "t_hashed = np.array(t_hashed, dtype = str)\n",
    "t_length = r_length.read().splitlines(False)\n",
    "t_length = np.array(t_length, dtype=int)\n",
    "t_text = r_text.read().splitlines(False)\n",
    "t_text = np.array(t_text, dtype=str)\n",
    "r_hashed.close()\n",
    "r_length.close()\n",
    "\n",
    "rules_hashed = np.array([t_length, t_hashed, t_text]).T\n",
    "rules_len_sort = rules_hashed[np.argsort(t_length,axis=0)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "674185ad-5325-473c-8a4e-1a56386f39f1",
   "metadata": {},
   "source": [
    "## Create generic arrays to use for HLS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2f461521-6ccb-40ba-92a5-eab0faa0891e",
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = (np.array(np.unique(rules_len_sort[:,0],return_counts=True),dtype=int))\n",
    "indices=np.argsort(arr[0])\n",
    "#specify the unique_lengths you would like to use, 3,8 will yield the lengths 4,5,6,7,8\n",
    "lengths_to_use = range(0,106)\n",
    "sorted_arr = np.array([arr[0][indices], arr[1][indices]])\n",
    "\n",
    "#write the number of elements per unique length in the order of the lengths\n",
    "f = open(path+\"elements.h\",\"w+\")\n",
    "f.write(\"const uint16_t elements [] = {\\n\")\n",
    "for element in sorted_arr[1][lengths_to_use]:\n",
    "    f.write(str(element)+\",\\n\")\n",
    "f.write(\"};\")\n",
    "\n",
    "#write the unique lengths in order\n",
    "f = open(path+\"lengths.h\",\"w+\")\n",
    "f.write(\"const uint16_t lengths [] = {\\n\")\n",
    "for k in sorted_arr[0][lengths_to_use]:\n",
    "    f.write(str(k)+\",\\n\")\n",
    "f.write(\"};\")\n",
    "f.close()    \n",
    "\n",
    "f = open(path+\"ruleset.h\", \"w+\")\n",
    "f.write(\"const uint32_t rules [] = {\\n\")\n",
    "for rule in rules_len_sort[(rules_len_sort[:,0].astype(int) >= lengths_to_use[0]+1) & (rules_len_sort[:,0].astype(int) <len(lengths_to_use)+lengths_to_use[0]+1)][:,1]:\n",
    "    f.write(\"0x\"+str(rule)+\",\\n\")\n",
    "f.write(\"};\")\n",
    "f.close()\n",
    "special_chars = [\"%\",\"\\\\\",\"\\\"\",\";\",\"'\"]\n",
    "f = open(path+\"str_ruleset.h\", \"w+\")\n",
    "curr_len = 0\n",
    "for length in sorted_arr[0][lengths_to_use]:\n",
    "    f.write(\"const char str_rules_\"+str(length)+ \"[][\"+str(length)+\"] = {\\n\")\n",
    "    for el in rules_len_sort[rules_len_sort[:,0] == str(length)]:\n",
    "        f.write(\"{\")\n",
    "        for byte in el[2]:\n",
    "            if(np.isin(byte, special_chars)):\n",
    "                byte = \"\\\\\"+byte\n",
    "            f.write(\"'\"+str(byte)+\"',\")\n",
    "        f.write(\"},\\n\")\n",
    "    f.write(\"};\\n\")\n",
    "f.close()\n",
    "\n",
    "\n",
    "f = open(path+\"switch.h\", \"w+\")\n",
    "switch_initalize = \"#define LONG_SWITCH switch (curr_max_len){\"\n",
    "switch_close = \"default: continue;}\"\n",
    "f.write(switch_initalize)\n",
    "for length in sorted_arr[0][lengths_to_use]:\n",
    "    case_body = \"case \"+str(length)+\": for(int i = 0; i<\"+str(length)+\"; i++){if(stream_mem[BUFFER_WIDTH-1][head-i] != str_rules_\"+str(length)+\"[curr_idx][i]){match = 1;}}break;\"\n",
    "    f.write(case_body)\n",
    "f.write(switch_close)\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35f56852-23f7-4eec-ab5d-55ce1095a680",
   "metadata": {},
   "source": [
    "## Create test vectors for the testbench, i.e generate input reference and golden reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "8ddc58c6-dbc8-4dbe-a85e-01f2ae173c64",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2320/2320 [00:06<00:00, 335.73it/s]\n"
     ]
    }
   ],
   "source": [
    "number_of_rules_to_test = 100\n",
    "number = 0\n",
    "lengths = sorted_arr[0]\n",
    "input_vec = open(path+\"test_vec_input.txt\",\"w+\")\n",
    "gold_vec = open(path+\"test_vec_gold.txt\",\"w+\")\n",
    "input_str = \"\"\n",
    "#mix randomness with rules from the ruleset\n",
    "for i in range(1,number_of_rules_to_test*3):\n",
    "    if(i%3 != 0):\n",
    "        length = random.randrange(1, 20)\n",
    "        randomstr = ''.join(random.choices(string.ascii_letters+string.digits,k=length))\n",
    "        input_vec.write(randomstr)\n",
    "        input_str += randomstr\n",
    "\n",
    "    else:\n",
    "        number = random.randrange(lengths_to_use[0],len(lengths_to_use))\n",
    "        text = rules_len_sort[number][2]\n",
    "        input_str += text \n",
    "        input_vec.write(text)\n",
    "        \n",
    "input_vec.close()\n",
    "cmp_vec = rules_len_sort[(rules_len_sort[:,0].astype(int) >= lengths_to_use[0]+1) & (rules_len_sort[:,0].astype(int) <lengths[len(lengths_to_use)-1]+lengths_to_use[0]+1)] #incorrect \n",
    "for i in tqdm(range(len(input_str))):\n",
    "    max_length = -1\n",
    "    curr_element = []\n",
    "    for length in lengths[lengths_to_use]:\n",
    "        res = np.where(rules_len_sort[:,2] == input_str[i:i+length])[0]\n",
    "        if(res.size != 0 and max_length < length):\n",
    "            max_length = length\n",
    "            curr_element = res[0]\n",
    "    if(curr_element):\n",
    "        gold_vec.write(str(curr_element)+\"\\n\")\n",
    "\n",
    "gold_vec.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}