demo/builtin/dlrm/DLRM_DEMO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e8962dd0",
   "metadata": {},
   "source": [
    "[![Open in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/intel/e2eAIOK/blob/main/demo/builtin/dlrm/DLRM_DEMO.ipynb)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d151072",
   "metadata": {},
   "source": [
    "# DLRM DEMO\n",
    "\n",
    "Deep Learning Recommendation Model for Personalization and Recommendation Systems\n",
    "\n",
    "* original source\n",
    "    * Source repo: https://github.com/facebookresearch/dlrm\n",
    "\n",
    "\n",
    "# Content\n",
    "* [Overview](#Overview)\n",
    "    * [Model Architecture](#Model-Architecture)\n",
    "    * [Optimizations](#Optimizations)\n",
    "    * [Performance](#Performance)\n",
    "* [Getting Started](#Getting-Started)\n",
    "    * [1. Environment Setup](#1.-Environment-Setup)\n",
    "    * [2. Workflow Prepare](#2.-Workflow-Prepare)\n",
    "    * [3. Data Prepare](#3.-Data-Prepare)\n",
    "    * [4. Train](#4.-Train)\n",
    "\n",
    "------\n",
    "\n",
    "# Overview\n",
    "\n",
    "## Model Architecture\n",
    "\n",
    "<img src=\"./img/dlrm.png\" alt=\"DLRM network\" style=\"width: 400px;\"/>\n",
    "\n",
    "\n",
    "## Optimizations\n",
    "\n",
    "### New Imports\n",
    "``` diff\n",
    "+ import dlrm_data_pytorch as dp\n",
    "+ from lamb_bin import Lamb, log_lamb_rs\n",
    "+ # For distributed run\n",
    "+ import extend_distributed as ext_dist\n",
    "+ try:\n",
    "+     import intel_pytorch_extension as ipex\n",
    "+     from intel_pytorch_extension import core\n",
    "+ except:\n",
    "+     pass\n",
    "\n",
    "- import torch.nn.functional as F\n",
    "- import extend_distributed as ext_dist\n",
    "```\n",
    "\n",
    "### MLP layer democratization\n",
    "\n",
    "``` diff\n",
    "def create_mlp(self, ln, sigmoid_layer):\n",
    "    # build MLP layer by layer\n",
    "    layers = nn.ModuleList()\n",
    "    for i in range(0, ln.size - 1):\n",
    "        n = ln[i]\n",
    "        m = ln[i + 1]\n",
    "\n",
    "        # construct fully connected operator\n",
    "+           LL = ipex.IpexMLPLinear(int(n), int(m), bias=True, output_stays_blocked=(i < ln.size - 2), default_blocking=32)\n",
    "-           LL = nn.Linear(int(n), int(m), bias=True)\n",
    "\n",
    "        ...\n",
    "+           LL.to(torch.bfloat16)\n",
    "+           if hasattr(LL, 'reset_weight_shape'):\n",
    "+               LL.reset_weight_shape(block_for_dtype=torch.bfloat16)\n",
    "        ...\n",
    "```\n",
    "\n",
    "### Embedding layer democratization\n",
    "\n",
    "``` diff\n",
    "def create_emb(self, m, ln, local_ln_emb_sparse=None, ln_emb_dense=None):\n",
    "    emb_l = nn.ModuleList()\n",
    "+   # save the numpy random state\n",
    "+   np_rand_state = np.random.get_state()\n",
    "     ...\n",
    "    for i in embs:\n",
    "         ...\n",
    "        # construct embedding operator, original all embedding are in same dimension            \n",
    "-       m_curr = (m[i] if self.max_emb_dim > 0 else m)\n",
    "-       EE = nn.EmbeddingBag(n, m_curr, mode=\"sum\", sparse=True)\n",
    "-       W = np.random.uniform(\n",
    "-           low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, m_curr)\n",
    "-       ).astype(np.float32)\n",
    "\n",
    "        # democratized, use two dimension, sparse and dense\n",
    "+       W = np.random.uniform(\n",
    "+           low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, m)\n",
    "+       ).astype(np.float32)\n",
    "+       if n >= self.sparse_dense_boundary:\n",
    "+           m_sparse = int(m/4)\n",
    "+           W = np.random.uniform(\n",
    "+               low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, m_sparse)\n",
    "+           ).astype(np.float32)\n",
    "+           EE = nn.EmbeddingBag(n, m_sparse, mode=\"sum\", sparse=True, _weight=torch.tensor(W, requires_grad=True))\n",
    "+       else:\n",
    "+           W = np.random.uniform(\n",
    "+               low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, m)\n",
    "+           ).astype(np.float32)\n",
    "+           EE = nn.EmbeddingBag(n, m, mode=\"sum\", sparse=False, _weight=torch.tensor(W, requires_grad=True))\n",
    "+           tensor(W, requires_grad=True))\n",
    "\n",
    "        # cast to BF16\n",
    "+       EE.to(torch.bfloat16)\n",
    "        ...\n",
    "```\n",
    "\n",
    "### Adding distributed training in forward function\n",
    "\n",
    "``` diff\n",
    "def forward(self, dense_x, lS_o, lS_i):\n",
    "+   if self.bf16:\n",
    "+       dense_x = dense_x.bfloat16()\n",
    "+   if ext_dist.my_size > 1:\n",
    "+       return self.distributed_forward(dense_x, lS_o, lS_i)\n",
    "    if self.ndevices <= 1:\n",
    "        return self.sequential_forward(dense_x, lS_o, lS_i)\n",
    "    else:\n",
    "        return self.parallel_forward(dense_x, lS_o, lS_i)\n",
    "```\n",
    "\n",
    "### Expanding Optimizer Options\n",
    "\n",
    "``` diff\n",
    "- optimizer = torch.optim.SGD(dlrm.parameters(), lr=args.learning_rate)\n",
    "+ optimizer_list = ([torch.optim.SGD, ([Lamb, False], torch.optim.SGD),\n",
    "+                    torch.optim.Adagrad, ([torch.optim.Adam, None], torch.optim.SparseAdam)],\n",
    "+                   [ipex.SplitSGD, ([Lamb, True], ipex.SplitSGD)])\n",
    "+ optimizers = optimizer_list[args.bf16 and ipex.is_available()][args.optimizer]\n",
    "+ optimizer_dense = optimizers[0][0]([\n",
    "+     {\"params\": [p for emb in dlrm.emb_dense for p in emb.parameters()], \"lr\": args.lamblr},\n",
    "+     {\"params\": dlrm.bot_l.parameters(), \"lr\": args.lamblr},\n",
    "+     {\"params\": dlrm.top_l.parameters(), \"lr\": args.lamblr}\n",
    "+ ], lr=args.lamblr, bf16=args.bf16)\n",
    "+ optimizer_sparse = optimizers[1]([\n",
    "+     {\"params\": [p for emb in dlrm.emb_sparse for p in emb.parameters()],\n",
    "+      \"lr\": args.learning_rate / ext_dist.my_size},\n",
    "+ ], lr=args.learning_rate)\n",
    "+ optimizer = (optimizer_dense, optimizer_sparse)\n",
    "```\n",
    "\n",
    "### HPO with SDA (Smart Democratization Advisor)\n",
    "SDA config\n",
    "\n",
    "Parameters for SDA auto optimization:\n",
    "- learning_rate: 5 ~ 50\n",
    "- lamb_lr: 5 ~ 50\n",
    "- warmup_steps: 2000 ~ 4500\n",
    "- decay_start_steps: 4501 ~ 9000\n",
    "- num_decay_steps: 5000 ~ 15000\n",
    "- sparse_feature_size: [128, 64, 16]\n",
    "- mlp_top_size: [\"1024-1024-512-256-1\",\"512-512-256-128-1\",\"512-256-128-1\",\"512-256-1\",\"256-128-1\",\"128-64-1\",\"256-1\",\"128-1\"]\n",
    "- mlp_bot_size = [\"13-512-256-\",\"13-512-256-\",\"13-256-\",\"13-128-\"]\n",
    "            \n",
    "\n",
    "metrics:\n",
    "- name: accuracy\n",
    "  objective: maximize\n",
    "- name: training_time\n",
    "  strategy: optimize\n",
    "  objective: minimize\n",
    "  \n",
    "\n",
    "## Performance\n",
    "\n",
    "\n",
    "![dlrm_performance](./img/dlrm_perf.png)\n",
    "\n",
    "### Key Optimizations\n",
    "\n",
    "* Intel Optimized training framework + Lamb optimizer: 1.29x speedup (1.64 to 2.13)​\n",
    "\n",
    "* LAMB + BF16: 1.45x speedup (2.87 to 4.18)​\n",
    "\n",
    "* Lighter Model-part1: model optimization(decrease both dense embedding and sparse embedding output dimension from 128 to 64, reduced bot_mlp layer size from 13-512-256-128 to 13-256-128-64, top_mlp layer size from 1024-1024-512-256-1 to 512-512-256-1) delivers 2.17x speedup(4.18 to 9.07)​\n",
    "\n",
    "* Optimizer: lamb optimizer delivers 1.04x speedup(9.07 to 9.50)​\n",
    "\n",
    "* Lighter Model-part2: model optimization(reduced bot_mlp layer size from 13-256-128-64 to 13-128-64, top_mlp layer size from 512-512-256-1 to 256-128-1) delivers 1.22x speedup(9.50 to 11.61) ​\n",
    "\n",
    "* Embedding: reducing sparse embedding table number from 16 to 8 delivers 1.36x speedup(11.61 to 15.85) ​\n",
    "\n",
    "* Framework optimization: using latest IPEX launch scripts (auto bind CCL to specific core) and add KMP setting delivers 1.06x speedup (15.85 to 16.90)​\n",
    "\n",
    "* AlltoAll optimization: reduce the output dimension size from 64 to 16 and then repeat 4 times to 64 dimension after alltoall, which delivers 1.25x speedup(16.90 to 21.24)​\n",
    "\n",
    "* Learning rate and test optimization: change lamb optimizer learning rate from 16 to 30;remove gradient computing in the test part; increase test batch size from 16K to 128K. These optimizations delivers 1.30x speedup(21.24 to 27.71)​\n",
    "\n",
    "* Other optimization: (1) Using NVMe to replace HDD as storage;(2) Leveraging prefetch_generator PyTorch dataloader optimization package to enable batch data generator working in background thread parallelly with training. These optimizations delivers 1.08x speedup(27.71 to 29.93) ​\n",
    "\n",
    "* SigOpt optimization: In limit experiments trials, SigOpt with metrics threshold delivers 1.06x speedup(29.93 to 31.89) than previous manual optimized result.​\n",
    "\n",
    "------\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "213dafa7",
   "metadata": {},
   "source": [
    "# Getting Started\n",
    "Noted: This demo needs to compile torch v1.5.0-rc3, ipex, torchCCL package, it was tested on bare metal environment. Due to the limitations of colab system, this notebook is only for presentation purpose. Result AUC showed is trained with a small sample dataset. Full dataset result please refer to [performance section](#performance). \n",
    "\n",
    "  * [1. Environment Setup](#1.-Environment-Setup)\n",
    "  * [2. Workflow Prepare](#2.-Workflow-Prepare)\n",
    "  * [3. Data Prepare](#3.-Data-Prepare)\n",
    "  * [4. Train](#4.-Train)\n",
    "  * [5. Inference](#5.-Inference)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfa78ed3",
   "metadata": {},
   "source": [
    "## 1. Enviroment Setup"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "61f9eafa",
   "metadata": {},
   "source": [
    "### (Option 1) Use Pip install\n",
    "\n",
    "  * source: https://github.com/mlperf/training_results_v0.7/tree/master/Intel/benchmarks/dlrm/1-node-4s-cpx-pytorch\n",
    "  * It has strict dependencies to GCC version, torch version, oneCCL version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5a6500d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# install GCC and dependencies\n",
    "! conda install gxx_linux-64==8.4.0\n",
    "! pip install sklearn onnx tqdm lark-parser\n",
    "! pip install -e git+https://github.com/mlperf/logging@0.7.0-rc2#egg=logging\n",
    "! conda config --append channels intel\n",
    "! conda install ninja pyyaml setuptools cmake cffi typing\n",
    "! conda install intel-openmp mkl mkl-include numpy -c intel --no-update-deps\n",
    "! conda install -c conda-forge gperftools\n",
    "# git clone pytorch v1.5.0-rc3\n",
    "! git clone https://github.com/pytorch/pytorch.git && cd pytorch && git checkout tags/v1.5.0-rc3 -b v1.5-rc3 && git submodule sync && git submodule update --init --recursive\n",
    "# git clone ipex v0.2\n",
    "! git clone https://github.com/intel/intel-extension-for-pytorch.git && cd intel-extension-for-pytorch && git checkout tags/v0.2 -b v0.2 && git submodule sync && git submodule update --init --recursive\n",
    "# apply ipex patch to pytorch and install\n",
    "! cd intel-extension-for-pytorch && cp torch_patches/0001-enable-Intel-Extension-for-CPU-enable-CCL-backend.patch ../pytorch/ && cd ../pytorch && patch -p1 < 0001-enable-Intel-Extension-for-CPU-enable-CCL-backend.patch\n",
    "! cd pytorch && python setup.py install\n",
    "! cd intel-extension-for-pytorch && python setup.py install\n",
    "# git clone oneCCL and install\n",
    "! git clone https://github.com/oneapi-src/oneCCL.git && cd oneCCL && git checkout 2021.1-beta07-1 && mkdir build && cd build && cmake .. -DCMAKE_INSTALL_PREFIX=~/.local && make install -j\n",
    "# git clone torchCCL and install\n",
    "! git clone https://github.com/intel/torch-ccl.git && cd torch-ccl && git checkout 2021.1-beta07-1\n",
    "! source ~/.local/env/setvars.sh && cd torch-ccl && python setup.py install\n",
    "! python -m pip install onnx tqdm lark-parser pyyaml prefetch_generator tensorboardX psutil sigopt pandas pyarrow lightgbm transformers xgboost\n",
    "\n",
    "# install pyrecdp and e2eAIOK-sda\n",
    "! python -m pip install pyrecdp\n",
    "! python -m pip install e2eAIOK-sda --pre"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4cf5ce5f",
   "metadata": {},
   "source": [
    "### (Option 2) Use Docker"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "abef025d",
   "metadata": {},
   "source": [
    "``` bash\n",
    "# 1. git clone codes\n",
    "git clone https://github.com/intel/e2eAIOK.git\n",
    "cd e2eAIOK\n",
    "git submodule update --init --recursive\n",
    "\n",
    "# 2. build docker image\n",
    "python3 scripts/start_e2eaiok_docker.py -b pytorch -w ${host0} ${host1} ${host2} ${host3} --proxy \"http://addr:ip\"\n",
    "\n",
    "# 3. Enter Docker\n",
    "sshpass -p docker ssh ${host0} -p 12344\n",
    "\n",
    "# 4. start jupyter notebook\n",
    "nohup jupyter notebook --notebook-dir=/home/vmagent/app/e2eaiok --ip=${hostname} --port=8899 --allow-root &\n",
    "Now you can visit demso in http://${hostname}:8899/.\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4de71f4",
   "metadata": {},
   "source": [
    "## 2. Workflow Prepare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f56b1f4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "! sh workflow_prepare_dlrm.sh"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17e23e85",
   "metadata": {},
   "source": [
    "## 3. Data Prepare"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b22961d1",
   "metadata": {},
   "source": [
    "Download from https://labs.criteo.com/2013/12/download-terabyte-click-logs/ and unzip them\n",
    "``` bash\n",
    "ls criteo/raw_data\n",
    "day_0  day_10  day_12  day_14  day_16  day_18  day_2   day_21  day_23  day_4  day_6  day_8\n",
    "day_1  day_11  day_13  day_15  day_17  day_19  day_20  day_22  day_3   day_5  day_7  day_9\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "82c7d3d4",
   "metadata": {},
   "source": [
    "lauch data process to downloaded files.\n",
    "original data contains 23 large csv files, below script will do Data Conversion from CSV to parquet files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ee070f24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sr140\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "22/10/29 05:51:25 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "22/10/29 05:51:25 WARN SparkConf: Note that spark.local.dir will be overridden by the value set by the cluster manager (via SPARK_LOCAL_DIRS in mesos/standalone/kubernetes and LOCAL_DIRS in YARN).\n",
      "per core memory size is 7.500 GB and shuffle_disk maximum capacity is 800.000 GB\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_parquet_train_day_0\n",
      "Convert day_0 to parquet completed, took 104.0467173489742 secs                 \n",
      "Splitting the last day into 2 parts of test and validation...\n",
      "head: cannot open '/home/vmagent/app/dataset/criteo_small/raw_data//day_23' for reading: No such file or directory\n",
      "tail: cannot open '/home/vmagent/app/dataset/criteo_small/raw_data//day_23' for reading: No such file or directory\n",
      "Split day_23 to test and valid completed, took 0.00858217105269432 secs\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_parquet_test\n",
      "Convert test to parquet completed, took 0.4288335369201377 secs\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_parquet_valid\n",
      "Convert valid to parquet completed, took 0.49960903904866427 secs\n",
      "Total process time is 109.44366508396342 secs\n"
     ]
    }
   ],
   "source": [
    "! cd e2eaiok/modelzoo/dlrm/data_processing/; python convert_to_parquet.py --dataset_path criteo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61a244f8",
   "metadata": {},
   "source": [
    "do data process to convert all 23 files to ready-to-train parquet files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "925358f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sr140\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
      "22/10/29 04:18:04 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "22/10/29 04:18:04 WARN SparkConf: Note that spark.local.dir will be overridden by the value set by the cluster manager (via SPARK_LOCAL_DIRS in mesos/standalone/kubernetes and LOCAL_DIRS in YARN).\n",
      "recdp-scala-extension is enabled\n",
      "per core memory size is 7.500 GB and shuffle_disk maximum capacity is 800.000 GB\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_parquet_train_proc_day_0\n",
      "Process day_0 categorified columns completed, took 317.1812623520382 secs       \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_parquet_test_proc\n",
      "Process test categorified columns completed, took 1.6086477079661563 secs\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_parquet_valid_proc\n",
      "Process valid categorified columns completed, took 1.1995910430559888 secs\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c14\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c15  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c16  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c17  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c18  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c19  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c20  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c21  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c22  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c23  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c24  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c25  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c26  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c27  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c28  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c29  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c30  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c31  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c32  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c33  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c34  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c35  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c36  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c37  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c38  \n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dicts/_c39  \n",
      "Generate Dictionary took 337.386                                                \n",
      "[14885288, 29419, 15123, 7291, 19899, 3, 6463, 1310, 61, 10155909, 618195, 218994, 10, 2208, 9779, 71, 4, 963, 14, 16967044, 4154705, 13180313, 289595, 10828, 95, 34]\n",
      "[14885288, 29419, 15123, 7291, 19899, 3, 6463, 1310, 61, 10155909, 618195, 218994, 10, 2208, 9779, 71, 4, 963, 14, 16967044, 4154705, 13180313, 289595, 10828, 95, 34]\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_categorified_day_0\n",
      "Categorify took 103.383                                                         \n",
      "Apply dicts to day_0 completed, took 108.56904100207612 secs\n",
      "[14885288, 29419, 15123, 7291, 19899, 3, 6463, 1310, 61, 10155909, 618195, 218994, 10, 2208, 9779, 71, 4, 963, 14, 16967044, 4154705, 13180313, 289595, 10828, 95, 34]\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_categorified_test\n",
      "Categorify took 29.728                                                          \n",
      "Apply dicts to test completed, took 35.5725970629137 secs\n",
      "[14885288, 29419, 15123, 7291, 19899, 3, 6463, 1310, 61, 10155909, 618195, 218994, 10, 2208, 9779, 71, 4, 963, 14, 16967044, 4154705, 13180313, 289595, 10828, 95, 34]\n",
      "save data to file:////home/vmagent/app/dataset/criteo_small/output//dlrm_categorified_valid\n",
      "Categorify took 33.698                                                          \n",
      "Apply dicts to valid completed, took 38.95257829094771 secs\n",
      "Total process time is 848.1591420699842 secs\n"
     ]
    }
   ],
   "source": [
    "! cd e2eaiok/modelzoo/dlrm/data_processing/; python preprocessing.py --dataset_path criteo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1064caee",
   "metadata": {},
   "source": [
    "DLRM model need to consume numpy files, do data conversion from parquet to numpy binary files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "56a0469b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start to generate day_fea_count.npz\n",
      "Get dimension of _c14\n",
      "Get dimension of _c15\n",
      "Get dimension of _c16\n",
      "Get dimension of _c17\n",
      "Get dimension of _c18\n",
      "Get dimension of _c19\n",
      "Get dimension of _c20\n",
      "Get dimension of _c21\n",
      "Get dimension of _c22\n",
      "Get dimension of _c23\n",
      "Get dimension of _c24\n",
      "Get dimension of _c25\n",
      "Get dimension of _c26\n",
      "Get dimension of _c27\n",
      "Get dimension of _c28\n",
      "Get dimension of _c29\n",
      "Get dimension of _c30\n",
      "Get dimension of _c31\n",
      "Get dimension of _c32\n",
      "Get dimension of _c33\n",
      "Get dimension of _c34\n",
      "Get dimension of _c35\n",
      "Get dimension of _c36\n",
      "Get dimension of _c37\n",
      "Get dimension of _c38\n",
      "Get dimension of _c39\n",
      "[14885288, 29419, 15123, 7291, 19899, 3, 6463, 1310, 61, 10155909, 618195, 218994, 10, 2208, 9779, 71, 4, 963, 14, 16967044, 4154705, 13180313, 289595, 10828, 95, 34]\n",
      "Start to convert train/valid/test\n",
      "multi process is 6\n",
      "Create subprocess 0 for train_data.bin[0:1], total 1\n",
      "Start to convert parquet files to numpy binary\n",
      "Start to write binary to /home/vmagent/app/dataset/criteo_small/output/train_data.bin\n",
      "Convert dlrm_categorified_day_0 to binary completed, took 218.64829254196957 secs\n",
      "All subprocess for train_data.bin completed, took 219.313 secs\n",
      "Completed for train_data.bin, took 219.313 secs\n",
      "multi process is 1\n",
      "Start to convert parquet files to numpy binary\n",
      "Start to write binary to /home/vmagent/app/dataset/criteo_small/output/test_data.bin\n",
      "Convert dlrm_categorified_test to binary completed, took 0.02760987705551088 secs\n",
      "Completed for test_data.bin, took 0.028 secs\n",
      "multi process is 1\n",
      "Start to convert parquet files to numpy binary\n",
      "Start to write binary to /home/vmagent/app/dataset/criteo_small/output/valid_data.bin\n",
      "Convert dlrm_categorified_valid to binary completed, took 0.014620395959354937 secs\n",
      "Completed for valid_data.bin, took 0.015 secs\n",
      "Total process time is 220.2311840520706 secs\n"
     ]
    }
   ],
   "source": [
    "! cd e2eaiok/modelzoo/dlrm/data_processing/; python convert_to_binary.py --dataset_path criteo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2922f326",
   "metadata": {},
   "source": [
    "# 4. Train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d518f73",
   "metadata": {},
   "source": [
    "SDA is an autoHPO component, we use SDA to trigger DLRM training and validation.\n",
    "Noticed: set enable_sigopt to True, SDA will explore HyperParameter for this model. In our demo, we will use our searched best parameter."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "2fbda6f7",
   "metadata": {},
   "source": [
    "> Note: Bellow training script is just for demonstration, and uses small sampled dataset and runs a small iterations. For actual performance result, please refer to [performance](#performance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b9c91c1b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data format is binary\n",
      "\n",
      "===============================================\n",
      "***    Best Trained Model    ***\n",
      "===============================================\n",
      "  Model Type: dlrm\n",
      "  Model Saved Path: \n",
      "  Sigopt Experiment id is None\n",
      "  === Result Metrics ===\n",
      "===============================================\n",
      "[1] torch_ccl is True\n",
      "[0] torch_ccl is True\n",
      "[1] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937085214, \"event_type\": \"POINT_IN_TIME\", \"key\": \"cache_clear\", \"value\": true, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 646}}\n",
      "[1] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937085294, \"event_type\": \"INTERVAL_START\", \"key\": \"init_start\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 648}}\n",
      "[1] world_size:2,rank:1\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937085224, \"event_type\": \"POINT_IN_TIME\", \"key\": \"cache_clear\", \"value\": true, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 646}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937085298, \"event_type\": \"INTERVAL_START\", \"key\": \"init_start\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 648}}\n",
      "[0] world_size:2,rank:0\n",
      "[0] sparse options: coalesce mode 0, result mode 0\n",
      "[1] Using CCL_ATL_TRANSPORT=ofi\n",
      "[1] Using CCL_ATL_SHM=(default)\n",
      "[0] Running on 2 ranks using ccl backend\n",
      "[0] Using CCL_ATL_TRANSPORT=ofi\n",
      "[0] Using CCL_ATL_SHM=(default)\n",
      "[0] command line args:  {\"arch_sparse_feature_size\": 128, \"arch_embedding_size\": \"4-3-2\", \"arch_mlp_bot\": \"13-512-256-128\", \"arch_mlp_top\": \"1024-1024-512-256-1\", \"arch_interaction_op\": \"dot\", \"arch_interaction_itself\": false, \"md_flag\": false, \"md_threshold\": 200, \"md_temperature\": 0.3, \"md_round_dims\": false, \"qr_flag\": false, \"qr_threshold\": 200, \"qr_operation\": \"mult\", \"qr_collisions\": 4, \"activation_function\": \"relu\", \"loss_function\": \"bce\", \"loss_weights\": \"1.0-1.0\", \"loss_threshold\": 0.0, \"round_targets\": true, \"data_size\": 1, \"num_batches\": 0, \"data_generation\": \"dataset\", \"data_trace_file\": \"./input/dist_emb_j.log\", \"data_set\": \"terabyte\", \"raw_data_file\": \"\", \"processed_data_file\": \"\", \"data_randomize\": \"total\", \"data_trace_enable_padding\": false, \"max_ind_range\": -1, \"data_sub_sample_rate\": 0.0, \"num_indices_per_lookup\": 10, \"num_indices_per_lookup_fixed\": false, \"num_workers\": 0, \"memory_map\": true, \"mini_batch_size\": 262144, \"nepochs\": 1, \"learning_rate\": 30.0, \"print_precision\": 5, \"numpy_rand_seed\": 12345, \"sync_dense_params\": true, \"inference_only\": false, \"save_onnx\": false, \"use_gpu\": false, \"dist_backend\": \"ccl\", \"print_freq\": 16, \"test_freq\": 800, \"test_mini_batch_size\": 131072, \"test_num_workers\": 0, \"print_time\": true, \"debug_mode\": false, \"enable_profiling\": false, \"plot_compute_graph\": false, \"profiling_start_iter\": 50, \"profiling_num_iters\": 100, \"out_dir\": \".\", \"save_model\": \"/home/vmagent/app/e2eaiok/result/44a4014329fed79f04b843ff2cfbe6af\", \"load_model\": \"\", \"mlperf_logging\": true, \"mlperf_acc_threshold\": 0.0, \"mlperf_auc_threshold\": 0.8025, \"mlperf_bin_loader\": true, \"mlperf_bin_shuffle\": true, \"lr_num_warmup_steps\": 4000, \"lr_decay_start_step\": 5000, \"lr_num_decay_steps\": 5000, \"sparse_dense_boundary\": 403346, \"bf16\": true, \"use_ipex\": true, \"optimizer\": 1, \"lamblr\": 16.0, \"train_data_path\": \"/home/vmagent/app/dataset/criteo/train/train_data.bin\", \"eval_data_path\": \"/home/vmagent/app/dataset/criteo/valid/test_data.bin\", \"day_feature_count\": \"/home/vmagent/app/dataset/criteo/day_fea_count.npz\"}\n",
      "[1] command line args:  {\"arch_sparse_feature_size\": 128, \"arch_embedding_size\": \"4-3-2\", \"arch_mlp_bot\": \"13-512-256-128\", \"arch_mlp_top\": \"1024-1024-512-256-1\", \"arch_interaction_op\": \"dot\", \"arch_interaction_itself\": false, \"md_flag\": false, \"md_threshold\": 200, \"md_temperature\": 0.3, \"md_round_dims\": false, \"qr_flag\": false, \"qr_threshold\": 200, \"qr_operation\": \"mult\", \"qr_collisions\": 4, \"activation_function\": \"relu\", \"loss_function\": \"bce\", \"loss_weights\": \"1.0-1.0\", \"loss_threshold\": 0.0, \"round_targets\": true, \"data_size\": 1, \"num_batches\": 0, \"data_generation\": \"dataset\", \"data_trace_file\": \"./input/dist_emb_j.log\", \"data_set\": \"terabyte\", \"raw_data_file\": \"\", \"processed_data_file\": \"\", \"data_randomize\": \"total\", \"data_trace_enable_padding\": false, \"max_ind_range\": -1, \"data_sub_sample_rate\": 0.0, \"num_indices_per_lookup\": 10, \"num_indices_per_lookup_fixed\": false, \"num_workers\": 0, \"memory_map\": true, \"mini_batch_size\": 262144, \"nepochs\": 1, \"learning_rate\": 30.0, \"print_precision\": 5, \"numpy_rand_seed\": 12345, \"sync_dense_params\": true, \"inference_only\": false, \"save_onnx\": false, \"use_gpu\": false, \"dist_backend\": \"ccl\", \"print_freq\": 16, \"test_freq\": 800, \"test_mini_batch_size\": 131072, \"test_num_workers\": 0, \"print_time\": true, \"debug_mode\": false, \"enable_profiling\": false, \"plot_compute_graph\": false, \"profiling_start_iter\": 50, \"profiling_num_iters\": 100, \"out_dir\": \".\", \"save_model\": \"/home/vmagent/app/e2eaiok/result/44a4014329fed79f04b843ff2cfbe6af\", \"load_model\": \"\", \"mlperf_logging\": true, \"mlperf_acc_threshold\": 0.0, \"mlperf_auc_threshold\": 0.8025, \"mlperf_bin_loader\": true, \"mlperf_bin_shuffle\": true, \"lr_num_warmup_steps\": 4000, \"lr_decay_start_step\": 5000, \"lr_num_decay_steps\": 5000, \"sparse_dense_boundary\": 403346, \"bf16\": true, \"use_ipex\": true, \"optimizer\": 1, \"lamblr\": 16.0, \"train_data_path\": \"/home/vmagent/app/dataset/criteo/train/train_data.bin\", \"eval_data_path\": \"/home/vmagent/app/dataset/criteo/valid/test_data.bin\", \"day_feature_count\": \"/home/vmagent/app/dataset/criteo/day_fea_count.npz\"}\n",
      "[0] Using IPEX...\n",
      "[1] Using IPEX...\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937086367, \"event_type\": \"INTERVAL_END\", \"key\": \"init_stop\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 805}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937086368, \"event_type\": \"INTERVAL_START\", \"key\": \"run_start\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 807}}\n",
      "[0] data file: /home/vmagent/app/dataset/criteo/train/train_data.bin number of batches: 256\n",
      "[1] data file: /home/vmagent/app/dataset/criteo/train/train_data.bin number of batches: 256\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937086369, \"event_type\": \"POINT_IN_TIME\", \"key\": \"train_samples\", \"value\": 67108864, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_data_pytorch.py\", \"lineno\": 405}}\n",
      "[1] data file: /home/vmagent/app/dataset/criteo/valid/test_data.bin number of batches: 681\n",
      "[0] data file: /home/vmagent/app/dataset/criteo/valid/test_data.bin number of batches: 681\n",
      "[1] Creating the model...\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937086370, \"event_type\": \"POINT_IN_TIME\", \"key\": \"eval_samples\", \"value\": 89137319, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_data_pytorch.py\", \"lineno\": 427}}\n",
      "[0] Creating the model...\n",
      "[0] Model created!\n",
      "[0] DLRM_Net(\n",
      "[0]   (emb_dense): ModuleList(\n",
      "[0]     (0): EmbeddingBag(39043, 128, mode=sum)\n",
      "[0]     (1): EmbeddingBag(17289, 128, mode=sum)\n",
      "[0]     (2): EmbeddingBag(7420, 128, mode=sum)\n",
      "[0]     (3): EmbeddingBag(20263, 128, mode=sum)\n",
      "[0]     (4): EmbeddingBag(3, 128, mode=sum)\n",
      "[0]     (5): EmbeddingBag(7120, 128, mode=sum)\n",
      "[0]     (6): EmbeddingBag(1543, 128, mode=sum)\n",
      "[0]     (7): EmbeddingBag(63, 128, mode=sum)\n",
      "[0]     (8): EmbeddingBag(10, 128, mode=sum)\n",
      "[0]     (9): EmbeddingBag(2208, 128, mode=sum)\n",
      "[0]     (10): EmbeddingBag(11938, 128, mode=sum)\n",
      "[0]     (11): EmbeddingBag(155, 128, mode=sum)\n",
      "[0]     (12): EmbeddingBag(4, 128, mode=sum)\n",
      "[0]     (13): EmbeddingBag(976, 128, mode=sum)\n",
      "[0]     (14): EmbeddingBag(14, 128, mode=sum)\n",
      "[0]     (15): EmbeddingBag(12972, 128, mode=sum)\n",
      "[0]     (16): EmbeddingBag(108, 128, mode=sum)\n",
      "[0]     (17): EmbeddingBag(36, 128, mode=sum)\n",
      "[0]   )\n",
      "[0]   (emb_sparse): ModuleList(\n",
      "[0]     (0): EmbeddingBag(39884406, 32, mode=sum)\n",
      "[0]     (1): EmbeddingBag(38532951, 32, mode=sum)\n",
      "[0]     (2): EmbeddingBag(2953546, 32, mode=sum)\n",
      "[0]     (3): EmbeddingBag(403346, 32, mode=sum)\n",
      "[0]   )\n",
      "[0]   (bot_l): Sequential(\n",
      "[0]     (0): IpexMLPLinear(C=13, K=512, bias=True)\n",
      "[0]     (1): IpexMLPLinear(C=512, K=256, bias=True)\n",
      "[0]     (2): IpexMLPLinear(C=256, K=128, bias=True)\n",
      "[0]   )\n",
      "[0]   (top_l): Sequential(\n",
      "[0]     (0): IpexMLPLinear(C=479, K=1024, bias=True)\n",
      "[0]     (1): IpexMLPLinear(C=1024, K=1024, bias=True)\n",
      "[0]     (2): IpexMLPLinear(C=1024, K=512, bias=True)\n",
      "[0]     (3): IpexMLPLinear(C=512, K=256, bias=True)\n",
      "[0]     (4): IpexMLPLinear(C=256, K=1, bias=True)\n",
      "[0]     (5): Cast(to(torch.float32))\n",
      "[0]     (6): Sigmoid()\n",
      "[0]   )\n",
      "[0] ) dpcpp True[0] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] Model created!\n",
      "[1] DLRM_Net(\n",
      "[1]   (emb_dense): ModuleList(\n",
      "[1]     (0): EmbeddingBag(39043, 128, mode=sum)\n",
      "[1]     (1): EmbeddingBag(17289, 128, mode=sum)\n",
      "[1]     (2): EmbeddingBag(7420, 128, mode=sum)\n",
      "[1]     (3): EmbeddingBag(20263, 128, mode=sum)\n",
      "[1]     (4): EmbeddingBag(3, 128, mode=sum)\n",
      "[1]     (5): EmbeddingBag(7120, 128, mode=sum)\n",
      "[1]     (6): EmbeddingBag(1543, 128, mode=sum)\n",
      "[1]     (7): EmbeddingBag(63, 128, mode=sum)\n",
      "[1]     (8): EmbeddingBag(10, 128, mode=sum)\n",
      "[1]     (9): EmbeddingBag(2208, 128, mode=sum)\n",
      "[1]     (10): EmbeddingBag(11938, 128, mode=sum)\n",
      "[1]     (11): EmbeddingBag(155, 128, mode=sum)\n",
      "[1]     (12): EmbeddingBag(4, 128, mode=sum)\n",
      "[1]     (13): EmbeddingBag(976, 128, mode=sum)\n",
      "[1]     (14): EmbeddingBag(14, 128, mode=sum)\n",
      "[1]     (15): EmbeddingBag(12972, 128, mode=sum)\n",
      "[1]     (16): EmbeddingBag(108, 128, mode=sum)\n",
      "[1]     (17): EmbeddingBag(36, 128, mode=sum)\n",
      "[1]   )\n",
      "[1]   (emb_sparse): ModuleList(\n",
      "[1]     (0): EmbeddingBag(39979771, 32, mode=sum)\n",
      "[1]     (1): EmbeddingBag(25641295, 32, mode=sum)\n",
      "[1]     (2): EmbeddingBag(39664984, 32, mode=sum)\n",
      "[1]     (3): EmbeddingBag(585935, 32, mode=sum)\n",
      "[1]   )\n",
      "[1]   (bot_l): Sequential(\n",
      "[1]     (0): IpexMLPLinear(C=13, K=512, bias=True)\n",
      "[1]     (1): IpexMLPLinear(C=512, K=256, bias=True)\n",
      "[1]     (2): IpexMLPLinear(C=256, K=128, bias=True)\n",
      "[1]   )\n",
      "[1]   (top_l): Sequential(\n",
      "[1]     (0): IpexMLPLinear(C=479, K=1024, bias=True)\n",
      "[1]     (1): IpexMLPLinear(C=1024, K=1024, bias=True)\n",
      "[1]     (2): IpexMLPLinear(C=1024, K=512, bias=True)\n",
      "[1]     (3): IpexMLPLinear(C=512, K=256, bias=True)\n",
      "[1]     (4): IpexMLPLinear(C=256, K=1, bias=True)\n",
      "[1]     (5): Cast(to(torch.float32))\n",
      "[1]     (6): Sigmoid()\n",
      "[1]   )\n",
      "[1] ) [1] dpcpp True\n",
      "[1] Chosen optimizer(s): ([<class 'lamb_bin.Lamb'>, True], <class 'intel_pytorch_extension.optim.split_sgd.SplitSGD'>)\n",
      "[0] Chosen optimizer(s): ([<class 'lamb_bin.Lamb'>, True], <class 'intel_pytorch_extension.optim.split_sgd.SplitSGD'>)\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258938, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_benchmark\", \"value\": \"dlrm\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 88}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258938, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_org\", \"value\": \"reference_implementation\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 93}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258938, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_division\", \"value\": \"closed\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 97}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_status\", \"value\": \"onprem\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 101}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_platform\", \"value\": \"reference_implementation\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 105}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_entry\", \"value\": \"reference_implementation\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 109}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_poc_name\", \"value\": \"reference_implementation\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 113}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"submission_poc_email\", \"value\": \"reference_implementation\", \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/mlperf_logger.py\", \"lineno\": 117}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"seed\", \"value\": 12345, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1128}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"global_batch_size\", \"value\": 262144, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1129}}\n",
      "[0] time/loss/accuracy (if enabled):\n",
      "[1] time/loss/accuracy (if enabled):\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258939, \"event_type\": \"POINT_IN_TIME\", \"key\": \"opt_base_learning_rate\", \"value\": 30.0, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1194}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258940, \"event_type\": \"POINT_IN_TIME\", \"key\": \"opt_learning_rate_warmup_steps\", \"value\": 4000, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1196}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258940, \"event_type\": \"POINT_IN_TIME\", \"key\": \"sgd_opt_base_learning_rate\", \"value\": 30.0, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1199}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258940, \"event_type\": \"POINT_IN_TIME\", \"key\": \"lr_decay_start_steps\", \"value\": 5000, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1200}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258940, \"event_type\": \"POINT_IN_TIME\", \"key\": \"sgd_opt_learning_rate_decay_steps\", \"value\": 5000, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1201}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258940, \"event_type\": \"POINT_IN_TIME\", \"key\": \"sgd_opt_learning_rate_decay_poly_power\", \"value\": 2, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1202}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258940, \"event_type\": \"INTERVAL_START\", \"key\": \"block_start\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1221, \"first_epoch_num\": 1, \"epoch_count\": 1}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666937258941, \"event_type\": \"INTERVAL_START\", \"key\": \"epoch_start\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1224, \"epoch_num\": 1}}\n",
      "[0] Finished training it 16/256 of epoch 0, 4759.81 ms/it, loss 0.746051, accuracy 48.666 %\n",
      "[1] Finished training it 16/256 of epoch 0, 4760.13 ms/it, loss 0.746662, accuracy 48.644 %\n",
      "[0] Finished training it 32/256 of epoch 0, 5021.98 ms/it, loss 0.226729, accuracy 96.685 %\n",
      "[1] Finished training it 32/256 of epoch 0, 5021.86 ms/it, loss 0.227117, accuracy 96.682 %\n",
      "[0] Finished training it 48/256 of epoch 0, 5025.35 ms/it, loss 0.157136, accuracy 96.693 %\n",
      "[1] Finished training it 48/256 of epoch 0, 5025.48 ms/it, loss 0.157170, accuracy 96.689 %\n",
      "[0] Finished training it 64/256 of epoch 0, 5019.01 ms/it, loss 0.140964, accuracy 96.688 %\n",
      "[1] Finished training it 64/256 of epoch 0, 5018.61 ms/it, loss 0.140567, accuracy 96.701 %\n",
      "[0] Finished training it 80/256 of epoch 0, 5022.91 ms/it, loss 0.137194, accuracy 96.678 %\n",
      "[1] Finished training it 80/256 of epoch 0, 5023.26 ms/it, loss 0.137025, accuracy 96.686 %\n",
      "[1] Finished training it 96/256 of epoch 0, 5025.52 ms/it, loss 0.134306, accuracy 96.689 %\n",
      "[0] Finished training it 96/256 of epoch 0, 5025.48 ms/it, loss 0.134062, accuracy 96.693 %\n",
      "[0] Finished training it 112/256 of epoch 0, 5021.88 ms/it, loss 0.131496, accuracy 96.702 %\n",
      "[1] Finished training it 112/256 of epoch 0, 5021.94 ms/it, loss 0.132198, accuracy 96.679 %\n",
      "[0] Finished training it 128/256 of epoch 0, 5018.60 ms/it, loss 0.130385, accuracy 96.692 %\n",
      "[1] Finished training it 128/256 of epoch 0, 5018.41 ms/it, loss 0.131410, accuracy 96.666 %[1] \n",
      "[0] Finished training it 144/256 of epoch 0, 5020.56 ms/it, loss 0.129443, accuracy 96.712 %\n",
      "[1] Finished training it 144/256 of epoch 0, 5019.80 ms/it, loss 0.129744, accuracy 96.698 %\n",
      "[0] Finished training it 160/256 of epoch 0, 5014.49 ms/it, loss 0.129208, accuracy 96.701 %\n",
      "[1] Finished training it 160/256 of epoch 0, 5014.61 ms/it, loss 0.129652, accuracy 96.685 %\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0] Finished training it 176/256 of epoch 0, 5016.83 ms/it, loss 0.129738, accuracy 96.677 %\n",
      "[1] Finished training it 176/256 of epoch 0, 5018.03 ms/it, loss 0.129163, accuracy 96.691 %\n",
      "[0] Finished training it 192/256 of epoch 0, 5024.25 ms/it, loss 0.128225, accuracy 96.722 %\n",
      "[1] Finished training it 192/256 of epoch 0, 5024.32 ms/it, loss 0.128767, accuracy 96.701 %\n",
      "[1] Finished training it 208/256 of epoch 0, 5018.23 ms/it, loss 0.128546, accuracy 96.701 %\n",
      "[0] Finished training it 208/256 of epoch 0, 5019.18 ms/it, loss 0.128171, accuracy 96.715 %\n",
      "[0] Finished training it 224/256 of epoch 0, 5022.23 ms/it, loss 0.128718, accuracy 96.699 %\n",
      "[1] Finished training it 224/256 of epoch 0, 5022.90 ms/it, loss 0.128312, accuracy 96.708 %\n",
      "[0] Finished training it 240/256 of epoch 0, 5010.63 ms/it, loss 0.128252, accuracy 96.703 %\n",
      "[1] Finished training it 240/256 of epoch 0, 5010.63 ms/it, loss 0.128856, accuracy 96.694 %\n",
      "[0] Finished training it 256/256 of epoch 0, 5014.20 ms/it, loss 0.128358, accuracy 96.695 %\n",
      "[1] Finished training it 256/256 of epoch 0, 5014.00 ms/it, loss 0.127615, accuracy 96.719 %\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666938544972, \"event_type\": \"INTERVAL_START\", \"key\": \"eval_start\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1356, \"epoch_num\": 2.0}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666938699459, \"event_type\": \"POINT_IN_TIME\", \"key\": \"eval_accuracy\", \"value\": 0.7686181909195497, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1474, \"epoch_num\": 2.0}}\n",
      "[0] Testing at - 256/256 of epoch 0, loss 0.131926, auc 0.7686, best auc 0.7686, accuracy 96.575 %, best accuracy 0.000 %\n",
      "[1] Testing at - 256/256 of epoch 0, loss 0.131926, auc 0.7686, best auc 0.7686, accuracy 96.575 %, best accuracy 0.000 %\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666938699470, \"event_type\": \"INTERVAL_END\", \"key\": \"eval_stop\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1498, \"epoch_num\": 2.0}}\n",
      "[1] Test time:154.4988715648651\n",
      "[0] Test time:154.50854873657227\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666938699471, \"event_type\": \"INTERVAL_END\", \"key\": \"epoch_stop\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1539, \"epoch_num\": 1}}\n",
      "[1] Total Time:1440.5315449237823\n",
      "[1] Saving model to /home/vmagent/app/e2eaiok/result/44a4014329fed79f04b843ff2cfbe6af\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1666938699471, \"event_type\": \"INTERVAL_END\", \"key\": \"block_stop\", \"value\": null, \"metadata\": {\"file\": \"/home/vmagent/app/e2eaiok/modelzoo/dlrm/dlrm/dlrm_s_pytorch.py\", \"lineno\": 1542, \"first_epoch_num\": 1}}\n",
      "[0] Total Time:1440.530757188797\n",
      "[0] Saving model to /home/vmagent/app/e2eaiok/result/44a4014329fed79f04b843ff2cfbe6af\n",
      "[0] Saved model to /home/vmagent/app/e2eaiok/result/44a4014329fed79f04b843ff2cfbe6af\n",
      "[1] Saved model to /home/vmagent/app/e2eaiok/result/44a4014329fed79f04b843ff2cfbe6af\n",
      "\n",
      "We found the best model! Here is the model explaination\n",
      "\n",
      "===============================================\n",
      "***    Best Trained Model    ***\n",
      "===============================================\n",
      "  Model Type: dlrm\n",
      "  Model Saved Path: /home/vmagent/app/e2eaiok/result/44a4014329fed79f04b843ff2cfbe6af\n",
      "  Sigopt Experiment id is None\n",
      "  === Result Metrics ===\n",
      "    accuracy: 0.7686181909195497\n",
      "    training_time: 1618.6775233745575\n",
      "===============================================\n"
     ]
    }
   ],
   "source": [
    "from e2eAIOK.SDA.SDA import SDA\n",
    "\n",
    "settings = dict()\n",
    "settings[\"data_path\"] = \"criteo/\"\n",
    "settings[\"ppn\"] = 2\n",
    "settings[\"ccl_worker_num\"] = 4\n",
    "settings[\"enable_sigopt\"] = False\n",
    "settings[\"train_script\"] = \"e2eaiok/modelzoo/dlrm/dlrm/\"\n",
    "\n",
    "sda = SDA(model=\"dlrm\", settings=settings) # default settings\n",
    "sda.launch()\n",
    "\n",
    "hydro_model = sda.snapshot()\n",
    "hydro_model.explain()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afd15fb7",
   "metadata": {},
   "source": [
    "# 5. Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "433ee1d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/intel/oneapi/intelpython/latest/envs/pytorch_mlperf/bin/python -u ./dlrm/launch.py --distributed --nproc_per_node=2 ./dlrm/dlrm_s_pytorch_inference.py --eval-data-path=/home/vmagent/app/dataset/criteo/test_data.bin --day-feature-count=/home/vmagent/app/dataset/criteo/day_fea_count.npz --load-model=./result/ --arch-sparse-feature-size=64 --arch-mlp-bot=13-128-64 --arch-mlp-top=256-128-1 --max-ind-range=40000000 --data-generation=dataset --data-set=terabyte --raw-data-file=/home/vmagent/app/dataset/criteo/day --processed-data-file=/home/vmagent/app/dataset/criteo/terabyte_processed.npz --loss-function=bce --round-targets=True --bf16 --num-workers=0 --test-num-workers=0 --use-ipex --optimizer=1 --dist-backend=ccl --learning-rate=16 --mini-batch-size=262144 --print-freq=16 --print-time --test-freq=800 --sparse-dense-boundary=403346 --test-mini-batch-size=131072 --lamblr=30 --lr-num-warmup-steps=4000 --lr-decay-start-step=5760 --lr-num-decay-steps=27000 --memory-map --mlperf-logging --mlperf-auc-threshold=0.8025 --mlperf-bin-loader --mlperf-bin-shuffle --numpy-rand-seed=12345\n",
      "2022-10-29 05:19:19,454 - __main__ - WARNING - Both TCMalloc and JeMalloc are not fount in $CONDA_PREFIX/lib or  /.local/lib/ or /usr/local/lib/ or /usr/local/lib64/ or /usr/lib or /usr/lib64 or ~/.local/lib/ so the LD_PRELOAD environment variable will not be set. This may drop the performance\n",
      "2022-10-29 05:19:19,504 - __main__ - INFO - MASTER_ADDR=127.0.0.1\n",
      "2022-10-29 05:19:19,504 - __main__ - INFO - MASTER_PORT=29500\n",
      "2022-10-29 05:19:19,504 - __main__ - INFO - I_MPI_PIN_DOMAIN=[0x3fff0,0xfffc00000,]\n",
      "2022-10-29 05:19:19,504 - __main__ - INFO - OMP_NUM_THREADS=14 \n",
      "2022-10-29 05:19:19,504 - __main__ - INFO - CCL_WORKER_COUNT=4\n",
      "2022-10-29 05:19:19,504 - __main__ - INFO - CCL_WORKER_AFFINITY=0,1,2,3,18,19,20,21,\n",
      "[0] torch_ccl is True\n",
      "[1] torch_ccl is True\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1667020760668, \"event_type\": \"POINT_IN_TIME\", \"key\": \"cache_clear\", \"value\": true, \"metadata\": {\"file\": \"./dlrm/dlrm_s_pytorch_inference.py\", \"lineno\": 602}}\n",
      "[0] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1667020760748, \"event_type\": \"INTERVAL_START\", \"key\": \"init_start\", \"value\": null, \"metadata\": {\"file\": \"./dlrm/dlrm_s_pytorch_inference.py\", \"lineno\": 604}}\n",
      "[0] world_size:2,rank:0\n",
      "[1] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1667020760673, \"event_type\": \"POINT_IN_TIME\", \"key\": \"cache_clear\", \"value\": true, \"metadata\": {\"file\": \"./dlrm/dlrm_s_pytorch_inference.py\", \"lineno\": 602}}\n",
      "[1] :::MLLOG {\"namespace\": \"\", \"time_ms\": 1667020760763, \"event_type\": \"INTERVAL_START\", \"key\": \"init_start\", \"value\": null, \"metadata\": {\"file\": \"./dlrm/dlrm_s_pytorch_inference.py\", \"lineno\": 604}}\n",
      "[1] world_size:2,rank:1\n",
      "[1] Using CCL_ATL_TRANSPORT=ofi\n",
      "[1] Using CCL_ATL_SHM=(default)\n",
      "[0] sparse options: coalesce mode 0, result mode 0\n",
      "[0] Running on 2 ranks using ccl backend\n",
      "[0] Using CCL_ATL_TRANSPORT=ofi\n",
      "[0] Using CCL_ATL_SHM=(default)\n",
      "[0] command line args:  {\"arch_sparse_feature_size\": 64, \"arch_embedding_size\": \"4-3-2\", \"arch_mlp_bot\": \"13-128-64\", \"arch_mlp_top\": \"256-128-1\", \"arch_interaction_op\": \"dot\", \"arch_interaction_itself\": false, \"md_flag\": false, \"md_threshold\": 200, \"md_temperature\": 0.3, \"md_round_dims\": false, \"qr_flag\": false, \"qr_threshold\": 200, \"qr_operation\": \"mult\", \"qr_collisions\": 4, \"activation_function\": \"relu\", \"loss_function\": \"bce\", \"loss_weights\": \"1.0-1.0\", \"loss_threshold\": 0.0, \"round_targets\": true, \"data_size\": 1, \"num_batches\": 0, \"data_generation\": \"dataset\", \"data_trace_file\": \"./input/dist_emb_j.log\", \"data_set\": \"terabyte\", \"raw_data_file\": \"/home/vmagent/app/dataset/criteo/day\", \"processed_data_file\": \"/home/vmagent/app/dataset/criteo/terabyte_processed.npz\", \"data_randomize\": \"total\", \"data_trace_enable_padding\": false, \"max_ind_range\": 40000000, \"data_sub_sample_rate\": 0.0, \"num_indices_per_lookup\": 10, \"num_indices_per_lookup_fixed\": false, \"num_workers\": 0, \"memory_map\": true, \"mini_batch_size\": 262144, \"nepochs\": 1, \"learning_rate\": 16.0, \"print_precision\": 5, \"numpy_rand_seed\": 12345, \"sync_dense_params\": true, \"inference_only\": false, \"save_onnx\": false, \"use_gpu\": false, \"dist_backend\": \"ccl\", \"print_freq\": 16, \"test_freq\": 800, \"test_mini_batch_size\": 131072, \"test_num_workers\": 0, \"print_time\": true, \"debug_mode\": false, \"enable_profiling\": false, \"plot_compute_graph\": false, \"profiling_start_iter\": 50, \"profiling_num_iters\": 100, \"out_dir\": \".\", \"save_model\": \"\", \"load_model\": \"./result/\", \"mlperf_logging\": true, \"mlperf_acc_threshold\": 0.0, \"mlperf_auc_threshold\": 0.8025, \"mlperf_bin_loader\": true, \"mlperf_bin_shuffle\": true, \"lr_num_warmup_steps\": 4000, \"lr_decay_start_step\": 5760, \"lr_num_decay_steps\": 27000, \"sparse_dense_boundary\": 403346, \"bf16\": true, \"use_ipex\": true, \"optimizer\": 1, \"lamblr\": 30.0, \"eval_data_path\": \"/home/vmagent/app/dataset/criteo/test_data.bin\", \"day_feature_count\": \"/home/vmagent/app/dataset/criteo/day_fea_count.npz\"}\n",
      "[1] command line args:  {\"arch_sparse_feature_size\": 64, \"arch_embedding_size\": \"4-3-2\", \"arch_mlp_bot\": \"13-128-64\", \"arch_mlp_top\": \"256-128-1\", \"arch_interaction_op\": \"dot\", \"arch_interaction_itself\": false, \"md_flag\": false, \"md_threshold\": 200, \"md_temperature\": 0.3, \"md_round_dims\": false, \"qr_flag\": false, \"qr_threshold\": 200, \"qr_operation\": \"mult\", \"qr_collisions\": 4, \"activation_function\": \"relu\", \"loss_function\": \"bce\", \"loss_weights\": \"1.0-1.0\", \"loss_threshold\": 0.0, \"round_targets\": true, \"data_size\": 1, \"num_batches\": 0, \"data_generation\": \"dataset\", \"data_trace_file\": \"./input/dist_emb_j.log\", \"data_set\": \"terabyte\", \"raw_data_file\": \"/home/vmagent/app/dataset/criteo/day\", \"processed_data_file\": \"/home/vmagent/app/dataset/criteo/terabyte_processed.npz\", \"data_randomize\": \"total\", \"data_trace_enable_padding\": false, \"max_ind_range\": 40000000, \"data_sub_sample_rate\": 0.0, \"num_indices_per_lookup\": 10, \"num_indices_per_lookup_fixed\": false, \"num_workers\": 0, \"memory_map\": true, \"mini_batch_size\": 262144, \"nepochs\": 1, \"learning_rate\": 16.0, \"print_precision\": 5, \"numpy_rand_seed\": 12345, \"sync_dense_params\": true, \"inference_only\": false, \"save_onnx\": false, \"use_gpu\": false, \"dist_backend\": \"ccl\", \"print_freq\": 16, \"test_freq\": 800, \"test_mini_batch_size\": 131072, \"test_num_workers\": 0, \"print_time\": true, \"debug_mode\": false, \"enable_profiling\": false, \"plot_compute_graph\": false, \"profiling_start_iter\": 50, \"profiling_num_iters\": 100, \"out_dir\": \".\", \"save_model\": \"\", \"load_model\": \"./result/\", \"mlperf_logging\": true, \"mlperf_acc_threshold\": 0.0, \"mlperf_auc_threshold\": 0.8025, \"mlperf_bin_loader\": true, \"mlperf_bin_shuffle\": true, \"lr_num_warmup_steps\": 4000, \"lr_decay_start_step\": 5760, \"lr_num_decay_steps\": 27000, \"sparse_dense_boundary\": 403346, \"bf16\": true, \"use_ipex\": true, \"optimizer\": 1, \"lamblr\": 30.0, \"eval_data_path\": \"/home/vmagent/app/dataset/criteo/test_data.bin\", \"day_feature_count\": \"/home/vmagent/app/dataset/criteo/day_fea_count.npz\"}\n",
      "[0] Using IPEX...\n",
      "[1] Using IPEX...\n",
      "[0] Inference complete\n",
      "[1] Inference complete\n"
     ]
    }
   ],
   "source": [
    "! cd e2eaiok/modelzoo/dlrm/; bash run_inference.sh"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "vscode": {
   "interpreter": {
    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}