Ex_03_Wide.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Ex-03-Wide.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_0y0i6xAfXiE"
      },
      "source": [
        "### Sanjoy Biswas"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1dNjOuZITmvJ"
      },
      "source": [
        "#### Setup Hadoop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LnTS_GV50yl3"
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://downloads.apache.org/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!tar xf spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vYgGaoSTG8tq",
        "outputId": "dba8ea70-b4ea-44ab-c499-07f718018ef4"
      },
      "source": [
        "!pip install pyspark"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.7/dist-packages (3.1.2)\n",
            "Requirement already satisfied: py4j==0.10.9 in /usr/local/lib/python3.7/dist-packages (from pyspark) (0.10.9)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "44SmOaWfT4lv"
      },
      "source": [
        "#### Import Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "67C8z90tO0pP"
      },
      "source": [
        "from pyspark import SparkContext, SparkConf"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17KqWp8pQrLI"
      },
      "source": [
        "conf = SparkConf().setAppName('Youtube').setMaster('local')\n",
        "sc = SparkContext(conf = conf)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HzynGZzslyYw",
        "outputId": "544c8884-4e98-46f1-80b2-1cae7ec6d1cc"
      },
      "source": [
        "names = sc.parallelize(['Java','Python','Javascript','Perl'])\n",
        "names.collect()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['Java', 'Python', 'Javascript', 'Perl']"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9lRXnfD8lyhI",
        "outputId": "2ee54cce-c198-4a96-a4bb-5870a6d80dba"
      },
      "source": [
        "names_gr = names.groupBy(lambda x : x[0])\n",
        "names_gr.collect()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('J', <pyspark.resultiterable.ResultIterable at 0x7f30455dee90>),\n",
              " ('P', <pyspark.resultiterable.ResultIterable at 0x7f3045627d50>)]"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_LLzQxo_lyko",
        "outputId": "51c370fc-07f6-467f-d605-2a91e1cc5a02"
      },
      "source": [
        "for (k,v) in names_gr.collect():\n",
        "  print(k,list(v))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "J ['Java', 'Javascript']\n",
            "P ['Python', 'Perl']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yf6wqLKjlynY",
        "outputId": "d25089b3-a7cf-4ee9-9c72-8c8efc1c58a1"
      },
      "source": [
        "aa = sc.parallelize([1,2,3,4,5,6,7,8,6])\n",
        "aa.collect()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 2, 3, 4, 5, 6, 7, 8, 6]"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OQmP8WS1lysv",
        "outputId": "4f263493-4516-4151-c4b4-e85b462d1536"
      },
      "source": [
        "result = aa.groupBy(lambda x :x%2)\n",
        "result.collect()"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[(1, <pyspark.resultiterable.ResultIterable at 0x7f3049773fd0>),\n",
              " (0, <pyspark.resultiterable.ResultIterable at 0x7f30497733d0>)]"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uTYMDm3wlyvn",
        "outputId": "8f81ba12-5425-4f7a-a612-7f5df6082a8a"
      },
      "source": [
        "for (k,v) in result.collect():\n",
        "  print(k,list(v))"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1 [1, 3, 5, 7]\n",
            "0 [2, 4, 6, 8, 6]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0hvFaYoxomHR"
      },
      "source": [
        "#### Intersection"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "emz-zSuilyx5",
        "outputId": "cbb28bc3-0c1c-459a-9378-2ffef56ca4d6"
      },
      "source": [
        "num1 = sc.parallelize([1,2,3,4,7])\n",
        "num1.collect()"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 2, 3, 4, 7]"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "59d4guFSo4OR",
        "outputId": "cdeadc9b-bb17-451e-a35a-4bfd6d17d0db"
      },
      "source": [
        "num2 = sc.parallelize([0,2,3,5,9])\n",
        "num2.collect()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0, 2, 3, 5, 9]"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jb4ktimfly0Y",
        "outputId": "b780d807-3a77-45d3-f08f-9b2a41a82898"
      },
      "source": [
        "num1.intersection(num2).collect()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[2, 3]"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5KyBWbcIpUDo"
      },
      "source": [
        "#### Substract"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IWCgq-Tfly3O",
        "outputId": "54dcae47-9719-46b3-9b02-c65f11517d47"
      },
      "source": [
        "num1.subtract(num2).collect()"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[4, 1, 7]"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8tOdelRDptGA"
      },
      "source": [
        "#### Distinct"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aPfgY37Gly5v",
        "outputId": "5ecb05be-9da3-411b-fda6-89e9b1a60d5a"
      },
      "source": [
        "num1.distinct().collect()"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 2, 3, 4, 7]"
            ]
          },
          "metadata": {},
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JghTwyJely8b"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}