step0_kaggle_dataset_download.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "kaggle_dataset_download.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BYfNOpiCrUFg"
      },
      "source": [
        "# Downloading datasets from Kaggle using Kaggle API\n",
        "##### @authors: Abhijeet Amle, Roshan Bhandari, Abhimanyu Abhinav"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ra9lhNjzxvsx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 228
        },
        "outputId": "94246d10-2497-4943-9ca9-6425e5e81a71"
      },
      "source": [
        "pip install kaggle"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.6/dist-packages (1.5.8)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.15.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.23.0)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: slugify in /usr/local/lib/python3.6/dist-packages (from kaggle) (0.0.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle) (2020.6.20)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.41.1)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.8.1)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.0.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (3.0.4)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.6/dist-packages (from python-slugify->kaggle) (1.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uW8vCylb_efu"
      },
      "source": [
        "! cd /root/.kaggle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1k-o493T_lSw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "3ae60611-4220-46ed-90a8-9700ed1e801f"
      },
      "source": [
        "pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SOXhYz-S_mBE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "18115150-88d1-46b5-f30a-ddb0c82e69bf"
      },
      "source": [
        "ls -ltra"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 24\n",
            "drwxr-xr-x 1 root root 4096 Oct 14 16:31 \u001b[0m\u001b[01;34msample_data\u001b[0m/\n",
            "drwxr-xr-x 1 root root 4096 Oct 14 16:32 \u001b[01;34m.config\u001b[0m/\n",
            "drwxr-xr-x 1 root root 4096 Oct 24 18:29 \u001b[01;34m..\u001b[0m/\n",
            "drwx------ 4 root root 4096 Oct 24 18:31 \u001b[01;34mdrive\u001b[0m/\n",
            "-rw-r--r-- 1 root root   68 Oct 24 18:31 kaggle.json\n",
            "drwxr-xr-x 1 root root 4096 Oct 24 18:31 \u001b[01;34m.\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7xeenmlW_nv2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "2e18becc-14ea-4672-d5fd-2908c4cefc48"
      },
      "source": [
        "pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s2kdvdFH_olL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "6504d452-18eb-42bd-ab08-3d1ff78cdfb6"
      },
      "source": [
        "cd /"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mOHhQbLY_0H_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527
        },
        "outputId": "2f9d7281-0f97-4776-e552-1af1d329673a"
      },
      "source": [
        "ls -ltra\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 108\n",
            "drwxr-xr-x   2 root root 4096 Apr 24  2018 \u001b[0m\u001b[01;34mhome\u001b[0m/\n",
            "drwxr-xr-x   2 root root 4096 Apr 24  2018 \u001b[01;34mboot\u001b[0m/\n",
            "drwxr-xr-x   2 root root 4096 Sep 21 17:14 \u001b[01;34msrv\u001b[0m/\n",
            "drwxr-xr-x   2 root root 4096 Sep 21 17:14 \u001b[01;34mmnt\u001b[0m/\n",
            "drwxr-xr-x   2 root root 4096 Sep 21 17:14 \u001b[01;34mmedia\u001b[0m/\n",
            "drwxr-xr-x   2 root root 4096 Sep 21 17:16 \u001b[01;34mlib64\u001b[0m/\n",
            "drwxr-xr-x   2 root root 4096 Oct 14 16:20 \u001b[01;34mlib32\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 14 16:21 \u001b[01;34mrun\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 14 16:24 \u001b[01;34msbin\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 14 16:24 \u001b[01;34mbin\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 14 16:26 \u001b[01;34mlib\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 14 16:26 \u001b[01;34musr\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 14 16:26 \u001b[01;34mopt\u001b[0m/\n",
            "drwxr-xr-x   4 root root 4096 Oct 14 16:58 \u001b[01;34mtensorflow-1.15.2\u001b[0m/\n",
            "drwxr-xr-x   4 root root 4096 Oct 14 17:03 \u001b[01;34mswift\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 19 16:16 \u001b[01;34mtools\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 19 16:17 \u001b[01;34mdatalab\u001b[0m/\n",
            "-rwxr-xr-x   1 root root    0 Oct 24 18:29 \u001b[01;32m.dockerenv\u001b[0m*\n",
            "drwxr-xr-x   1 root root 4096 Oct 24 18:29 \u001b[01;34metc\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 24 18:29 \u001b[01;34m..\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 24 18:29 \u001b[01;34m.\u001b[0m/\n",
            "dr-xr-xr-x 124 root root    0 Oct 24 18:29 \u001b[01;34mproc\u001b[0m/\n",
            "dr-xr-xr-x  12 root root    0 Oct 24 18:29 \u001b[01;34msys\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 24 18:29 \u001b[01;34mvar\u001b[0m/\n",
            "drwxr-xr-x   5 root root  360 Oct 24 18:29 \u001b[01;34mdev\u001b[0m/\n",
            "drwxr-xr-x   1 root root 4096 Oct 24 18:31 \u001b[01;34mcontent\u001b[0m/\n",
            "drwxrwxrwt   1 root root 4096 Oct 24 18:32 \u001b[30;42mtmp\u001b[0m/\n",
            "drwx------   1 root root 4096 Oct 24 18:32 \u001b[01;34mroot\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qBgmeyn3_1Gt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "451105f6-a401-48da-8ec7-ed71f0218266"
      },
      "source": [
        "cd root/.kaggle"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/root/.kaggle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sZXqwUkA_40C",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "60af55c2-6cb8-4cc9-c7d3-a1696ceb5f25"
      },
      "source": [
        "pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/root/.kaggle'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EA5FHUqS_5lL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "99a255d0-c365-4ab5-8429-be305b8a2590"
      },
      "source": [
        "cd .."
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/root\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JQj-lZaNsMIx"
      },
      "source": [
        "#### Download Kaggle API token from https://www.kaggle.com/ and upload it to /root/.kaggle directory on Google-colab"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DsC8TX7tAQIi"
      },
      "source": [
        "mv /content/kaggle.json .kaggle/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j00t8FFZAS3L",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "85cc2be9-c9f7-404c-8518-36a2fc804561"
      },
      "source": [
        "cd .kaggle/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/root/.kaggle\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RSdIcKHoAT5k",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "e975383e-919d-489d-cf97-dca4176175ef"
      },
      "source": [
        "ls -tlra\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 16\n",
            "-rw-r--r-- 1 root root   68 Oct 24 18:31 kaggle.json\n",
            "drwx------ 1 root root 4096 Oct 24 18:32 \u001b[0m\u001b[01;34m..\u001b[0m/\n",
            "drwxr-xr-x 2 root root 4096 Oct 24 18:32 \u001b[01;34m.\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4PH43MloAV5p"
      },
      "source": [
        "!kaggle --help"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Uv7Qar-BAmeT"
      },
      "source": [
        "! chmod 600 /root/.kaggle/kaggle.json"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7iPP3tUKtLZK"
      },
      "source": [
        "#### With the help of kaggle libreary we can download public datasets from kaggle"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7UnhzX7yAaXS"
      },
      "source": [
        "!kaggle datasets list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NKQjaq02AvKb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "f3ba6160-50e4-4db9-bfd0-884c4be43014"
      },
      "source": [
        "! kaggle datasets files ramjidoolla/ipl-data-set"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.9 / client 1.5.4)\n",
            "name                               size  creationDate         \n",
            "--------------------------------  -----  -------------------  \n",
            "deliveries.csv                     17MB  2020-09-14 10:57:42  \n",
            "most_runs_average_strikerate.csv   24KB  2020-09-14 10:57:42  \n",
            "teamwise_home_and_away.csv         937B  2020-09-14 10:57:42  \n",
            "matches.csv                       141KB  2020-09-14 10:57:42  \n",
            "teams.csv                          304B  2020-09-14 10:57:42  \n",
            "Players.xlsx                       28KB  2020-09-14 10:57:42  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JEpkf2xDDLPo",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "edaaedee-d41e-40db-dfb9-4380945f29d1"
      },
      "source": [
        "! kaggle datasets download -f matches.csv ramjidoolla/ipl-data-set"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading matches.csv to /root/.kaggle\n",
            "\r  0% 0.00/141k [00:00<?, ?B/s]\n",
            "\r100% 141k/141k [00:00<00:00, 58.5MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D7VnQW3VDfvQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "775bbeaa-8692-4b4c-a14f-8a6a8a052c96"
      },
      "source": [
        "ls -ltra"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 16\n",
            "drwx------ 1 root root 4096 Oct 19 04:49 \u001b[0m\u001b[01;34m..\u001b[0m/\n",
            "-rw------- 1 root root   68 Oct 19 04:50 kaggle.json\n",
            "drwxr-xr-x 2 root root 4096 Oct 19 04:50 \u001b[01;34m.\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4zrvs4KGDjhQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "17ee821d-5288-428a-ddd5-d96eedfd4f9a"
      },
      "source": [
        "! du -sh *"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4.0K\tkaggle.json\n",
            "144K\tmatches.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iZtpcQ6Ah0fM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d768adc6-0068-45c3-8071-2c8e194a1080"
      },
      "source": [
        "dataset = 'ramjidoolla/ipl-data-set'\n",
        "line = 'kaggle datasets files ' + dataset\n",
        "myCmd = line + \" >> output.txt\"\n",
        "os.system(myCmd)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uaL3K9dGJgnV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "2df8803a-f431-439a-863a-a40471c39bd2"
      },
      "source": [
        "ls -ltr"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 148\n",
            "-rw------- 1 root root     68 Oct 11 23:31 kaggle.json\n",
            "-rw-r--r-- 1 root root 144135 Oct 11 23:45 matches.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6u8ge4_GiaVK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "4c8ab3c4-53b0-4848-dd16-51073fbeb4bc"
      },
      "source": [
        "cd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/root\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JO6GkSw4ibaJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "297ee21d-5d53-46cc-c15d-73f5f8b9b9e1"
      },
      "source": [
        "ls -ltr /content/list1.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "-rw-r--r-- 1 root root 26617 Oct 12 02:00 /content/list1.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pyQWsQp0ieOh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d28b90e6-f71f-4304-a0fd-8dc3c5a25009"
      },
      "source": [
        "cd /content/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0bgWaFFsnDKW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "c0a354e9-e327-426d-b788-ef279c2762bf"
      },
      "source": [
        "pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Col5YtjsnDgx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "3504230a-043b-4c01-c4ed-3bc9cee48bac"
      },
      "source": [
        "ls -trl"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 36\n",
            "drwxr-xr-x 1 root root  4096 Oct 14 16:31 \u001b[0m\u001b[01;34msample_data\u001b[0m/\n",
            "drwx------ 4 root root  4096 Oct 20 12:26 \u001b[01;34mdrive\u001b[0m/\n",
            "-rw-r--r-- 1 root root 26617 Oct 20 12:27 list1.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Li-IeXInz-X"
      },
      "source": [
        "import os"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3brUX5InEIO"
      },
      "source": [
        "file1 = open('list1.txt', 'r')\n",
        "#output_file = open(\"output.txt\", \"a\")  # append mo\n",
        "count = 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DfiqGZVonPSS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "outputId": "4daa817b-6b52-429e-8ace-658a814ec4d5"
      },
      "source": [
        "while True: \n",
        "    # Get next line from file \n",
        "    line = file1.readline() \n",
        "  \n",
        "    # if line is empty \n",
        "    # end of file is reached \n",
        "    \n",
        "    if not line:\n",
        "        print(count) \n",
        "        break\n",
        "    line = line.strip()\n",
        "\n",
        "    output_file = open(\"output.txt\", \"a\")  # append mo\n",
        "    output_file.write(\"\\n#####\" + line + \"#####.csv\\n\") \n",
        "    output_file.close()\n",
        "\n",
        "    line = 'kaggle datasets files ' + line\n",
        "    myCmd = line + ' >> output.txt' \n",
        "    os.system(myCmd)\n",
        "    \n",
        "    if (count % 100 == 0):\n",
        "        print(count)\n",
        "    count += 1\n",
        "\n",
        "file1.close()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100\n",
            "200\n",
            "300\n",
            "400\n",
            "500\n",
            "600\n",
            "700\n",
            "800\n",
            "900\n",
            "1000\n",
            "1100\n",
            "1200\n",
            "1300\n",
            "1400\n",
            "1441\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1bBw--QB60NG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "outputId": "da3bd318-611a-4a91-f1fd-30f13f5e1a37"
      },
      "source": [
        "! tail -f output.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.9 / client 1.5.4)\n",
            "name           size  creationDate         \n",
            "-------------  ----  -------------------  \n",
            "Countries.CSV   3KB  2020-05-12 04:24:57  \n",
            "\n",
            "#####roshansharma/online-shoppers-intention#####.csv\n",
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.9 / client 1.5.4)\n",
            "name                           size  creationDate         \n",
            "-----------------------------  ----  -------------------  \n",
            "online_shoppers_intention.csv   1MB  2019-05-23 05:28:15  \n",
            "^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SsquPCpprFDw",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123
        },
        "outputId": "9fa62a65-4ceb-4925-c2a9-9471dd53c446"
      },
      "source": [
        "ls -ltr"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 1888\n",
            "drwxr-xr-x 1 root root    4096 Oct  5 16:31 \u001b[0m\u001b[01;34msample_data\u001b[0m/\n",
            "drwx------ 4 root root    4096 Oct 11 22:34 \u001b[01;34mdrive\u001b[0m/\n",
            "-rw-r--r-- 1 root root   26617 Oct 12 02:00 list1.txt\n",
            "-rw-r--r-- 1 root root     352 Oct 12 02:38 out.txt\n",
            "-rw-r--r-- 1 root root 1884482 Oct 12 02:58 output.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lrg5AhT9wM-l"
      },
      "source": [
        "! cat output.txt | sed 's/\\|/ /' | awk '{print $1}' > output1.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OjSsRXwywYHP"
      },
      "source": [
        "! tail -100f output1.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PYt9WAjcyLzW"
      },
      "source": [
        "! grep -nr \".csv\" output1.txt > output2.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcqRBMBAGnAe"
      },
      "source": [
        "! cat output2.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gVzuOUO50Rbx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 387
        },
        "outputId": "16c9fa63-edf9-4952-d06c-5cc37f0a86f1"
      },
      "source": [
        "! tail -20f output2.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "123859:data_eeg_age_v1/data2kaggle/eval/00002508_s002_t001.csv\n",
            "123860:data_eeg_age_v1/data2kaggle/eval/00005458_s001_t000.csv\n",
            "123861:data_eeg_age_v1/data2kaggle/eval/00005942_s002_t000.csv\n",
            "123862:data_eeg_age_v1/data2kaggle/eval/00002272_s002_t000.csv\n",
            "123863:data_eeg_age_v1/data2kaggle/eval/00005473_s001_t000.csv\n",
            "123864:data_eeg_age_v1/data2kaggle/eval/00002940_s003_t001.csv\n",
            "123865:data_eeg_age_v1/data2kaggle/eval/00001402_s004_t000.csv\n",
            "123866:data_eeg_age_v1/data2kaggle/eval/00006005_s001_t001.csv\n",
            "123867:data_eeg_age_v1/data2kaggle/eval/00005412_s001_t000.csv\n",
            "123868:data_eeg_age_v1/data2kaggle/eval/00005390_s001_t000.csv\n",
            "123869:data_eeg_age_v1/data2kaggle/eval/00000647_s002_t000.csv\n",
            "123870:data_eeg_age_v1/data2kaggle/eval/00005221_s002_t000.csv\n",
            "123871:data_eeg_age_v1/data2kaggle/eval/00005468_s001_t000.csv\n",
            "123872:data_eeg_age_v1/data2kaggle/eval/00005398_s001_t000.csv\n",
            "123873:data_eeg_age_v1/data2kaggle/eval/00004586_s002_t001.csv\n",
            "123876:#####kyanyoga/sample-sales-data#####.csv\n",
            "123880:sales_data_sample.csv\n",
            "123882:#####smid80/coronavirus-covid19-tweets#####.csv\n",
            "123888:#####roshansharma/online-shoppers-intention#####.csv\n",
            "123892:online_shoppers_intention.csv\n",
            "^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gWjp_T4H0Taq"
      },
      "source": [
        "! cat output2.txt | cut -d':' -f2 > output3.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yJu5ga3VHXZa"
      },
      "source": [
        "! cat output3.txt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pjo_TQQdIMpN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 387
        },
        "outputId": "9fbeb08b-aa36-4276-ea6b-19d7e3ee8913"
      },
      "source": [
        "! tail -20f output3.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "data_eeg_age_v1/data2kaggle/eval/00002508_s002_t001.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005458_s001_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005942_s002_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00002272_s002_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005473_s001_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00002940_s003_t001.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00001402_s004_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00006005_s001_t001.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005412_s001_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005390_s001_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00000647_s002_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005221_s002_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005468_s001_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00005398_s001_t000.csv\n",
            "data_eeg_age_v1/data2kaggle/eval/00004586_s002_t001.csv\n",
            "#####kyanyoga/sample-sales-data#####.csv\n",
            "sales_data_sample.csv\n",
            "#####smid80/coronavirus-covid19-tweets#####.csv\n",
            "#####roshansharma/online-shoppers-intention#####.csv\n",
            "online_shoppers_intention.csv\n",
            "^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J-qNNvj_ttvA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "31c1434a-7235-4be9-cb9c-d575fe12db40"
      },
      "source": [
        "line = \"#####zynicide/wine-reviews#####.csv   \"\n",
        "line.split(\"#####\")[1]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'zynicide/wine-reviews'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eQO4TkgTIYoV"
      },
      "source": [
        "import os\n",
        "file_op = open('output3.txt', 'r') \n",
        "Lines = file_op.readlines() "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V_TmNeZjuVnF"
      },
      "source": [
        "#### Downloading from different pages of Kaggle datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_-1BGwnVuQ50"
      },
      "source": [
        "import os\n",
        "for i in range(0, 151):\n",
        "    cmd = \"kaggle datasets list -p \" + str(i) + \" | sed 's/\\|/ /'|awk '{print $1}' >> list.txt\"\n",
        "    os.system(cmd)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GT_zhOLRuiG-"
      },
      "source": [
        "! kaggle datasets download -f matches.csv ramjidoolla/ipl-data-set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S1ChnQy7reID"
      },
      "source": [
        "count = 1\n",
        "dataset = \"\"\n",
        "filename = \"\"\n",
        "for line in Lines:\n",
        "\n",
        "    if line.startswith( '#####', 0, 5):\n",
        "        line = line.strip()\n",
        "        dataset = line.split(\"#####\")[1]\n",
        "    \n",
        "    else:\n",
        "        filename = line.strip()\n",
        "        cmd = \"kaggle datasets download -f \" + filename + \" \" + dataset\n",
        "        os.system(cmd)\n",
        "    \n",
        "    if (count % 100 == 0):\n",
        "        print(count)\n",
        "        \n",
        "    count += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_-QdkXN0ndj"
      },
      "source": [
        "! mkdir -p ramjidoolla/ipl-data-set"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qITvGRVo0x8d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 140
        },
        "outputId": "63817172-8729-4391-e003-ea54ee1dd080"
      },
      "source": [
        "! ls -lrta"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 24\n",
            "drwxr-xr-x 1 root root 4096 Oct 14 16:31 sample_data\n",
            "drwxr-xr-x 1 root root 4096 Oct 14 16:32 .config\n",
            "drwxr-xr-x 1 root root 4096 Oct 23 23:15 ..\n",
            "drwx------ 4 root root 4096 Oct 23 23:19 drive\n",
            "drwxr-xr-x 3 root root 4096 Oct 23 23:37 ramjidoolla\n",
            "drwxr-xr-x 1 root root 4096 Oct 23 23:37 .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fdxmbwKd00tv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0e2e0b86-47b2-4987-854c-08c81dde2700"
      },
      "source": [
        "cd ramjidoolla/"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/ramjidoolla\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N6sC1m2V03Cx"
      },
      "source": [
        "!cd /content/ramjidoolla/ipl-data-set/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRrMGYYX1BSM"
      },
      "source": [
        "! cd ipl-data-set/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U0OY1QWM1U4B",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8464bbee-2450-45e9-ac35-1742b8afd407"
      },
      "source": [
        "pwd"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/ramjidoolla'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9oyS_Psf1Fgc"
      },
      "source": [
        "! touch abc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "To4IrVvn1G9C",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        },
        "outputId": "74f1f0fa-664b-4d8b-a8a0-5aca7cedd17e"
      },
      "source": [
        "ls -tlra"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 12\n",
            "drwxr-xr-x 2 root root 4096 Oct 23 23:37 \u001b[0m\u001b[01;34mipl-data-set\u001b[0m/\n",
            "drwxr-xr-x 1 root root 4096 Oct 23 23:37 \u001b[01;34m..\u001b[0m/\n",
            "drwxr-xr-x 3 root root 4096 Oct 23 23:38 \u001b[01;34m.\u001b[0m/\n",
            "-rw-r--r-- 1 root root    0 Oct 23 23:40 abc\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GwkzPNB2whBt",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "c545b26a-2562-4943-ed1f-cf37a2524fff"
      },
      "source": [
        "! kaggle datasets download -f matches.csv ramjidoolla/ipl-data-set"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "usage: kaggle [-h] [-v] {competitions,c,datasets,d,kernels,k,config} ...\n",
            "kaggle: error: unrecognized arguments: /content/ramjidoolla/ipl-data-set/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f_jcJKVy1cF5"
      },
      "source": [
        "cmd = \"mkdir -p ramjidoolla/ipl-data-set \\n cd ramjidoolla/ipl-data-set\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gJz2s29J7emE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "a8a690eb-e43f-47e3-cda3-ad202f94df54"
      },
      "source": [
        "os.system(cmd)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Gcr6vua7kD4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 211
        },
        "outputId": "7b7da627-703f-4a09-a2c4-708d952c450a"
      },
      "source": [
        "! tail -f list.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "testdotai/common-mobile-web-app-icons\n",
            "rogeriochaves/boatos-de-whatsapp-boatosorg\n",
            "sdorius/countryses\n",
            "roamresearch/prescriptionbasedprediction\n",
            "juliancienfuegos/what-is-a-note\n",
            "jboysen/scientist-migrations\n",
            "jobspikr/50000-job-board-record-from-reed-uk\n",
            "chicago/chicago-energy-usage-2010\n",
            "jolasa/waves-measuring-buoys-data-mooloolaba\n",
            "therohk/global-news-week\n",
            "^C\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g1qFRnqG4pmf"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}