Download_AWS_WSR_88D_data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tjturnage/radar/blob/main/Download_AWS_WSR_88D_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### **This script allows the user to download selected WSR-88D Archive Level 2 files from Amazon Web Services**\n",
        "\n",
        "Retrieved files will be placed into a zip file shown to left that can then be downloaded.\n",
        "\n",
        "Authors:\n",
        "* Eric Lenning -- NWS Chicago -- @eric.lenning@noaa.gov\n",
        "* TJ Turnage -- NWS Grand Rapids -- @thomas.turnage@noaa.gov\n"
      ],
      "metadata": {
        "id": "7a0Xhbp1cq8q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title Install the necessary modules\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "!pip install s3fs &> /dev/null\n",
        "import s3fs\n",
        "import sys\n",
        "import os\n",
        "import math\n",
        "from datetime import timedelta\n",
        "from datetime import datetime\n",
        "import numpy as np\n",
        "\n",
        "class NexradLevel2():\n",
        "\n",
        "    def __init__(self, site, start_datetime, end_datetime):\n",
        "        self.site = site\n",
        "        self.start_datetime = start_datetime\n",
        "        self.end_datetime = end_datetime\n",
        "\n",
        "    def daterange(self):\n",
        "        \"\"\" Yields list of dates within specified time range. \"\"\"\n",
        "\n",
        "        span = self.end_datetime - self.start_datetime\n",
        "\n",
        "        day_span = math.ceil(float(span.total_seconds()) / (3600*24))\n",
        "\n",
        "        if self.start_datetime.hour > self.end_datetime.hour:\n",
        "            day_span += 1\n",
        "\n",
        "        for n in range(day_span):\n",
        "            yield self.start_datetime + timedelta(n)\n",
        "\n",
        "    def filelist(self):\n",
        "        \"\"\" List of files for site within specified time range. \"\"\"\n",
        "\n",
        "        fs = s3fs.S3FileSystem(anon=True)\n",
        "        fs.ls('s3://noaa-nexrad-level2/')\n",
        "\n",
        "        radarfiles = []\n",
        "\n",
        "        for single_date in self.daterange():\n",
        "\n",
        "            YYYY = single_date.year  # strftime(\"%Y\")\n",
        "            mm = single_date.month  # strftime(\"%m\")\n",
        "            dd = single_date.day  # strftime(\"%d\")\n",
        "\n",
        "            # sample bucket dir : 'noaa-nexrad-level2/2018/07/19/KDMX/'\n",
        "            # broken into two substrings to stay <79 columns\n",
        "            bucket_str1 = f'noaa-nexrad-level2/{YYYY:.0f}/'\n",
        "            bucket_str2 = f'{mm:02.0f}/{dd:02.0f}/{self.site}/'\n",
        "            bucket_dir_str = bucket_str1 + bucket_str2\n",
        "            # list available files in bucket\n",
        "            # sample filename :  KDMX20180719_221153_V06\n",
        "            files = np.array(fs.ls(bucket_dir_str))\n",
        "\n",
        "            for f in range(0, len(files)):\n",
        "\n",
        "                filename = files[f].split('/')[-1]  # extracts fname after /\n",
        "\n",
        "                if 'MDM' not in filename:\n",
        "                    file_date = filename[4:19]\n",
        "\n",
        "                    try:\n",
        "                        file_datetime = datetime.strptime(file_date,\n",
        "                                                          '%Y%m%d_%H%M%S')\n",
        "\n",
        "                        if (file_datetime >= self.start_datetime\n",
        "                           and file_datetime <= self.end_datetime):\n",
        "\n",
        "                            print(files[f])\n",
        "\n",
        "                            radarfiles.append(files[f])\n",
        "\n",
        "                            info = fs.info(files[f])\n",
        "\n",
        "                            #print(\"size \", info['size'])\n",
        "\n",
        "                            #print(info)\n",
        "\n",
        "                    except Exception:\n",
        "                        print(\"Unexpected error:\", sys.exc_info()[0])\n",
        "\n",
        "        return radarfiles\n",
        "\n",
        "    def download(self, filelist):\n",
        "        \"\"\"Download level 2 radar files from AWS.\n",
        "\n",
        "        Files will be named according to the format on AWS but assumes nothing\n",
        "        about the destination folder.  self is because different users may\n",
        "        have different requirements for where data should be stored.\n",
        "\n",
        "        Args:\n",
        "            filelist: A list of files in the AWS NEXRAD inventory.\n",
        "            raw_data_dir: Full pathname of download destination.\n",
        "\n",
        "        Returns:\n",
        "            The return value. True for success, False otherwise.\n",
        "        \"\"\"\n",
        "\n",
        "        #os.makedirs(raw_data_dir, exist_ok=True)\n",
        "\n",
        "        fs = s3fs.S3FileSystem(anon=True)\n",
        "\n",
        "        # https://noaa-nexrad-level2.s3.amazonaws.com/\n",
        "        #           2015/04/10/KLOT/KLOT20150410_235635_V06.gz\n",
        "        # sample filename :  KDMX20180719_221153_V06\n",
        "\n",
        "        # downloaded = []\n",
        "\n",
        "        download_count = 0\n",
        "\n",
        "        # sample source filepath\n",
        "        # 'noaa-nexrad-level2/2018/07/19/KDMX/KDMX20180719_221153_V06'\n",
        "        for f in range(0, len(filelist)):\n",
        "            try:\n",
        "                print('getting... ' + str(filelist[f]))\n",
        "                dst_filepath = os.path.join('radarfiles', filelist[f].split('/')[-1])\n",
        "\n",
        "                #print('...to ', dst_filepath)\n",
        "\n",
        "                info = fs.info(filelist[f])\n",
        "\n",
        "                remote_filesize = info['size']\n",
        "\n",
        "                try:\n",
        "                    stat = os.stat(dst_filepath)\n",
        "                    #print(stat)\n",
        "                    local_filesize = stat.st_size\n",
        "\n",
        "                    print('remote and local filesize: ',\n",
        "                          remote_filesize, local_filesize)\n",
        "\n",
        "                    if local_filesize < remote_filesize:\n",
        "                        raise Exception('File exists but is too small.')\n",
        "\n",
        "                    print('Already downloaded.')\n",
        "                    download_count += 1\n",
        "                except Exception:\n",
        "                    fs.get(filelist[f], dst_filepath)  # download to dest dir\n",
        "                    print('  Download complete!')\n",
        "                    download_count += 1\n",
        "                    # downloaded.append(filelist[f])\n",
        "            except Exception:\n",
        "                pass\n",
        "\n",
        "        # Tell caller if you got all the files it wanted.\n",
        "        if download_count == len(filelist):\n",
        "            return True\n",
        "        else:\n",
        "            return False\n"
      ],
      "metadata": {
        "id": "8c30vxDtAUmX",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# @markdown <br><font color=\"blue\"><h2>Select the radar to download data for</h2></font>\n",
        "radarId = \"KGRR\" # @param {type:\"string\"}\n",
        "\n",
        "# @markdown <br><font color=\"blue\"><h2>Enter the start time (in UTC) for the radar data download</h2></font>\n",
        "start_ymd = \"2024-02-28\" # @param {type:\"date\"}\n",
        "start_hour = \"03\" # @param [\"00\",\"01\",\"02\",\"03\",\"04\",\"05\",\"06\",\"07\",\"08\",\"09\",\"10\",\"11\",\"12\",\"13\",\"14\",\"15\",\"16\",\"17,\",\"18\",\"19\",\"20\",\"21\",\"22\",\"23\"]\n",
        "start_minute = \"00\" # @param [\"00\", \"05\", \"10\", \"15\", \"20\", \"25\", \"30\", \"35\", \"40\", \"45\", \"50\", \"55\"]\n",
        "start_time_str = f\"{start_ymd} {start_hour} {start_minute}\"\n",
        "start_date = datetime.strptime(start_time_str, \"%Y-%m-%d %H %M\")\n",
        "\n",
        "\n",
        "# @markdown <br><font color=\"blue\"><h2>Enter the end time (in UTC) for the radar data download</h2></font>\n",
        "end_ymd = \"2024-02-28\" # @param {type:\"date\"}\n",
        "end_hour = \"06\" # @param [\"00\",\"01\",\"02\",\"03\",\"04\",\"05\",\"06\",\"07\",\"08\",\"09\",\"10\",\"11\",\"12\",\"13\",\"14\",\"15\",\"16\",\"17,\",\"18\",\"19\",\"20\",\"21\",\"22\",\"23\"]\n",
        "end_minute = \"45\" # @param [\"00\", \"05\", \"10\", \"15\", \"20\", \"25\", \"30\", \"35\", \"40\", \"45\", \"50\", \"55\"]\n",
        "\n",
        "end_time_str = f\"{end_ymd} {end_hour} {end_minute}\"\n",
        "end_date = datetime.strptime(end_time_str, \"%Y-%m-%d %H %M\")\n",
        "\n",
        "# datetime(year, month, day, hour, minute)\n",
        "# example: datetime(2023, 6, 16, 3, 0)\n",
        "\n",
        "print(f\"start time: {start_date}\")\n",
        "print(f\"  end time: {end_date}\")\n"
      ],
      "metadata": {
        "id": "jvToTQQof7u-",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "<h3>Running the code below will download files to a \"radarfiles\" directory that is then made into a \"radarfiles.zip\" file that can be downloaded. If you don't see these appear right away, click the folder refresh icon at the top of the Files window.</h3>\n",
        "\n",
        "<h4><i>- Right-click over \"radarfiles.zip\" and select Download <br>- There will be a delay as the zip file preps for download<br>- You can monitor progress with the growing circle on the right</i></h4>"
      ],
      "metadata": {
        "id": "8IIeB7ZlgRNN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# @title\n",
        "example = NexradLevel2(radarId, start_date, end_date)\n",
        "example.download(example.filelist())\n",
        "\n",
        "!zip -r /content/radarfiles.zip /content/radarfiles"
      ],
      "metadata": {
        "id": "8c3QfmKdfl4-",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}