scripts/01_download_secc.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download SECC Data from Dataverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dataverse API Token\n",
    "# https://dataverse.harvard.edu/dataverseuser.xhtml?selectTab=apiTokenTab\n",
    "token = '<API TOKEN>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyDataverse.api import Api\n",
    "from pyDataverse.models import Dataverse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'OK'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "api = Api('https://dataverse.harvard.edu/', token)\n",
    "api.status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "resp = api.get_dataset('doi:10.7910/DVN/LIIBNB')\n",
    "j = resp.json()\n",
    "j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3298920 secc_01_clean.7z\n",
      "3298923 secc_02_clean.7z\n",
      "3298943 secc_03_clean.7z\n",
      "3298947 secc_04_clean.7z\n",
      "3298926 secc_05_clean.7z\n",
      "3298936 secc_06_clean.7z\n",
      "3298929 secc_07_clean.7z\n",
      "3298945 secc_08_clean.7z\n",
      "3298935 secc_09_clean.7z\n",
      "3298925 secc_10_clean.7z\n",
      "3298918 secc_11_clean.7z\n",
      "3298916 secc_12_clean.7z\n",
      "3298922 secc_13_clean.7z\n",
      "3298940 secc_14_clean.7z\n",
      "3298930 secc_18_clean.7z\n",
      "3298924 secc_19_clean.7z\n",
      "3298921 secc_20_clean.7z\n",
      "3298919 secc_22_clean.7z\n",
      "3298915 secc_23_clean.7z\n",
      "3298934 secc_24_clean.7z\n",
      "3298944 secc_25_clean.7z\n",
      "3298928 secc_26_clean.7z\n",
      "3298948 secc_27_clean.7z\n",
      "3298941 secc_28_clean.7z\n",
      "3298946 secc_29_clean.7z\n",
      "3298933 secc_30_clean.7z\n",
      "3298937 secc_31_clean.7z\n",
      "3298939 secc_32_clean.7z\n",
      "3298931 secc_33_clean.7z\n",
      "3298932 secc_34_clean.7z\n",
      "3298917 secc_35_clean.7z\n",
      "3298927 secc_36_clean.7z\n",
      "3298938 secc_37_clean.7z\n",
      "3298942 secc_38_clean.7z\n",
      "3298951 secc_39_clean.7z\n",
      "3298952 secc_40_clean.7z\n",
      "3298953 secc_41_clean.7z\n",
      "3298954 secc_42_clean.7z\n",
      "3298955 secc_43_clean.7z\n",
      "3298956 secc_44_clean.7z\n",
      "3298957 secc_45_clean.7z\n",
      "3298958 secc_46_clean.7z\n",
      "3298959 secc_47_clean.7z\n",
      "3298950 secc_48_clean.7z\n",
      "3319981 ssec_15_not_clean.7z\n",
      "3319982 ssec_17_not_clean.7z\n"
     ]
    }
   ],
   "source": [
    "files = []\n",
    "for file in j['data']['latestVersion']['files']:\n",
    "    fid = file['dataFile']['id']\n",
    "    fn = file['dataFile']['filename']\n",
    "    print(fid, fn)\n",
    "    files.append((fid, fn))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "def download_file(url, fn):\n",
    "    local_filename = fn # url.split('/')[-1]\n",
    "    # NOTE the stream=True parameter below\n",
    "    with requests.get(url, stream=True) as r:\n",
    "        r.raise_for_status()\n",
    "        with open(local_filename, 'wb') as f:\n",
    "            for chunk in r.iter_content(chunk_size=8192): \n",
    "                if chunk: # filter out keep-alive new chunks\n",
    "                    f.write(chunk)\n",
    "                    # f.flush()\n",
    "    return local_filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "secc_01_clean.7z\n",
      "secc_02_clean.7z\n",
      "secc_03_clean.7z\n",
      "secc_04_clean.7z\n",
      "secc_05_clean.7z\n",
      "secc_06_clean.7z\n",
      "secc_07_clean.7z\n",
      "secc_08_clean.7z\n",
      "secc_09_clean.7z\n",
      "secc_10_clean.7z\n",
      "secc_11_clean.7z\n",
      "secc_12_clean.7z\n",
      "secc_13_clean.7z\n",
      "secc_14_clean.7z\n",
      "secc_18_clean.7z\n",
      "secc_19_clean.7z\n",
      "secc_20_clean.7z\n",
      "secc_22_clean.7z\n",
      "secc_23_clean.7z\n",
      "secc_24_clean.7z\n",
      "secc_25_clean.7z\n",
      "secc_26_clean.7z\n",
      "secc_27_clean.7z\n",
      "secc_28_clean.7z\n",
      "secc_29_clean.7z\n",
      "secc_30_clean.7z\n",
      "secc_31_clean.7z\n",
      "secc_32_clean.7z\n",
      "secc_33_clean.7z\n",
      "secc_34_clean.7z\n",
      "secc_35_clean.7z\n",
      "secc_36_clean.7z\n",
      "secc_37_clean.7z\n",
      "secc_38_clean.7z\n",
      "secc_39_clean.7z\n",
      "secc_40_clean.7z\n",
      "secc_41_clean.7z\n",
      "secc_42_clean.7z\n",
      "secc_43_clean.7z\n",
      "secc_44_clean.7z\n",
      "secc_45_clean.7z\n",
      "secc_46_clean.7z\n",
      "secc_47_clean.7z\n",
      "secc_48_clean.7z\n",
      "ssec_15_not_clean.7z\n",
      "ssec_17_not_clean.7z\n"
     ]
    }
   ],
   "source": [
    "for fid, fn in files:\n",
    "    print(fn)\n",
    "    url = 'https://dataverse.harvard.edu//api/v1/access/datafile/%s?key=%s' % (fid, token)\n",
    "    download_file(url, fn)\n",
    "    #break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}