test_strings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# activate crawler env 1st!\n",
    "import netCDF4 as nc\n",
    "from netCDF4 import Dataset\n",
    "import thredds_crawler\n",
    "from thredds_crawler.crawl import Crawl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "My first try with a single directory worked great. I can loop through the files the crawler finds, per the example on git-hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fps='https://gamone.whoi.edu/thredds/silt/usgs/Projects/stellwagen/Data/DAUPHIN/catalog.xml'\n",
    "if type(fps) is str:\n",
    "    print('is a string')\n",
    "    c = Crawl(fps, select=[\".*nc\"]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c.datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "However, when I try to loop throught more than 1 directory, it doesn't seem to think the string is something it can operate on in crawl.  I've tried with various os* options, glob, and I can't seem to fabricate a way to loop through more than 1 catalog.\n",
    "\n",
    "Below are 2 ways that work- the second is preferable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dirlist=('ARGO_MERCHANT','BARNEGAT','BARNEGAT_2014','BUZZ_BAY','BW_2011')\n",
    "#dirlist=('CALSCTWET','CAMP','CAPE_COD_BAY','CC_MISC','CHANDELEUR_13','CHINCOTEAGUE')\n",
    "#dirlist=('DAUPHIN','DEEP_REEF','DIAMONDSHOALS','DWDS_106','ECOHAB_I','ECOHAB_II','EUROSTRATAFORM')\n",
    "dirlist=('DAUPHIN','DEEP_REEF')  \n",
    "proot='https://gamone.whoi.edu/thredds/silt/usgs/Projects/stellwagen/Data/'\n",
    "sufx='/catalog.xml'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx in dirlist:\n",
    "    #fpath=[proot + idx + sufx]     # this usage with the [] makes a list not a string\n",
    "    fpath=proot + idx + sufx\n",
    "    if type(fpath) is str:\n",
    "        print(fpath, 'is a string')\n",
    "    # this syntax works on both linux and Windows; \n",
    "    # but be careful the directory has .cdf if that's what you specified.\n",
    "    # also note, each cnew has to be treated within the loop\n",
    "    cn = Crawl(fpath, select=[\".*.nc\"])\n",
    "    # Syntax like select=[\".*cdf\", \"*.nc\"] or select=[\".*cdf\" | \"*.nc\"] doesn't work!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#type(cnew)     # returns thredds_crawler.crawl.Crawl\n",
    "cn.datasets  # only displays the content of the last item in dirlist, since out of the loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del c, cn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is the better way- it gets everything and puts int 1 big cnew entry\n",
    "catalog='https://gamone.whoi.edu/thredds/silt/usgs/Projects/stellwagen/Data/catalog.xml'\n",
    "dirlist=('DWDS_106','DEEP_REEF')\n",
    "# DWDS_106 has .cdf and DEEP_REEF has .nc\n",
    "cnew = Crawl(catalog, select=[\".*{}.*\".format(x) for x in dirlist])\n",
    "cnew.datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:crawler]",
   "language": "python",
   "name": "conda-env-crawler-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}