DM3_to_H5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DM3 Translator\n",
    "* This file is used to translate the raw DM3 data file into standard H5 file. \n",
    "\n",
    "* The data will be initially processed and stored in the same file for future analysis. \n",
    " - The process includes: ZLP alignment, Thickness computing, ZLP substraction. \n",
    " - Please be aware that the data is limited to low-loss data so far. <br>\n",
    " - If the raw data is core-loss signal, please include additional low-loss data for the ZLP alignment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Software:\n",
    "* Standard distribution (include numpy, scipy, matplotlib, sci-kit learn).\n",
    "* **hyperspy** : can load and process DM3 data directly. (import hyperspy.api, not hyperspy)\n",
    "* **pycroscopy** : mainly used here for plotting purposes only. \n",
    "* **hyperspy_tools** : to extract EELS zlp."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:hyperspy.api:The ipywidgets GUI elements are not available, probably because the hyperspy_gui_ipywidgets package is not installed.\n",
      "WARNING:hyperspy.api:The traitsui GUI elements are not available, probably because the hyperspy_gui_traitui package is not installed.\n"
     ]
    }
   ],
   "source": [
    "#Import packages\n",
    "\n",
    "# Ensure that this code works on both python 2 and python 3\n",
    "from __future__ import division, print_function, absolute_import, unicode_literals\n",
    "\n",
    "# basic numeric computation:\n",
    "import numpy as np\n",
    "\n",
    "# basic system operation:\n",
    "import os\n",
    "\n",
    "# The package used for creating and manipulating HDF5 files:\n",
    "import h5py\n",
    "import hyperspy.api as hs\n",
    "\n",
    "# To extract EELS ZLP, put 'eels.py' from hyperspy_tools in the python path and then import here:\n",
    "from eels import extract_ZLP\n",
    "\n",
    "# Plotting and visualization:\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.axes_grid1 import make_axes_locatable\n",
    "\n",
    "# multivariate analysis:\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.decomposition import NMF\n",
    "\n",
    "# finally import pycroscopy:\n",
    "import pycroscopy as px"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load DM3 data:\n",
    "* Use **hyperspy** to read DM3 data.\n",
    "* Reshape the data as h5 file required.\n",
    "## Tips:\n",
    "* The 'np.reshape' will reshape the matrix acoording to the C memory order, i.e. the most rapidly changing index comes last: (k,j,i)\n",
    "* The h5 file for pycroscopy requires the 3D EELS data (i,j,k) to be flattened as 2D (i*j, k) in the C memory order as well, i.e. the order of row in the flattened data is [0,0],[0,1]...[0,N-1],[1,0],[1,1] ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:hyperspy.signal:<EELSSpectrum, title: EELS Spectrum Image2, dimensions: (19, 18|1340)> data is replaced by its optimized copy\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(18, 19, 1340)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "├── Acquisition_instrument\n",
       "│   └── TEM\n",
       "│       ├── Detector\n",
       "│       │   └── EELS\n",
       "│       │       ├── collection_angle = 50.0\n",
       "│       │       └── frame_number = 1\n",
       "│       ├── acquisition_mode = STEM\n",
       "│       ├── beam_current = 0.0\n",
       "│       ├── beam_energy = 80.0\n",
       "│       ├── camera_length = 400.0\n",
       "│       ├── convergence_angle = 20.0\n",
       "│       ├── magnification = 2500000.0\n",
       "│       └── microscope = JEOL COM\n",
       "├── General\n",
       "│   ├── date = 2016-05-07\n",
       "│   ├── original_filename = EELS Spectrum Image.dm3\n",
       "│   ├── time = 21:32:29\n",
       "│   └── title = EELS Spectrum Image2\n",
       "└── Signal\n",
       "    ├── Noise_properties\n",
       "    │   └── Variance_linear_model\n",
       "    │       ├── gain_factor = 0.20789135992527008\n",
       "    │       └── gain_offset = 0.0\n",
       "    ├── binned = True\n",
       "    ├── quantity = Electrons (Counts)\n",
       "    └── signal_type = EELS"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "folder_path=''\n",
    "file_name='EELS Spectrum Image.dm3'\n",
    "raw = hs.load(folder_path+file_name,signal_type='EELS')\n",
    "data = raw.data\n",
    "np.shape(data)\n",
    "raw.metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count = 0\n",
    "data_shape = np.shape(data)\n",
    "ind_mat = np.zeros(data_shape)\n",
    "for (ind1,ind2,ind3), val in np.ndenumerate(data):\n",
    "    ind_mat[ind1,ind2,ind3] = count\n",
    "    count += 1\n",
    "#print(np.shape(data))\n",
    "\n",
    "h5_data = np.reshape(data,(-1,data_shape[2]))\n",
    "h5_ind = np.reshape(ind_mat,(-1,data_shape[2]))\n",
    "##present the ordering:\n",
    "#make a function to transfer count of the memory order to 3D position.\n",
    "#def getindex(number):\n",
    "#    id1 = number//(data_shape[2]*data_shape[1])\n",
    "#    res1 = number%(data_shape[2]*data_shape[1])\n",
    "#    id2 = res1//data_shape[2]\n",
    "#    id3 = res1 - id2*data_shape[2]\n",
    "#    return (id1,id2,id3)\n",
    "\n",
    "#for i in range(30):\n",
    "#    print(getindex(h5_ind[i,0]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Ancillary Data and Translate into H5 file:\n",
    "* Create the new file path.\n",
    "* Specify the spectroscopic axis: minimum of the axis, and dispersion\n",
    "* Translate the data into h5 file using **NumpyTranslator**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The h5 file will be saved in the following path:\n",
      "EELS Spectrum Image.h5\n",
      "├── TagGroup0\n",
      "│   ├── Origin = 0.0\n",
      "│   ├── Scale = 0.002550000324845314\n",
      "│   └── Units = µm\n",
      "├── TagGroup1\n",
      "│   ├── Origin = 0.0\n",
      "│   ├── Scale = 0.002550000324845314\n",
      "│   └── Units = µm\n",
      "└── TagGroup2\n",
      "    ├── Origin = 100.0\n",
      "    ├── Scale = 0.05000000074505806\n",
      "    └── Units = eV\n",
      "\n",
      "beam_energy : 80.0\n",
      "convergence_angle : 20.0\n",
      "EELS_collection_angle : 50.0\n",
      "num_rows : 18\n",
      "num_cols : 19\n",
      "spec_length : 1340\n",
      "spec_axis : [ -5.00000007  -4.95000007  -4.90000007 ...,  61.85000092  61.90000092\n",
      "  61.95000092]\n",
      "spec_pixel_size : 0.05000000074505806\n",
      "spec_unit : eV\n",
      "spatial_pixel_size : 0.002550000324845314\n",
      "spatial_unit : µm\n"
     ]
    }
   ],
   "source": [
    "h5_path = os.path.join(folder_path, file_name[:-4] + '.h5')\n",
    "print('The h5 file will be saved in the following path:')\n",
    "print(h5_path)\n",
    "\n",
    "image_cal = raw.original_metadata.ImageList.TagGroup0.ImageData.Calibrations.Dimension\n",
    "pos_cal = image_cal.TagGroup0\n",
    "spec_cal = image_cal.TagGroup2\n",
    "print(image_cal)\n",
    "\n",
    "# Generate the x / spectroscopic axis:\n",
    "pos_pixel_size = pos_cal.Scale\n",
    "pos_units = pos_cal.Units\n",
    "\n",
    "\n",
    "spec_pixel_size = spec_cal.Scale\n",
    "spec_min = (0-spec_cal.Origin)*spec_cal.Scale\n",
    "spec_units = spec_cal.Units\n",
    "spec_vec = np.linspace(spec_min, spec_min+spec_pixel_size*(data_shape[2]-1), data_shape[2])\n",
    "\n",
    "\n",
    "# Generate attributes dictionary:\n",
    "parm_dict=dict()\n",
    "tem = raw.metadata.Acquisition_instrument.TEM\n",
    "parm_dict['beam_energy'] = tem.beam_energy\n",
    "parm_dict['convergence_angle'] = tem.convergence_angle\n",
    "parm_dict['EELS_collection_angle']=tem.Detector.EELS.collection_angle\n",
    "parm_dict['num_rows']=data_shape[0]\n",
    "parm_dict['num_cols']=data_shape[1]\n",
    "parm_dict['spec_length']=data_shape[2]\n",
    "parm_dict['spec_axis']=spec_vec\n",
    "parm_dict['spec_pixel_size']=spec_pixel_size\n",
    "parm_dict['spec_unit']=spec_units\n",
    "parm_dict['spatial_pixel_size']=pos_pixel_size\n",
    "parm_dict['spatial_unit']=pos_units\n",
    "for key in parm_dict:\n",
    "    print(key, ':', parm_dict[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tran = px.io.NumpyTranslator()\n",
    "h5_path = tran.translate(h5_path, h5_data, data_shape[0], data_shape[1], \n",
    "                         qty_name='Count', data_unit='', spec_name='Energy Loss',\n",
    "                         spec_unit='eV', spec_val=spec_vec, spatial_unit='um', \n",
    "                         data_type='STS',translator_name='ASC', parms_dict=parm_dict)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}