11th-practice.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pathlib\nimport imageio\nimport numpy as np","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-08T06:35:01.782134Z","iopub.execute_input":"2022-01-08T06:35:01.782492Z","iopub.status.idle":"2022-01-08T06:35:01.787997Z","shell.execute_reply.started":"2022-01-08T06:35:01.782451Z","shell.execute_reply":"2022-01-08T06:35:01.786795Z"},"trusted":true},"execution_count":2,"outputs":[]},{"cell_type":"code","source":"import zipfile\npath_to_zip_file = '../input/data-science-bowl-2018/stage1_train.zip'\ndirectory_to_extract_to = '../working/input/data-science-bowl-2018/'\nwith zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref:\n    zip_ref.extractall(directory_to_extract_to)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:35:01.789684Z","iopub.execute_input":"2022-01-08T06:35:01.789999Z","iopub.status.idle":"2022-01-08T06:35:08.512428Z","shell.execute_reply.started":"2022-01-08T06:35:01.789963Z","shell.execute_reply":"2022-01-08T06:35:08.511161Z"},"trusted":true},"execution_count":3,"outputs":[]},{"cell_type":"code","source":"# Glob the training data and load a single image path\ntraining_paths = pathlib.Path('../working/input/data-science-bowl-2018/').glob('*/images/*.png')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-08T06:35:08.513975Z","iopub.execute_input":"2022-01-08T06:35:08.514297Z","iopub.status.idle":"2022-01-08T06:35:08.518194Z","shell.execute_reply.started":"2022-01-08T06:35:08.514257Z","shell.execute_reply":"2022-01-08T06:35:08.517309Z"},"trusted":true},"execution_count":4,"outputs":[]},{"cell_type":"code","source":"training_sorted = sorted([x for x in training_paths])\ntraining_sorted","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-08T06:35:08.519388Z","iopub.execute_input":"2022-01-08T06:35:08.520126Z","iopub.status.idle":"2022-01-08T06:35:08.605397Z","shell.execute_reply.started":"2022-01-08T06:35:08.520087Z","shell.execute_reply":"2022-01-08T06:35:08.593082Z"},"trusted":true},"execution_count":5,"outputs":[]},{"cell_type":"code","source":"im_path = training_sorted[45]","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-08T06:35:08.608177Z","iopub.execute_input":"2022-01-08T06:35:08.608734Z","iopub.status.idle":"2022-01-08T06:35:08.613296Z","shell.execute_reply.started":"2022-01-08T06:35:08.608689Z","shell.execute_reply":"2022-01-08T06:35:08.612554Z"},"trusted":true},"execution_count":6,"outputs":[]},{"cell_type":"code","source":"im = imageio.imread(str(im_path))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-08T06:35:08.615067Z","iopub.execute_input":"2022-01-08T06:35:08.615724Z","iopub.status.idle":"2022-01-08T06:35:08.670215Z","shell.execute_reply.started":"2022-01-08T06:35:08.615683Z","shell.execute_reply":"2022-01-08T06:35:08.669496Z"},"trusted":true},"execution_count":7,"outputs":[]},{"cell_type":"code","source":"# Print the image dimensions\nprint('Original image shape: {}'.format(im.shape))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:35:08.671511Z","iopub.execute_input":"2022-01-08T06:35:08.672271Z","iopub.status.idle":"2022-01-08T06:35:08.677391Z","shell.execute_reply.started":"2022-01-08T06:35:08.672216Z","shell.execute_reply":"2022-01-08T06:35:08.676767Z"},"trusted":true},"execution_count":8,"outputs":[]},{"cell_type":"code","source":"# Coerce the image into grayscale format (if not already)\nfrom skimage.color import rgb2gray\nim_gray = rgb2gray(im)\nprint('New image shape : {}'.format(im_gray.shape))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:35:08.678605Z","iopub.execute_input":"2022-01-08T06:35:08.678948Z","iopub.status.idle":"2022-01-08T06:35:09.122732Z","shell.execute_reply.started":"2022-01-08T06:35:08.678920Z","shell.execute_reply":"2022-01-08T06:35:09.121410Z"},"trusted":true},"execution_count":9,"outputs":[]},{"cell_type":"code","source":"# Now, let's plot the data\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(10,4))\n\nplt.subplot(1,2,1)\nplt.imshow(im)\nplt.axis('off')\nplt.title('Original Image')\n\nplt.subplot(1,2,2)\nplt.imshow(im_gray, cmap='gray')\nplt.axis('off')\nplt.title('Grayscale Image')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:35:09.124251Z","iopub.execute_input":"2022-01-08T06:35:09.124516Z","iopub.status.idle":"2022-01-08T06:35:09.495336Z","shell.execute_reply.started":"2022-01-08T06:35:09.124487Z","shell.execute_reply":"2022-01-08T06:35:09.494105Z"},"trusted":true},"execution_count":10,"outputs":[]},{"cell_type":"code","source":"from skimage.filters import threshold_otsu\nthresh_val = threshold_otsu(im_gray)\nmask = np.where(im_gray > thresh_val, 1, 0)\n\n# Make sure the larger portion of the mask is considered background\nif np.sum(mask == 0) < np.sum(mask == 1):\n    mask = np.where(mask, 0, 1)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:35:09.496889Z","iopub.execute_input":"2022-01-08T06:35:09.497135Z","iopub.status.idle":"2022-01-08T06:35:11.384234Z","shell.execute_reply.started":"2022-01-08T06:35:09.497106Z","shell.execute_reply":"2022-01-08T06:35:11.383459Z"},"trusted":true},"execution_count":11,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10,4))\n\nplt.subplot(1,2,1)\nim_pixels = im_gray.flatten()\nplt.hist(im_pixels, bins = 50)\nplt.vlines(thresh_val, 0, 100000, linestyle='--')\nplt.ylim([0, 50000])\nplt.title('Grayscale Histogram')\n\nplt.subplot(1,2,2)\nmask_for_display = np.where(mask, mask, np.nan)\nplt.imshow(im_gray, cmap = 'gray')\nplt.imshow(mask_for_display, cmap = 'rainbow', alpha = 0.5)\nplt.axis('off')\nplt.title('Image w/ Mask')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:35:11.385501Z","iopub.execute_input":"2022-01-08T06:35:11.386243Z","iopub.status.idle":"2022-01-08T06:35:11.858335Z","shell.execute_reply.started":"2022-01-08T06:35:11.386202Z","shell.execute_reply":"2022-01-08T06:35:11.857170Z"},"trusted":true},"execution_count":12,"outputs":[]},{"cell_type":"code","source":"from scipy import ndimage\nlabels, nlabels = ndimage.label(mask)\nlabel_arrays = []\nfor label_num in range(1, nlabels+1):\n    label_mask = np.where(labels == label_num, 1, 0)\n    label_arrays.append(label_mask)\n    \nprint('There are {} seperate components / objects detected'.format(nlabels))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:35:11.859998Z","iopub.execute_input":"2022-01-08T06:35:11.860634Z","iopub.status.idle":"2022-01-08T06:35:12.148492Z","shell.execute_reply.started":"2022-01-08T06:35:11.860588Z","shell.execute_reply":"2022-01-08T06:35:12.146726Z"},"trusted":true},"execution_count":13,"outputs":[]},{"cell_type":"code","source":"# Create a random colormat\nfrom matplotlib.colors import ListedColormap\nrand_cmap = ListedColormap(np.random.rand(256,3))\n\nlabels_for_display = np.where(labels > 0, labels, np.nan)\nplt.imshow(im_gray, cmap = 'gray')\nplt.imshow(labels_for_display, cmap = rand_cmap)\nplt.axis('off')\nplt.title('Labeled Cells ({} Nuclei)'.format(nlabels))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:36:46.385582Z","iopub.execute_input":"2022-01-08T06:36:46.385932Z","iopub.status.idle":"2022-01-08T06:36:46.621742Z","shell.execute_reply.started":"2022-01-08T06:36:46.385890Z","shell.execute_reply":"2022-01-08T06:36:46.620894Z"},"trusted":true},"execution_count":15,"outputs":[]},{"cell_type":"code","source":"for label_ind, label_coords in enumerate(ndimage.find_objects(labels)):\n    cell = im_gray[label_coords]\n    \n    # Check if the label size is too small\n    if np.product(cell.shape) < 10:\n        print('Label {} is too small! Setting to 0.'.format(label_ind))\n        mask = np.where(labels==label_ind + 1, 0, mask)\n# Regenerate the labels\nlabels, nlabels = ndimage.label(mask)\nprint('There are now {} seperate components / objects detected'.format(nlabels))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:39:19.436695Z","iopub.execute_input":"2022-01-08T06:39:19.437036Z","iopub.status.idle":"2022-01-08T06:39:19.467686Z","shell.execute_reply.started":"2022-01-08T06:39:19.437002Z","shell.execute_reply":"2022-01-08T06:39:19.466652Z"},"trusted":true},"execution_count":16,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 6, figsize = (10,6))\n\nfor ii, obj_indices in enumerate(ndimage.find_objects(labels)[0:6]):\n    cell = im_gray[obj_indices]\n    axes[ii].imshow(cell, cmap = 'gray')\n    axes[ii].axis('off')\n    axes[ii].set_title('Label #{}\\nSize: {}'.format(ii+1, cell.shape))\n    \nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:42:01.846493Z","iopub.execute_input":"2022-01-08T06:42:01.846856Z","iopub.status.idle":"2022-01-08T06:42:02.222238Z","shell.execute_reply.started":"2022-01-08T06:42:01.846800Z","shell.execute_reply":"2022-01-08T06:42:02.221310Z"},"trusted":true},"execution_count":18,"outputs":[]},{"cell_type":"code","source":"# Get the object indices, and perform a binary opening procedure\ntwo_cell_indices = ndimage.find_objects(labels)[1]\ncell_mask = mask[two_cell_indices]\ncell_mask_opened = ndimage.binary_opening(cell_mask, iterations = 8)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:43:11.487288Z","iopub.execute_input":"2022-01-08T06:43:11.488333Z","iopub.status.idle":"2022-01-08T06:43:11.494721Z","shell.execute_reply.started":"2022-01-08T06:43:11.488293Z","shell.execute_reply":"2022-01-08T06:43:11.494117Z"},"trusted":true},"execution_count":19,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 4, figsize = (12,4))\n\naxes[0].imshow(im_gray[two_cell_indices], cmap = 'gray')\naxes[0].set_title('Original object')\naxes[1].imshow(mask[two_cell_indices], cmap = 'gray')\naxes[1].set_title('Original mask')\naxes[2].imshow(cell_mask_opened, cmap = 'gray')\naxes[2].set_title('Opened mask')\naxes[3].imshow(im_gray[two_cell_indices]*cell_mask_opened, cmap='gray')\naxes[3].set_title('Opened object')\n\nfor ax in axes:\n    ax.axis('off')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:46:07.749984Z","iopub.execute_input":"2022-01-08T06:46:07.750720Z","iopub.status.idle":"2022-01-08T06:46:08.032789Z","shell.execute_reply.started":"2022-01-08T06:46:07.750676Z","shell.execute_reply":"2022-01-08T06:46:08.032230Z"},"trusted":true},"execution_count":20,"outputs":[]},{"cell_type":"code","source":"def rle_encoding(x):\n    '''\n    x : numpy array of shape (height, width), 1 - mask, 0 - background\n    Returns run length as list\n    '''\n    dots = np.where(x.T.flatten()==1)[0]\n    # .T sets Fortran order down-then-right\n    run_lengths = []\n    prev = -2\n    for b in dots:\n        if (b > prev+1): run_lengths.extend((b+1, 0))\n        run_lengths[-1] += 1\n        prev = b\n    return \" \".join([str(i) for i in run_lengths])\n\nprint('RLE Encoding for the current mask is: {}'.format(rle_encoding(label_mask)))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T06:48:57.175510Z","iopub.execute_input":"2022-01-08T06:48:57.175963Z","iopub.status.idle":"2022-01-08T06:48:57.189724Z","shell.execute_reply.started":"2022-01-08T06:48:57.175916Z","shell.execute_reply":"2022-01-08T06:48:57.188682Z"},"trusted":true},"execution_count":21,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\ndef analyze_image(im_path):\n    '''\n    Take an image_path (pathlib.Path object), preprocess and label it, extract the RLE strings and dump it into a Pandas DataFrame\n    '''\n    # Read in data and convert to grayscale\n    im_id = im_path.parts[-3]\n    im = imageio.imread(str(im_path))\n    im_gray = rgb2gray(im)\n    \n    # Mask out background and extract connected objects\n    thresh_val = threshold_otsu(im_gray)\n    mask = np.where(im_gray > thresh_val, 1, 0)\n    if np.sum(mask == 0) < np.sum(mask == 1):\n        mask = np.where(mask, 0, 1)\n        labels, nlabels = ndimage.label(mask)\n    labels, nlabels = ndimage.label(mask)\n    \n    # Loop through labels and add each to a DataFrame\n    im_df = pd.DataFrame()\n    for label_num in range(1, nlabels+1):\n        label_mask = np.where(labels == label_num, 1, 0)\n        if label_mask.flatten().sum() > 10:\n            rle = rle_encoding(label_mask)\n            s = pd.Series({'ImageId' : im_id, 'EncodedPixels' : rle})\n            im_df = im_df.append(s, ignore_index = True)\n            \n    return im_df\n\ndef analyze_list_of_images(im_path_list):\n    '''\n    Takes a list of image paths (pathlib.Path objects), analyzes each, and returns a submission-ready DataFrame.\n    '''\n    all_df = pd.DataFrame()\n    for im_path in im_path_list:\n        im_df = analyze_image(im_path)\n        all_df = all_df.append(im_df, ignore_index = True)\n        \n    return all_df","metadata":{"execution":{"iopub.status.busy":"2022-01-08T07:00:42.966083Z","iopub.execute_input":"2022-01-08T07:00:42.968272Z","iopub.status.idle":"2022-01-08T07:00:42.986641Z","shell.execute_reply.started":"2022-01-08T07:00:42.968183Z","shell.execute_reply":"2022-01-08T07:00:42.985225Z"},"trusted":true},"execution_count":22,"outputs":[]},{"cell_type":"code","source":"path_to_zip_file = '../input/data-science-bowl-2018/stage1_test.zip'\ndirectory_to_extract_to = '../working/input/data-science-bowl-2018/stage1_test'\nwith zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref:\n    zip_ref.extractall(directory_to_extract_to)\n\ntesting = pathlib.Path('../working/input/data-science-bowl-2018/stage1_test').glob('*/images/*.png')\ndf = analyze_list_of_images(list(testing))\ndf.to_csv('submission.csv', index = None)\n\n# import zipfile\n# path_to_zip_file = '../input/data-science-bowl-2018/stage1_train.zip'\n# directory_to_extract_to = '../working/input/data-science-bowl-2018/'\n# with zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref:\n#     zip_ref.extractall(directory_to_extract_to)\n    \n# # Glob the training data and load a single image path\n# training_paths = pathlib.Path('../working/input/data-science-bowl-2018/').glob('*/images/*.png')","metadata":{"execution":{"iopub.status.busy":"2022-01-08T07:03:46.980637Z","iopub.execute_input":"2022-01-08T07:03:46.981419Z","iopub.status.idle":"2022-01-08T07:04:00.447418Z","shell.execute_reply.started":"2022-01-08T07:03:46.981362Z","shell.execute_reply":"2022-01-08T07:04:00.446453Z"},"trusted":true},"execution_count":23,"outputs":[]}]}