TabularDataPreparationUsingPycaret.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TabularDataPreparationUsingPycaret.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "private_outputs": true,
      "authorship_tag": "ABX9TyMia9y9S5qbTn5o+WMAYutt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ShaswataJash/kfpcomponent/blob/main/TabularDataPreparationUsingPycaret.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This notebook is the development workflow for kubeflow pipeline component of the same name as this notebook. Refer https://github.com/ShaswataJash/kfpcomponent"
      ],
      "metadata": {
        "id": "_fMb4fVRFsPj"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Install required softwares"
      ],
      "metadata": {
        "id": "HYI5LKdFLCTN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!uname -a"
      ],
      "metadata": {
        "id": "JoSae8JMvVgq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!lsb_release -a"
      ],
      "metadata": {
        "id": "ZZTBdke9vme4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python --version"
      ],
      "metadata": {
        "id": "P4orJiv6orBy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lus4TEd-8DbB"
      },
      "outputs": [],
      "source": [
        "!pip install pycaret==2.3.10"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Ref: https://github.com/pycaret/pycaret/issues/2490\n",
        "!pip install Jinja2==3.1.2"
      ],
      "metadata": {
        "id": "yKAa1jnEnDL3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!apt-get install ca-certificates fuse tzdata curl unzip && \\\n",
        "  echo \"user_allow_other\" >> /etc/fuse.conf"
      ],
      "metadata": {
        "id": "3p89zTHfYqwc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!curl https://rclone.org/install.sh | bash"
      ],
      "metadata": {
        "id": "YGe2hZo0U7DL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!rclone --version"
      ],
      "metadata": {
        "id": "Byk9g7UWUtJO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Develop source code files"
      ],
      "metadata": {
        "id": "EMVyzkX_LIoG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile data_preparation.py\n",
        "#!/usr/bin/env python3\n",
        "import os\n",
        "import sys\n",
        "for arg in sys.argv:\n",
        "    print(arg)\n",
        "sys.stdout.flush()\n",
        "\n",
        "import argparse\n",
        "import logging\n",
        "parser = argparse.ArgumentParser(description='kubeflow pipeline component to read csv file and prepare the data')\n",
        "parser.add_argument('--log-level', default='INFO', choices=['ERROR', 'INFO', 'DEBUG'])\n",
        "parser.add_argument('--bypass-rclone-for-input-data', default=False, action=\"store_true\", help='whether input csv file should be read like local file - rclone is completely bypassed')\n",
        "parser.add_argument('--bypass-rclone-for-output-data', default=False, action=\"store_true\", help='whether output csv file should be written like local file - rclone is completely bypassed')\n",
        "parser.add_argument('--rclone-environment-var', type=str, default= '{}', help='json formatted key-value pairs of strings which will be set as environment variables before executing rclone commands')\n",
        "parser.add_argument('--input-datasource-directory-mountable', default=False, action=\"store_true\", help='whether input csv file is present in mountable remote location when rclone is used')\n",
        "parser.add_argument('--input-datasource-file-name', type=str, default='', help='name of the csv file including file extension and the directory/bucket path holding the specific file(if any) when rclone is used')\n",
        "parser.add_argument('--additional-options-csv-parsing', type=str, default= '{}', help='json formatted key-value pairs of strings which will be passed to pandas.read_csv()')\n",
        "parser.add_argument('--type-of-data-analysis-task', choices=['classification', 'regression', 'clustering', 'anomaly_detection'])\n",
        "parser.add_argument('--target-variable-name', type=str, help='for classification and regression, specify the column name holding target variable')\n",
        "parser.add_argument('--target-emptyindicator', type=str, default='', help='if target variable column holds null or na, those rows will be dropped. Sometime empty can be indicated by other representative string like - or *** etc')\n",
        "parser.add_argument('--data-preparations-options', type=str, default= '{}', help='json formatted key-value pairs of strings which will be passed to pycaret setup() function')\n",
        "parser.add_argument('--additional-options-csv-writing', type=str, default= '{}', help='json formatted key-value pairs of strings which will be passed to pandas.to_csv()')\n",
        "parser.add_argument('--output-datasource-directory-mountable', default=False, action=\"store_true\", help='whether output csv file will be written in mountable remote location when rclone is used')\n",
        "parser.add_argument('--output-datasource-file-name', type=str, default='', help='filename of the prepared data including the directory/bucket path holding the specific file(if any) when rclone is used')\n",
        "parser.add_argument('--input-datasource-local-file-path-when-rclone-bypassed', type=str, default='', help='absolute local path of the input csv file when rclone is NOT used i.e. when bypass-rclone-for-input-data is enabled')\n",
        "parser.add_argument('--output-datasource-local-file-path-when-rclone-bypassed', type=str, default= '', help='absolute local path of the output csv file when rclone is NOT used i.e. when bypass-rclone-for-output-data is enabled')\n",
        "args = parser.parse_args()\n",
        "\n",
        "#keeping the log format same as used in pycaret for consistency (refer: https://github.com/pycaret/pycaret/blob/master/pycaret/internal/logging.py)\n",
        "logging.basicConfig(level=args.log_level, format='%(asctime)s:%(levelname)s:%(message)s')\n",
        "os.environ[\"PYCARET_CUSTOM_LOGGING_LEVEL\"] = args.log_level\n",
        "\n",
        "#sanity check of arguments\n",
        "if args.bypass_rclone_for_input_data:\n",
        "    args.input_datasource_directory_mountable = False\n",
        "    args.input_datasource_file_name = None\n",
        "else:\n",
        "    args.input_datasource_local_file_path_when_rclone_bypassed = None\n",
        "\n",
        "if args.bypass_rclone_for_output_data:\n",
        "    args.output_datasource_directory_mountable = False\n",
        "    args.output_datasource_file_name = None\n",
        "else:\n",
        "    args.output_datasource_local_file_path_when_rclone_bypassed = None\n",
        "\n",
        "if args.bypass_rclone_for_input_data and args.bypass_rclone_for_output_data:\n",
        "    args.rclone_environment_var = '{}'\n",
        "\n",
        "#setting rclone related env\n",
        "import json\n",
        "try:\n",
        "    rclone_config = json.loads(args.rclone_environment_var)\n",
        "    logging.info(\"rclone_config: type=%s content=%s\", type(rclone_config), rclone_config)\n",
        "    for item in rclone_config.items():\n",
        "        #converting explicitely item[1] to str because rclone config can have nested json. In that case, item[1] will be of dictonary type\n",
        "        #replacing quote with double quote to make the values json compatible (note for string without ', below replacement has no effect)\n",
        "        os.environ[item[0]] = str(item[1]).replace('\\'', '\"')\n",
        "        logging.debug('%s => %s', item[0], os.getenv(item[0]))\n",
        "except BaseException as err:\n",
        "    logging.error(\"rclone configuration loading related error\", exc_info=True)\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as exception encountered while loading rclone_config\")    \n",
        "\n",
        "#temporary directory creation\n",
        "import tempfile\n",
        "try:\n",
        "    if not args.bypass_rclone_for_input_data:\n",
        "        local_datastore_read_dir = tempfile.mkdtemp(prefix=\"my_local_read-\")\n",
        "        logging.debug('local_datastore_read_dir:%s',local_datastore_read_dir)\n",
        "\n",
        "    if not args.bypass_rclone_for_output_data:\n",
        "        local_datastore_write_dir = tempfile.mkdtemp(prefix=\"my_local_write-\")\n",
        "        logging.debug('local_datastore_write_dir:%s',local_datastore_write_dir)\n",
        "except BaseException as err:\n",
        "    logging.error(\"temporary directory creation related error\", exc_info=True)\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as exception encountered while creating temporary directories\")\n",
        "\n",
        "#input file handling\n",
        "import subprocess\n",
        "import ntpath\n",
        "if args.input_datasource_directory_mountable:\n",
        "    input_data_read_cmd = \"rclone -v mount remoteread:\" + ntpath.dirname(args.input_datasource_file_name) + ' ' + local_datastore_read_dir + ' --daemon'\n",
        "else:\n",
        "    input_data_read_cmd = \"rclone -v copy remoteread:\" + args.input_datasource_file_name + ' ' + local_datastore_read_dir\n",
        "logging.info(input_data_read_cmd)\n",
        "input_data_read_call = subprocess.run(input_data_read_cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n",
        "logging.info(input_data_read_call.stdout)\n",
        "if input_data_read_call.returncode != 0:\n",
        "    logging.error(\"Error in rclone, errorcode= %s\", input_data_read_call.returncode)\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as rclone returned error in context of reading\")\n",
        "\n",
        "#output file handling\n",
        "if args.output_datasource_directory_mountable:\n",
        "    output_data_write_cmd = \"rclone -v mount remotewrite:\" + ntpath.dirname(args.output_datasource_file_name) + ' ' + local_datastore_write_dir + ' --daemon'\n",
        "    logging.info(output_data_write_cmd)\n",
        "    output_data_write_call = subprocess.run(output_data_write_cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n",
        "    logging.info(output_data_write_call.stdout)\n",
        "    if output_data_write_call.returncode != 0:\n",
        "        logging.error(\"Error in rclone, errorcode=%s\", output_data_write_call.returncode)\n",
        "        sys.stdout.flush()\n",
        "        sys.exit(\"Forceful exit as rclone returned error in context of mounted writing\")\n",
        "\n",
        "#handling input csv file reading\n",
        "import pandas\n",
        "try:\n",
        "    parse_config = json.loads(args.additional_options_csv_parsing)\n",
        "    parse_config['filepath_or_buffer'] =  args.input_datasource_local_file_path_when_rclone_bypassed \\\n",
        "        if args.bypass_rclone_for_input_data else os.path.join(local_datastore_read_dir,ntpath.basename(args.input_datasource_file_name))\n",
        "    logging.info(\"parse_config: type=%s content=%s\", type(parse_config), parse_config)\n",
        "    my_data = pandas.read_csv(**parse_config)\n",
        "    logging.debug('%s', my_data)\n",
        "    \n",
        "except BaseException as err:\n",
        "    logging.error(\"csv file reading related error\", exc_info=True)\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as exception encountered while parsing input csv file\")\n",
        "\n",
        "#handling data preprocessing\n",
        "import pycaret\n",
        "try:\n",
        "    if os.path.exists(\"logs.log\"):\n",
        "        os.remove(\"logs.log\") #removing any content from log which pycaret will internally use for its own logging\n",
        "    logging.info('pycaret version = %s ', pycaret.utils.version())\n",
        "    setup_config = json.loads(args.data_preparations_options)\n",
        "    if args.type_of_data_analysis_task == 'classification':\n",
        "        import pycaret.classification\n",
        "        setup_fn = pycaret.classification.setup\n",
        "        get_config_fn = pycaret.classification.get_config\n",
        "        setup_config['target'] = args.target_variable_name\n",
        "        \n",
        "    elif args.type_of_data_analysis_task == 'regression':\n",
        "        import pycaret.regression\n",
        "        setup_fn = pycaret.regression.setup\n",
        "        get_config_fn = pycaret.regression.get_config\n",
        "        setup_config['target'] = args.target_variable_name\n",
        "\n",
        "    elif args.type_of_data_analysis_task == 'clustering':\n",
        "        import pycaret.clustering\n",
        "        setup_fn = pycaret.clustering.setup\n",
        "        get_config_fn = pycaret.clustering.get_config\n",
        "\n",
        "    elif args.type_of_data_analysis_task == 'anomaly':\n",
        "        import pycaret.anomaly\n",
        "        setup_fn = pycaret.anomaly.setup\n",
        "        get_config_fn = pycaret.anomaly.get_config\n",
        "        \n",
        "    #as part of pycaret's data cleaning the rows with target column = nan are not being cleaned up. Thus, cleaning those rows explicitely\n",
        "    if len(args.target_emptyindicator) > 0:\n",
        "        #ref: https://stackoverflow.com/questions/49291740/delete-rows-if-there-are-null-values-in-a-specific-column-in-pandas-dataframe\n",
        "        import numpy as np\n",
        "        my_data[args.target_variable_name] = my_data[args.target_variable_name].replace(args.target_emptyindicator, np.nan)\n",
        "        my_data = my_data.dropna(axis=0, subset=[args.target_variable_name])\n",
        "\n",
        "    setup_config['log_experiment'] = False\n",
        "    setup_config['data_split_shuffle'] = False\n",
        "    setup_config['html'] = False\n",
        "    setup_config['silent'] = True\n",
        "    logging.info(\"setup_config: type=%s content=%s\", type(setup_config), setup_config)\n",
        "    setup_config['data'] = my_data #adding dataframe after logging, or else a big dataframe print happens as part of logging\n",
        "    setup_fn(**setup_config)\n",
        "    #ref: https://www.kdnuggets.com/2020/11/5-things-doing-wrong-pycaret.html\n",
        "    X_transformed = get_config_fn('X')\n",
        "    my_transformed_data = X_transformed\n",
        "    if args.type_of_data_analysis_task == 'classification' or args.type_of_data_analysis_task == 'regression':\n",
        "        y_transformed = get_config_fn('y')\n",
        "        my_transformed_data = X_transformed.merge(y_transformed,left_index=True, right_index=True)\n",
        "    \n",
        "    logging.debug(\"====== PREPARED DATA ====\")\n",
        "    logging.debug('%s', my_transformed_data)\n",
        "    logging.debug(\"=========================\")\n",
        "\n",
        "    #pycaret.utils.get_system_logs() #this will print the pycaret's own log into console\n",
        "    \n",
        "except BaseException as err:\n",
        "    #pycaret.utils.get_system_logs()\n",
        "    logging.error(\"exception encountered while transforming input dataframe\", exc_info=True)\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as exception encountered while transforming input dataframe\")\n",
        "\n",
        "#handling output csv file writing\n",
        "try:\n",
        "    to_csv_config = json.loads(args.additional_options_csv_writing)\n",
        "    to_csv_config['path_or_buf'] = args.output_datasource_local_file_path_when_rclone_bypassed \\\n",
        "        if args.bypass_rclone_for_output_data else os.path.join(local_datastore_write_dir,ntpath.basename(args.output_datasource_file_name))\n",
        "    logging.info(\"to_csv_config: type=%s content=%s\", type(to_csv_config), to_csv_config)\n",
        "    my_transformed_data.to_csv(**to_csv_config)\n",
        "except BaseException as err:\n",
        "    logging.error(\"exception encountered while trying to write prepared data\", exc_info=True)\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(\"Forceful exit as exception encountered while trying to write prepared data\")\n",
        "\n",
        "if args.bypass_rclone_for_output_data:\n",
        "    sys.stdout.flush()\n",
        "    sys.exit(0)\n",
        "\n",
        "if not args.output_datasource_directory_mountable:\n",
        "    output_data_write_cmd = \"rclone -v copy \" + os.path.join(local_datastore_write_dir,ntpath.basename(args.output_datasource_file_name)) \\\n",
        "        + \" remotewrite:\" + ntpath.dirname(args.output_datasource_file_name)\n",
        "    logging.info(output_data_write_cmd)\n",
        "    output_data_write_call = subprocess.run(output_data_write_cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)\n",
        "    logging.info(output_data_write_call.stdout)\n",
        "    if output_data_write_call.returncode != 0:\n",
        "        logging.error(\"Error in rclone, errorcode=%s\", output_data_write_call.returncode)\n",
        "        sys.stdout.flush()\n",
        "        sys.exit(\"Forceful exit as rclone returned error in context of writing final csv file (copy mode)\")"
      ],
      "metadata": {
        "id": "WU9Vh7P4P4_h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Docker size reduction tips:\n",
        "\n",
        "\n",
        "*   https://devopscube.com/reduce-docker-image-size/\n",
        "*   https://www.ecloudcontrol.com/best-practices-to-reduce-docker-images-size/\n",
        "\n"
      ],
      "metadata": {
        "id": "eQXf07hoPzxM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile Dockerfile\n",
        "FROM python:3.7.13-slim\n",
        "\n",
        "RUN python3 -m pip install pycaret==2.3.10\n",
        "#installing jinja2 additionally due to Ref: https://github.com/pycaret/pycaret/issues/2490\n",
        "RUN python3 -m pip install Jinja2==3.1.2\n",
        "\n",
        "#install fuse as dependency for rclone. Additionally, install curl, unzip for rclone installer to work\n",
        "#libgomp1 installation for pycaret in python-slim\n",
        "RUN apt-get update \\\n",
        "    && apt-get install --no-install-recommends -y curl fuse libgomp1 unzip \\\n",
        "    && echo \"user_allow_other\" >> /etc/fuse.conf \\\n",
        "    && curl https://rclone.org/install.sh | bash \\\n",
        "    && apt-get -y remove --purge curl unzip \\\n",
        "    && apt-get -y autoremove \\\n",
        "    && rm -rf /var/lib/apt/lists/* \\\n",
        "    && rclone --version\n",
        "\n",
        "COPY src/data_preparation.py /tmp\n",
        "COPY tests/test_validation.py /tmp\n",
        "COPY run_tests.sh /tmp\n",
        "RUN chmod 544 /tmp/run_tests.sh"
      ],
      "metadata": {
        "id": "hk00hb780Qo_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile run_tests.sh\n",
        "#!/bin/bash\n",
        "\n",
        "mkdir /tmp/my_local_dir_for_test\n",
        "\n",
        "#Test: csv reading source from http, rclone read in copy\n",
        "python /tmp/data_preparation.py --rclone-environment-var '{\"RCLONE_CONFIG_REMOTEREAD_TYPE\":\"http\", \"RCLONE_CONFIG_REMOTEREAD_URL\":\"https://raw.githubusercontent.com/pycaret/datasets/main/data/common/\"}' \\\n",
        "    --input-datasource-file-name 'CTG.csv' --additional-options-csv-parsing '{\"sep\":\",\" , \"header\":0}' \\\n",
        "    --type-of-data-analysis-task 'classification' --target-variable-name 'NSP' \\\n",
        "    --data-preparations-options '{\"ignore_low_variance\":true, \"remove_outliers\":true, \"remove_multicollinearity\":true, \"multicollinearity_threshold\":0.7}' \\\n",
        "    --bypass-rclone-for-output-data --output-datasource-local-file-path-when-rclone-bypassed '/tmp/my_local_dir_for_test/CTG_data-prep.csv' \\\n",
        "    --additional-options-csv-writing '{\"index\":false}' --log-level 'DEBUG'\n",
        "\n",
        "#https://registry.opendata.aws/humor-detection/\n",
        "#Test: csv reading source from s3(AWS provider), rclone read in mount\n",
        "python /tmp/data_preparation.py --rclone-environment-var '{\"RCLONE_CONFIG_REMOTEREAD_TYPE\":\"s3\", \"RCLONE_CONFIG_REMOTEREAD_PROVIDER\":\"AWS\", \"RCLONE_CONFIG_REMOTEREAD_REGION\":\"us-west-2\"}' \\\n",
        "    --input-datasource-directory-mountable --input-datasource-file-name 'humor-detection-pds/Non-humours-biased.csv' \\\n",
        "    --type-of-data-analysis-task 'classification' --target-variable-name 'label' \\\n",
        "    --data-preparations-options '{\"preprocess\":false, \"ignore_features\":[\"image_url\"]}' \\\n",
        "    --bypass-rclone-for-output-data --output-datasource-local-file-path-when-rclone-bypassed '/tmp/my_local_dir_for_test/Non-humours-biased_data-prep.csv' \\\n",
        "    --additional-options-csv-writing '{\"index\":false}' --log-level 'DEBUG'\n",
        "\n",
        "python /tmp/test_validation.py"
      ],
      "metadata": {
        "id": "jzJg29vTVGcf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile test_validation.py\n",
        "#!/usr/bin/env python3\n",
        "import pandas\n",
        "df = pandas.read_csv(filepath_or_buffer = '/tmp/my_local_dir_for_test/CTG_data-prep.csv')\n",
        "assert len(df.index) == 2126 #original data had 2129 rows, amongst that 3 rows have no target\n",
        "assert df.isnull().sum().sum() == 0 #pycaret will remove all missing values\n",
        "\n",
        "df = pandas.read_csv(filepath_or_buffer = '/tmp/my_local_dir_for_test/Non-humours-biased_data-prep.csv')\n",
        "assert len(df.columns) == 3 #original data had 4 columns, amongst that one will be dropped\n",
        "print ('test-validation done successfully')"
      ],
      "metadata": {
        "id": "7maMc-RVqDV5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Refer: https://github.com/RealOrangeOne/docker-rclone-mount/blob/master/docker-compose.yml\n",
        "\n",
        "If we have to use mount feature of rclone, it needs to have fuse support in underneath linux kernel. For that we are adding SYS_ADMIN in capability. But note without using mount feature also, we can do testing. in that case, rclone will use only copy feature."
      ],
      "metadata": {
        "id": "YRSP8Vk8vyoM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile docker-compose.test.yml\n",
        "services:\n",
        "  sut:\n",
        "    build: .\n",
        "    command: /tmp/run_tests.sh\n",
        "    cap_add:\n",
        "      - SYS_ADMIN\n",
        "    security_opt:\n",
        "      - apparmor:unconfined\n",
        "    devices:\n",
        "      - \"/dev/fuse:/dev/fuse\""
      ],
      "metadata": {
        "id": "cVrXyQZhSIm_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "*   https://www.kubeflow.org/docs/components/pipelines/sdk/component-development/#designing-a-pipeline-component\n",
        "*   https://github.com/kubeflow/pipelines/blob/sdk/release-1.8/sdk/python/kfp/dsl/types.py\n",
        "*   https://kubeflow-pipelines.readthedocs.io/en/stable/_modules/kfp/components/_structures.html\n",
        "\n"
      ],
      "metadata": {
        "id": "W7uyWGT6Sccr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile component_both_input_output_as_artifact.yaml\n",
        "name: TabularDataPreparationUsingPycaretWhereBothInputOutputAsArtifact\n",
        "description: |\n",
        "    Prepare tabular data (csv file) using pycaret library. (For pycaret's data pre-processing capabilities, refer https://pycaret.gitbook.io/docs/get-started/preprocessing)\n",
        "    Refer data-preparations-options in command line arguments. \n",
        "    pycaret internally uses pandas dataframe to read and write csv file. You can utilize options exposed by panda's read_csv() and to_csv(). \n",
        "    Refer additional-options-csv-parsing and additional-options-csv-writing in command line arguments\n",
        "    Input and output csv files are stored in input and output artifacts. Thus the csv files are read or written like locally mounted POSIX files.\n",
        "metadata:\n",
        "  annotations:\n",
        "    author: Shaswata Jash <29448766+ShaswataJash@users.noreply.github.com>\n",
        "    canonical_location: https://raw.githubusercontent.com/ShaswataJash/kfpcomponent/main/TabularDataPreparationUsingPycaret/component_both_input_output_as_artifact.yaml\n",
        "inputs:\n",
        "- name: log_level\n",
        "  type: String\n",
        "  description: 'choice amongst ERROR, INFO, DEBUG'\n",
        "  optional: true\n",
        "- name: additional_options_csv_parsing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.read_csv()'\n",
        "  optional: true\n",
        "- name: type_of_data_analysis_task\n",
        "  type: String\n",
        "  description: 'choice amongst classification, regression, clustering, anomaly_detection'\n",
        "- name: target_variable_name \n",
        "  type: String\n",
        "  description: 'for classification and regression, specify the column name holding target variable'\n",
        "  optional: true\n",
        "- name: target_emptyindicator\n",
        "  type: String\n",
        "  description: 'if target variable column holds null or na, those rows will be dropped. Sometime empty can be indicated by other representative string like - or *** etc'\n",
        "  optional: true\n",
        "- name: data_preparations_options\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pycaret setup() function'\n",
        "  optional: true\n",
        "- name: additional_options_csv_writing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.to_csv()'\n",
        "  optional: true\n",
        "- name: input_datasource_local_file_path_when_rclone_bypassed\n",
        "  description: 'absolute local path of the input csv file when rclone is NOT used i.e. when input csv file is stored in input artifact of pipeline engine (e.g. argo)'\n",
        "\n",
        "outputs:\n",
        "- name: output_datasource_local_file_path_when_rclone_bypassed\n",
        "  description: 'absolute local path of the output csv file when rclone is NOT used i.e. when output csv file is stored in output artifact of pipeline engine (e.g. argo)'\n",
        "\n",
        "implementation:\n",
        "  container:\n",
        "    image: shasjash/kfpcomponents:TabularDataPreparationUsingPycaret_devlatest\n",
        "    command:\n",
        "    - python3 \n",
        "    - /tmp/data_preparation.py\n",
        "    args:\n",
        "    - --bypass-rclone-for-input-data\n",
        "    - --bypass-rclone-for-output-data\n",
        "    - if:\n",
        "        cond: {isPresent: log_level}\n",
        "        then:\n",
        "        - --log-level\n",
        "        - {inputValue: log_level}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_parsing}\n",
        "        then:\n",
        "        - --additional-options-csv-parsing\n",
        "        - {inputValue: additional_options_csv_parsing}\n",
        "    - --type-of-data-analysis-task \n",
        "    - {inputValue: type_of_data_analysis_task}\n",
        "    - if:\n",
        "        cond: {isPresent: target_variable_name}\n",
        "        then:\n",
        "        - --target-variable-name\n",
        "        - {inputValue: target_variable_name}\n",
        "    - if:\n",
        "        cond: {isPresent: target_emptyindicator}\n",
        "        then:\n",
        "        - --target-emptyindicator\n",
        "        - {inputValue: target_emptyindicator}\n",
        "    - if:\n",
        "        cond: {isPresent: data_preparations_options}\n",
        "        then:\n",
        "        - --data-preparations-options\n",
        "        - {inputValue: data_preparations_options}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_writing}\n",
        "        then:\n",
        "        - --additional-options-csv-writing\n",
        "        - {inputValue: additional_options_csv_writing}\n",
        "    - --input-datasource-local-file-path-when-rclone-bypassed\n",
        "    - {inputPath: input_datasource_local_file_path_when_rclone_bypassed}\n",
        "    - --output-datasource-local-file-path-when-rclone-bypassed\n",
        "    - {outputPath: output_datasource_local_file_path_when_rclone_bypassed}"
      ],
      "metadata": {
        "id": "dhXccnmvIo7u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile component_input_using_rclone_output_as_artifact.yaml\n",
        "name: TabularDataPreparationUsingPycaretWhereInputUsingRcloneOutputAsArtifact\n",
        "description: |\n",
        "    Prepare tabular data (csv file) using pycaret library. (For pycaret's data pre-processing capabilities, refer https://pycaret.gitbook.io/docs/get-started/preprocessing)\n",
        "    Refer data-preparations-options in command line arguments. \n",
        "    pycaret internally uses pandas dataframe to read and write csv file. You can utilize options exposed by panda's read_csv() and to_csv(). \n",
        "    Refer additional-options-csv-parsing and additional-options-csv-writing in command line arguments\n",
        "    Input csv files can be stored in rclone compatible storage. Both mount and copy mode are supported. (refer: https://rclone.org/)\n",
        "    rclone configurations have to be shared through environment variables (refer: https://rclone.org/docs/#environment-variables). \n",
        "    Create rclone read configuration file name as 'REMOTEREAD' . Because the same is used within code.\n",
        "    So convention for creating any environment variables related to rclone-read should start with 'RCLONE_CONFIG_REMOTEREAD'.\n",
        "    Output csv files are stored in output artifacts. Thus the csv files are written like locally mounted POSIX files.\n",
        "metadata:\n",
        "  annotations:\n",
        "    author: Shaswata Jash <29448766+ShaswataJash@users.noreply.github.com>\n",
        "    canonical_location: https://raw.githubusercontent.com/ShaswataJash/kfpcomponent/main/TabularDataPreparationUsingPycaret/component_input_using_rclone_output_as_artifact.yaml\n",
        "inputs:\n",
        "- name: log_level\n",
        "  type: String\n",
        "  description: 'choice amongst ERROR, INFO, DEBUG'\n",
        "  optional: true\n",
        "- name: rclone_environment_var\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be set as environment variables before executing rclone commands'\n",
        "- name: input_datasource_directory_mountable\n",
        "  type: Boolean \n",
        "  description: 'whether input csv file is present in mountable remote location  when rclone is used'\n",
        "  optional: true\n",
        "- name: input_datasource_file_name\n",
        "  type: String\n",
        "  description: 'name of the csv file including file extension and the directory/bucket path holding the specific file(if any)  when rclone is used'\n",
        "- name: additional_options_csv_parsing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.read_csv()'\n",
        "  optional: true\n",
        "- name: type_of_data_analysis_task\n",
        "  type: String\n",
        "  description: 'choice amongst classification, regression, clustering, anomaly_detection'\n",
        "- name: target_variable_name \n",
        "  type: String\n",
        "  description: 'for classification and regression, specify the column name holding target variable'\n",
        "  optional: true\n",
        "- name: target_emptyindicator\n",
        "  type: String\n",
        "  description: 'if target variable column holds null or na, those rows will be dropped. Sometime empty can be indicated by other representative string like - or *** etc'\n",
        "  optional: true\n",
        "- name: data_preparations_options\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pycaret setup() function'\n",
        "  optional: true\n",
        "- name: additional_options_csv_writing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.to_csv()'\n",
        "  optional: true\n",
        "\n",
        "outputs:\n",
        "- name: output_datasource_local_file_path_when_rclone_bypassed\n",
        "  description: 'absolute local path of the output csv file when rclone is NOT used i.e. when output csv file is stored in output artifact of pipeline engine (e.g. argo)'\n",
        "\n",
        "implementation:\n",
        "  container:\n",
        "    image: shasjash/kfpcomponents:TabularDataPreparationUsingPycaret_devlatest\n",
        "    command:\n",
        "    - python3 \n",
        "    - /tmp/data_preparation.py\n",
        "    args:\n",
        "    - --bypass-rclone-for-output-data\n",
        "    - if:\n",
        "        cond: {isPresent: log_level}\n",
        "        then:\n",
        "        - --log-level\n",
        "        - {inputValue: log_level}\n",
        "    - --rclone-environment-var\n",
        "    - {inputValue: rclone_environment_var}\n",
        "    - if:\n",
        "        cond: {isPresent: input_datasource_directory_mountable}\n",
        "        then:\n",
        "        - --input-datasource-directory-mountable\n",
        "        - {inputValue: input_datasource_directory_mountable}\n",
        "    - --input-datasource-file-name\n",
        "    - {inputValue: input_datasource_file_name}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_parsing}\n",
        "        then:\n",
        "        - --additional-options-csv-parsing\n",
        "        - {inputValue: additional_options_csv_parsing}\n",
        "    - --type-of-data-analysis-task \n",
        "    - {inputValue: type_of_data_analysis_task}\n",
        "    - if:\n",
        "        cond: {isPresent: target_variable_name}\n",
        "        then:\n",
        "        - --target-variable-name\n",
        "        - {inputValue: target_variable_name}\n",
        "    - if:\n",
        "        cond: {isPresent: target_emptyindicator}\n",
        "        then:\n",
        "        - --target-emptyindicator\n",
        "        - {inputValue: target_emptyindicator}\n",
        "    - if:\n",
        "        cond: {isPresent: data_preparations_options}\n",
        "        then:\n",
        "        - --data-preparations-options\n",
        "        - {inputValue: data_preparations_options}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_writing}\n",
        "        then:\n",
        "        - --additional-options-csv-writing\n",
        "        - {inputValue: additional_options_csv_writing}\n",
        "    - --output-datasource-local-file-path-when-rclone-bypassed\n",
        "    - {outputPath: output_datasource_local_file_path_when_rclone_bypassed}"
      ],
      "metadata": {
        "id": "SBI4M_Blh8l5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile component_input_as_artifact_output_using_rclone.yaml\n",
        "name: TabularDataPreparationUsingPycaretWhereInputAsArtifactOutputUsingRclone\n",
        "description: |\n",
        "    Prepare tabular data (csv file) using pycaret library. (For pycaret's data pre-processing capabilities, refer https://pycaret.gitbook.io/docs/get-started/preprocessing)\n",
        "    Refer data-preparations-options in command line arguments. \n",
        "    pycaret internally uses pandas dataframe to read and write csv file. You can utilize options exposed by panda's read_csv() and to_csv(). \n",
        "    Refer additional-options-csv-parsing and additional-options-csv-writing in command line arguments\n",
        "    Output csv files can be stored in rclone compatible storage. Both mount and copy mode are supported. (refer: https://rclone.org/)\n",
        "    rclone configurations have to be shared through environment variables (refer: https://rclone.org/docs/#environment-variables). \n",
        "    Create rclone write configuration file name as 'REMOTEWRITE'. Because the same is used within code.\n",
        "    So convention for creating any environment variables related to rclone should start either with 'RCLONE_CONFIG_REMOTEWRITE'.\n",
        "    Intput csv files are stored in intput artifacts. Thus the csv files are read like locally mounted POSIX files.\n",
        "metadata:\n",
        "  annotations:\n",
        "    author: Shaswata Jash <29448766+ShaswataJash@users.noreply.github.com>\n",
        "    canonical_location: https://raw.githubusercontent.com/ShaswataJash/kfpcomponent/main/TabularDataPreparationUsingPycaret/component_input_as_artifact_output_using_rclone.yaml\n",
        "inputs:\n",
        "- name: log_level\n",
        "  type: String\n",
        "  description: 'choice amongst ERROR, INFO, DEBUG'\n",
        "  optional: true\n",
        "- name: rclone_environment_var\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be set as environment variables before executing rclone commands'\n",
        "- name: additional_options_csv_parsing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.read_csv()'\n",
        "  optional: true\n",
        "- name: type_of_data_analysis_task\n",
        "  type: String\n",
        "  description: 'choice amongst classification, regression, clustering, anomaly_detection'\n",
        "- name: target_variable_name \n",
        "  type: String\n",
        "  description: 'for classification and regression, specify the column name holding target variable'\n",
        "  optional: true\n",
        "- name: target_emptyindicator\n",
        "  type: String\n",
        "  description: 'if target variable column holds null or na, those rows will be dropped. Sometime empty can be indicated by other representative string like - or *** etc'\n",
        "  optional: true\n",
        "- name: data_preparations_options\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pycaret setup() function'\n",
        "  optional: true\n",
        "- name: additional_options_csv_writing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.to_csv()'\n",
        "  optional: true\n",
        "- name: output_datasource_directory_mountable\n",
        "  type: Boolean\n",
        "  description: 'whether output csv file will be written in mountable remote location  when rclone is used'\n",
        "  optional: true\n",
        "- name: output_datasource_file_name\n",
        "  type: String \n",
        "  description: 'filename of the prepared data including the directory/bucket path holding the specific file(if any) when rclone is used'\n",
        "- name: input_datasource_local_file_path_when_rclone_bypassed\n",
        "  description: 'absolute local path of the input csv file when rclone is NOT used i.e. when input csv file is stored in input artifact of pipeline engine (e.g. argo)'\n",
        "\n",
        "implementation:\n",
        "  container:\n",
        "    image: shasjash/kfpcomponents:TabularDataPreparationUsingPycaret_devlatest\n",
        "    command:\n",
        "    - python3 \n",
        "    - /tmp/data_preparation.py\n",
        "    args:\n",
        "    - --bypass-rclone-for-input-data\n",
        "    - if:\n",
        "        cond: {isPresent: log_level}\n",
        "        then:\n",
        "        - --log-level\n",
        "        - {inputValue: log_level}\n",
        "    - --rclone-environment-var\n",
        "    - {inputValue: rclone_environment_var}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_parsing}\n",
        "        then:\n",
        "        - --additional-options-csv-parsing\n",
        "        - {inputValue: additional_options_csv_parsing}\n",
        "    - --type-of-data-analysis-task \n",
        "    - {inputValue: type_of_data_analysis_task}\n",
        "    - if:\n",
        "        cond: {isPresent: target_variable_name}\n",
        "        then:\n",
        "        - --target-variable-name\n",
        "        - {inputValue: target_variable_name}\n",
        "    - if:\n",
        "        cond: {isPresent: target_emptyindicator}\n",
        "        then:\n",
        "        - --target-emptyindicator\n",
        "        - {inputValue: target_emptyindicator}\n",
        "    - if:\n",
        "        cond: {isPresent: data_preparations_options}\n",
        "        then:\n",
        "        - --data-preparations-options\n",
        "        - {inputValue: data_preparations_options}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_writing}\n",
        "        then:\n",
        "        - --additional-options-csv-writing\n",
        "        - {inputValue: additional_options_csv_writing}\n",
        "    - if:\n",
        "        cond: {isPresent: output_datasource_directory_mountable}\n",
        "        then:\n",
        "        - --output-datasource-directory-mountable\n",
        "        - {inputValue: output_datasource_directory_mountable}\n",
        "    - if:\n",
        "        cond: {isPresent: output_datasource_file_name}\n",
        "        then:\n",
        "        - --output-datasource-file-name\n",
        "        - {inputValue: output_datasource_file_name}\n",
        "    - --input-datasource-local-file-path-when-rclone-bypassed\n",
        "    - {inputPath: input_datasource_local_file_path_when_rclone_bypassed}"
      ],
      "metadata": {
        "id": "W5Iofu9sh9_a"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile component_both_input_output_using_rclone.yaml\n",
        "name: TabularDataPreparationUsingPycaretWhereBothInputOutputUsingRclone\n",
        "description: |\n",
        "    Prepare tabular data (csv file) using pycaret library. (For pycaret's data pre-processing capabilities, refer https://pycaret.gitbook.io/docs/get-started/preprocessing)\n",
        "    Refer data-preparations-options in command line arguments. \n",
        "    pycaret internally uses pandas dataframe to read and write csv file. You can utilize options exposed by panda's read_csv() and to_csv(). \n",
        "    Refer additional-options-csv-parsing and additional-options-csv-writing in command line arguments\n",
        "    Input and output csv files can be stored in rclone compatible storage. Both mount and copy mode are supported. (refer: https://rclone.org/)\n",
        "    rclone configurations have to be shared through environment variables (refer: https://rclone.org/docs/#environment-variables). \n",
        "    Create rclone read and write configuration file name as 'REMOTEREAD' and 'REMOTEWRITE'. Because the same are used within code.\n",
        "    So convention for creating any environment variables related to rclone should start either with 'RCLONE_CONFIG_REMOTEREAD' or 'RCLONE_CONFIG_REMOTEWRITE'.\n",
        "metadata:\n",
        "  annotations:\n",
        "    author: Shaswata Jash <29448766+ShaswataJash@users.noreply.github.com>\n",
        "    canonical_location: https://raw.githubusercontent.com/ShaswataJash/kfpcomponent/main/TabularDataPreparationUsingPycaret/component_both_input_output_using_rclone.yaml\n",
        "inputs:\n",
        "- name: log_level\n",
        "  type: String\n",
        "  description: 'choice amongst ERROR, INFO, DEBUG'\n",
        "  optional: true\n",
        "- name: rclone_environment_var\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be set as environment variables before executing rclone commands'\n",
        "- name: input_datasource_directory_mountable\n",
        "  type: Boolean \n",
        "  description: 'whether input csv file is present in mountable remote location  when rclone is used'\n",
        "  optional: true\n",
        "- name: input_datasource_file_name\n",
        "  type: String\n",
        "  description: 'name of the csv file including file extension and the directory/bucket path holding the specific file(if any)  when rclone is used'\n",
        "- name: additional_options_csv_parsing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.read_csv()'\n",
        "  optional: true\n",
        "- name: type_of_data_analysis_task\n",
        "  type: String\n",
        "  description: 'choice amongst classification, regression, clustering, anomaly_detection'\n",
        "- name: target_variable_name \n",
        "  type: String\n",
        "  description: 'for classification and regression, specify the column name holding target variable'\n",
        "  optional: true\n",
        "- name: target_emptyindicator\n",
        "  type: String\n",
        "  description: 'if target variable column holds null or na, those rows will be dropped. Sometime empty can be indicated by other representative string like - or *** etc'\n",
        "  optional: true\n",
        "- name: data_preparations_options\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pycaret setup() function'\n",
        "  optional: true\n",
        "- name: additional_options_csv_writing\n",
        "  type: String\n",
        "  description: 'json formatted key-value pairs of strings which will be passed to pandas.to_csv()'\n",
        "  optional: true\n",
        "- name: output_datasource_directory_mountable\n",
        "  type: Boolean\n",
        "  description: 'whether output csv file will be written in mountable remote location  when rclone is used'\n",
        "  optional: true\n",
        "- name: output_datasource_file_name\n",
        "  type: String \n",
        "  description: 'filename of the prepared data including the directory/bucket path holding the specific file(if any) when rclone is used'\n",
        "\n",
        "implementation:\n",
        "  container:\n",
        "    image: shasjash/kfpcomponents:TabularDataPreparationUsingPycaret_devlatest\n",
        "    command:\n",
        "    - python3 \n",
        "    - /tmp/data_preparation.py\n",
        "    args:\n",
        "    - if:\n",
        "        cond: {isPresent: log_level}\n",
        "        then:\n",
        "        - --log-level\n",
        "        - {inputValue: log_level}\n",
        "    - --rclone-environment-var\n",
        "    - {inputValue: rclone_environment_var}\n",
        "    - if:\n",
        "        cond: {isPresent: input_datasource_directory_mountable}\n",
        "        then:\n",
        "        - --input-datasource-directory-mountable\n",
        "    - --input-datasource-file-name\n",
        "    - {inputValue: input_datasource_file_name}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_parsing}\n",
        "        then:\n",
        "        - --additional-options-csv-parsing\n",
        "        - {inputValue: additional_options_csv_parsing}\n",
        "    - --type-of-data-analysis-task \n",
        "    - {inputValue: type_of_data_analysis_task}\n",
        "    - if:\n",
        "        cond: {isPresent: target_variable_name}\n",
        "        then:\n",
        "        - --target-variable-name\n",
        "        - {inputValue: target_variable_name}\n",
        "    - if:\n",
        "        cond: {isPresent: target_emptyindicator}\n",
        "        then:\n",
        "        - --target-emptyindicator\n",
        "        - {inputValue: target_emptyindicator}\n",
        "    - if:\n",
        "        cond: {isPresent: data_preparations_options}\n",
        "        then:\n",
        "        - --data-preparations-options\n",
        "        - {inputValue: data_preparations_options}\n",
        "    - if:\n",
        "        cond: {isPresent: additional_options_csv_writing}\n",
        "        then:\n",
        "        - --additional-options-csv-writing\n",
        "        - {inputValue: additional_options_csv_writing}\n",
        "    - if:\n",
        "        cond: {isPresent: output_datasource_directory_mountable}\n",
        "        then:\n",
        "        - --output-datasource-directory-mountable\n",
        "    - --output-datasource-file-name\n",
        "    - {inputValue: output_datasource_file_name}"
      ],
      "metadata": {
        "id": "bdfdOj1Zh-vX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Software testing"
      ],
      "metadata": {
        "id": "9uiUWKu4LWUU"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now let us simulate testing what will be done by docker-hub infrastructure as part of auto-testing by using docker-compose.test.yml present in github source repository."
      ],
      "metadata": {
        "id": "i3TK5UnMvfMS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -rf /tmp/my_local_dir_for_test\n",
        "!chmod 544 run_tests.sh\n",
        "!cp data_preparation.py /tmp\n",
        "!cp test_validation.py /tmp\n",
        "!./run_tests.sh"
      ],
      "metadata": {
        "id": "5i9h_W3T4H87"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Following test uses Google Cloud Storage (GCS). To share the credential of gcs, we will create service account which can access the GCS on end-user's behalf. For that refer \n",
        "https://cloud.google.com/iam/docs/service-accounts (Go to 'User-managed service accounts' section)\n",
        "create a new user-managed service account for accessing gcs bucket. Under 'Grant this service account access to project' option, find roles filter by google product -> select -> roles for 'cloud storage' -> provide roles corresponding to object creation in bucket. Once service account is created, download the key file and preserve it secretly in your personal capacity. Refer: https://cloud.google.com/iam/docs/creating-managing-service-account-keys for how to generate the service-account json file. Rename that file as 'sa_gcs_service_account.json' and then upload to google colab using 'upload to session storage' icon.\n",
        "\n",
        "It is needless to say that before running these tests, two buckets were created - namely -(a)kfpcomponent and (b)shastest. If you want to run the below test for other bucket name, please change the test command accordingly. These test cannot be made part of docker-UT as it will require GCS service account related confidential file to be exposed in docker."
      ],
      "metadata": {
        "id": "D-AExe5ISYMs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "with open('/content/sa_gcs_service_account.json','r') as file:\n",
        "    gcs_sa_content = json.loads(file.read())\n",
        "\n",
        "rclone_env_str = '{\"RCLONE_CONFIG_REMOTEREAD_TYPE\":\"gcs\",' + \\\n",
        "                 '\"RCLONE_CONFIG_REMOTEREAD_SERVICE_ACCOUNT_CREDENTIALS\":{},'.format(json.dumps(gcs_sa_content)) + \\\n",
        "                 '\"RCLONE_CONFIG_REMOTEREAD_LOCATION\":\"us-east1\",' + \\\n",
        "                 '\"RCLONE_CONFIG_REMOTEWRITE_TYPE\":\"gcs\",' + \\\n",
        "                 '\"RCLONE_CONFIG_REMOTEWRITE_SERVICE_ACCOUNT_CREDENTIALS\":{},'.format(json.dumps(gcs_sa_content)) + \\\n",
        "                 '\"RCLONE_CONFIG_REMOTEWRITE_LOCATION\":\"us-east1\",' + \\\n",
        "                 '\"RCLONE_CONFIG_REMOTEWRITE_BUCKET_POLICY_ONLY\":\"true\"}'\n",
        "\n",
        "print(rclone_env_str)\n",
        "#to remain compatible with bash command line argument (refer below how $rclone_env_str is passed in %%bash)\n",
        "#Read: handling of double quotes in bash - https://stackoverflow.com/questions/19579546/can-i-access-python-variables-within-a-bash-or-script-ipython-notebook-c\n",
        "rclone_env_str = rclone_env_str.replace('\"', '\\\\\"') \n",
        "\n",
        "print(rclone_env_str)"
      ],
      "metadata": {
        "id": "W3-AwP_4rJXm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash -s \"$rclone_env_str\"\n",
        "rm -rf /tmp/my_local_dir_for_test\n",
        "cp data_preparation.py /tmp\n",
        "mkdir /tmp/my_local_dir_for_test\n",
        "#note the difference in command line handling with double quoted and single quoted string. if single quoted, the inside string is passed into the program as it is.\n",
        "#if double quoted, the inside string is evaluated - as part of that all double quotes are further removed unless they are escaped.\n",
        "python /tmp/data_preparation.py --rclone-environment-var \"$1\" \\\n",
        "    --input-datasource-file-name 'kfpcomponent/CTG.csv' --additional-options-csv-parsing '{\"sep\":\",\" , \"header\":0}' \\\n",
        "    --type-of-data-analysis-task 'classification' --target-variable-name 'NSP' \\\n",
        "    --data-preparations-options '{\"ignore_low_variance\":true, \"remove_outliers\":true, \"remove_multicollinearity\":true, \"multicollinearity_threshold\":0.7}' \\\n",
        "    --output-datasource-file-name 'shastest/CTG_data-prep.csv' \\\n",
        "    --additional-options-csv-writing '{\"index\":false}' --log-level 'DEBUG'"
      ],
      "metadata": {
        "id": "-3O-lwDsSXKZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#validation test to check file was correctly uploaded in GCS storage.\n",
        "from google.cloud import storage\n",
        "import os\n",
        "os.environ[\"GOOGLE_APPLICATION_CREDENTIALS\"] = \"/content/sa_gcs_service_account.json\"\n",
        "\n",
        "client = storage.Client()\n",
        "with open('/content/CTG_data-prep_downloaded_for_test.csv', mode='wb') as file_obj:\n",
        "    client.download_blob_to_file('gs://shastest/CTG_data-prep.csv', file_obj)\n",
        "\n",
        "import pandas\n",
        "df = pandas.read_csv(filepath_or_buffer = '/content/CTG_data-prep_downloaded_for_test.csv')\n",
        "assert len(df.index) == 2126 #original data had 2129 rows, amongst that 3 rows have no target\n",
        "assert df.isnull().sum().sum() == 0 #pycaret will remove all missing values"
      ],
      "metadata": {
        "id": "Qy0jbeumXKKk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "! pip3 install kfp==1.8.12"
      ],
      "metadata": {
        "id": "jhPf6ZSOnv0m"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "First validate the component.yaml file in http://www.yamllint.com/. Once component.yaml file is corrected, execute the below cell to finally check"
      ],
      "metadata": {
        "id": "tQalOIRBDzSS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import kfp\n",
        "\n",
        "csv_data_prepare_op_out_to_artifact = kfp.components.load_component_from_file('component_input_using_rclone_output_as_artifact.yaml')\n",
        "csv_data_prepare_op_both_in_out_artifact = kfp.components.load_component_from_file('component_both_input_output_as_artifact.yaml')\n",
        "\n",
        "@kfp.dsl.pipeline(name=\"testpipeline1\")\n",
        "def my_sample_pipeline_first_step_input_using_rclone_rest_from_artifacts():\n",
        "    csv_prepared_file_step1 = csv_data_prepare_op_out_to_artifact(rclone_environment_var='{\"RCLONE_CONFIG_REMOTEREAD_TYPE\":\"http\", \"RCLONE_CONFIG_REMOTEREAD_URL\":\"https://raw.githubusercontent.com/pycaret/datasets/main/data/common/\"}',\n",
        "                                            input_datasource_file_name='CTG.csv',\n",
        "                                            type_of_data_analysis_task='classification',\n",
        "                                            target_variable_name='NSP',\n",
        "                                            data_preparations_options='{\"ignore_low_variance\":true, \"remove_outliers\":true}', \n",
        "                                            additional_options_csv_writing='{\"index\":false}'\n",
        "                                            ).outputs['output_datasource_local_file_path_when_rclone_bypassed']\n",
        "\n",
        "    csv_data_prepare_op_both_in_out_artifact(input_datasource_local_file_path_when_rclone_bypassed = csv_prepared_file_step1,\n",
        "                                            type_of_data_analysis_task='classification',\n",
        "                                            target_variable_name='NSP',\n",
        "                                            data_preparations_options='{\"remove_multicollinearity\":true, \"multicollinearity_threshold\":0.7}', \n",
        "                                            additional_options_csv_writing='{\"index\":false}'\n",
        "                                            )\n",
        "\n",
        "\n",
        "kfp.compiler.Compiler().compile(\n",
        "    pipeline_func=my_sample_pipeline_first_step_input_using_rclone_rest_from_artifacts,\n",
        "    package_path='my_sample_pipeline_first_step_input_using_rclone_rest_from_artifacts.yaml')\n",
        "\n",
        "kfp.v2.compiler.Compiler().compile(\n",
        "    pipeline_func=my_sample_pipeline_first_step_input_using_rclone_rest_from_artifacts,\n",
        "    package_path='my_sample_pipeline_first_step_input_using_rclone_rest_from_artifacts_v2.json')\n"
      ],
      "metadata": {
        "id": "54vGWKvUDu8E"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import kfp\n",
        "import json\n",
        "csv_data_prepare_op_both_in_out_using_rclone = kfp.components.load_component_from_file('component_both_input_output_using_rclone.yaml')\n",
        "\n",
        "@kfp.dsl.pipeline(name=\"testpipeline2\")\n",
        "def my_sample_pipeline_all_steps_using_rclone(rclone_env_val:str):\n",
        "    \n",
        "    step1_op = csv_data_prepare_op_both_in_out_using_rclone(rclone_environment_var=rclone_env_val,\n",
        "                                            #input_datasource_directory_mountable=True,\n",
        "                                            input_datasource_file_name='kfpcomponent/CTG.csv',\n",
        "                                            type_of_data_analysis_task='classification',\n",
        "                                            target_variable_name='NSP',\n",
        "                                            data_preparations_options='{\"ignore_low_variance\":true, \"remove_outliers\":true}', \n",
        "                                            additional_options_csv_writing='{\"index\":false}',\n",
        "                                            #output_datasource_directory_mountable=True,\n",
        "                                            output_datasource_file_name='shastest/CTG_data-prep1.csv',\n",
        "                                            log_level='DEBUG'\n",
        "                                            ).set_cpu_limit('1')  \n",
        "\n",
        "    csv_data_prepare_op_both_in_out_using_rclone(rclone_environment_var=rclone_env_val,\n",
        "                                            #input_datasource_directory_mountable=True,\n",
        "                                            input_datasource_file_name='shastest/CTG_data-prep1.csv',\n",
        "                                            type_of_data_analysis_task='classification',\n",
        "                                            target_variable_name='NSP',\n",
        "                                            data_preparations_options='{\"remove_multicollinearity\":true, \"multicollinearity_threshold\":0.7}', \n",
        "                                            additional_options_csv_writing='{\"index\":false}',\n",
        "                                            #output_datasource_directory_mountable=True,\n",
        "                                            output_datasource_file_name='shastest/CTG_data-prep2.csv',\n",
        "                                            log_level='DEBUG'\n",
        "                                            ).set_cpu_limit('1').after(step1_op)\n",
        "\n",
        "\n",
        "kfp.compiler.Compiler().compile(\n",
        "    pipeline_func=my_sample_pipeline_all_steps_using_rclone,\n",
        "    package_path='my_sample_pipeline_all_steps_using_rclone.yaml')\n",
        "\n",
        "kfp.v2.compiler.Compiler().compile(\n",
        "    pipeline_func=my_sample_pipeline_all_steps_using_rclone,\n",
        "    package_path='my_sample_pipeline_all_steps_using_rclone_v2.json')"
      ],
      "metadata": {
        "id": "-I1kb0j45dBc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Push the code to github"
      ],
      "metadata": {
        "id": "a62pERh-Lasc"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Before commiting code to github, install github client (gh) by following instruction mentioned in https://github.com/cli/cli/blob/trunk/docs/install_linux.md (Choose Debian, Ubuntu Linux way of installation) \n",
        "\n",
        "Use the colab's 'Terminal' icon present in left vertical pane to open linux terminal to type commands. Once 'gh' is installed, type **$gh auth login** (refer https://docs.github.com/en/get-started/getting-started-with-git/caching-your-github-credentials-in-git) to follow onscreen prompts. For colab, use **Paste an authentication token** option. Personal tokens can be generated in https://github.com/settings/tokens\n",
        "\n",
        "You can use Shift+Ctrl+v shortcut to paste any string in colab console"
      ],
      "metadata": {
        "id": "2Nix3R2joPdP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pwd"
      ],
      "metadata": {
        "id": "G12wQfVT1ann"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!rm -Rf kfpcomponent"
      ],
      "metadata": {
        "id": "JVc_RmC9MrgG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/ShaswataJash/kfpcomponent.git"
      ],
      "metadata": {
        "id": "S5LCKX0SXUxs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Follow directory structure according to https://www.kubeflow.org/docs/components/pipelines/sdk/component-development/#organizing-the-component-files"
      ],
      "metadata": {
        "id": "AIwZrmfnCXPW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir kfpcomponent/TabularDataPreparationUsingPycaret\n",
        "!mkdir kfpcomponent/TabularDataPreparationUsingPycaret/src\n",
        "!mkdir kfpcomponent/TabularDataPreparationUsingPycaret/tests"
      ],
      "metadata": {
        "id": "PWT-29j1XYhi"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#it will ensure file is coped in git repo only if file content is changed by checking checksum of file content\n",
        "!rsync -c data_preparation.py kfpcomponent/TabularDataPreparationUsingPycaret/src\n",
        "!rsync -c component_both_input_output_as_artifact.yaml kfpcomponent/TabularDataPreparationUsingPycaret/component_both_input_output_as_artifact.yaml\n",
        "!rsync -c component_both_input_output_using_rclone.yaml kfpcomponent/TabularDataPreparationUsingPycaret/component_both_input_output_using_rclone.yaml\n",
        "!rsync -c component_input_as_artifact_output_using_rclone.yaml kfpcomponent/TabularDataPreparationUsingPycaret/component_input_as_artifact_output_using_rclone.yaml\n",
        "!rsync -c component_input_using_rclone_output_as_artifact.yaml kfpcomponent/TabularDataPreparationUsingPycaret/component_input_using_rclone_output_as_artifact.yaml\n",
        "!rsync -c test_validation.py kfpcomponent/TabularDataPreparationUsingPycaret/tests\n",
        "!rsync -c Dockerfile kfpcomponent/TabularDataPreparationUsingPycaret/\n",
        "!rsync -c run_tests.sh kfpcomponent/TabularDataPreparationUsingPycaret/\n",
        "!rsync -c docker-compose.test.yml kfpcomponent/TabularDataPreparationUsingPycaret/"
      ],
      "metadata": {
        "id": "n3b7gcGhXde6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd kfpcomponent"
      ],
      "metadata": {
        "id": "1RUDYMnmXlIW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git add -A"
      ],
      "metadata": {
        "id": "iAyaTjClX9S0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git status"
      ],
      "metadata": {
        "id": "hW4fzKBsYCGO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "For git-user who has set their email visibility as private, git provides alternate email address to use in web-based Git operations, e.g., edits and merges. The alias email can be viewed in https://github.com/settings/emails"
      ],
      "metadata": {
        "id": "bmHCKK7CzXNU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!git config --global user.email \"29448766+ShaswataJash@users.noreply.github.com\""
      ],
      "metadata": {
        "id": "FBLk0UVR04lj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git commit -a -m \"introduced autoremove in docker\""
      ],
      "metadata": {
        "id": "VbaNqafyYHV7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!git push origin main"
      ],
      "metadata": {
        "id": "uLZfkrleaqQ3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd .."
      ],
      "metadata": {
        "id": "LwhyFmmz5bnu"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}