notebooks/02_create_pipeline.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.6.12 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "23181a1f8d092a550afd91e2df594161a080a135fdd639e3f5e4e05e6251c17d"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "source": [
    "# Machine Learning Pipeline with Python"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## Connect to the Workspace"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Workspace, Datastore, Experiment\n",
    "\n",
    "\n",
    "ws = Workspace.from_config(\".azure\") \n",
    "datastore = ws.get_default_datastore()\n",
    "compute_target = ws.compute_targets[\"cpu-cluster\"]"
   ]
  },
  {
   "source": [
    "## Create a new Experiment"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment = Experiment(ws, 'Pipeline_Tutorial') "
   ]
  },
  {
   "source": [
    "## Simgle Step Pipeline"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Overwriting src/sample_script_1.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/sample_script_1.py\n",
    "\n",
    "print(\"Sample_1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.pipeline.steps import PythonScriptStep\n",
    "from azureml.pipeline.core import Pipeline\n",
    "\n",
    "# Define the Python script to run\n",
    "step1 = PythonScriptStep(\n",
    "    name=\"sample_step\",\n",
    "    source_directory=\"src\",\n",
    "    script_name=\"sample_script_1.py\",\n",
    "    compute_target=compute_target)\n",
    "\n",
    "# create the Pipeline and add the steps\n",
    "pipeline = Pipeline(workspace=ws, steps=[step1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Created step sample_step [992dfe4b][3ae14a81-0efc-4487-9d7f-a326cd3eae05], (This step will run and generate new outputs)\n",
      "Submitted PipelineRun 20d38dd5-6149-494b-8244-5a8ac48681df\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/Pipeline_Tutorial/runs/20d38dd5-6149-494b-8244-5a8ac48681df?wsid=/subscriptions/3a0172d3-ec0d-46bb-a88a-ff41a302711a/resourcegroups/Evonik/workspaces/AMLWorkspace\n",
      "PipelineRunId: 20d38dd5-6149-494b-8244-5a8ac48681df\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/Pipeline_Tutorial/runs/20d38dd5-6149-494b-8244-5a8ac48681df?wsid=/subscriptions/3a0172d3-ec0d-46bb-a88a-ff41a302711a/resourcegroups/Evonik/workspaces/AMLWorkspace\n",
      "PipelineRun Status: NotStarted\n",
      "PipelineRun Status: Running\n",
      "Expected a StepRun object but received <class 'azureml.core.run.Run'> instead.\n",
      "This usually indicates a package conflict with one of the dependencies of azureml-core or azureml-pipeline-core.\n",
      "Please check for package conflicts in your python environment\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "PipelineRun Execution Summary\n",
      "==============================\n",
      "PipelineRun Status: Finished\n",
      "{'runId': '20d38dd5-6149-494b-8244-5a8ac48681df', 'status': 'Completed', 'startTimeUtc': '2021-02-09T12:18:59.181688Z', 'endTimeUtc': '2021-02-09T12:26:00.772939Z', 'properties': {'azureml.runsource': 'azureml.PipelineRun', 'runSource': 'SDK', 'runType': 'SDK', 'azureml.parameters': '{}'}, 'inputDatasets': [], 'outputDatasets': [], 'logFiles': {'logs/azureml/executionlogs.txt': 'https://amlstorageacc.blob.core.windows.net/azureml/ExperimentRun/dcid.20d38dd5-6149-494b-8244-5a8ac48681df/logs/azureml/executionlogs.txt?sv=2019-02-02&sr=b&sig=%2BjqWDsqAKg%2BdYhw93PjC8H3kAd9qGnGc5C51wRrkgzM%3D&st=2021-02-09T12%3A11%3A28Z&se=2021-02-09T20%3A21%3A28Z&sp=r', 'logs/azureml/stderrlogs.txt': 'https://amlstorageacc.blob.core.windows.net/azureml/ExperimentRun/dcid.20d38dd5-6149-494b-8244-5a8ac48681df/logs/azureml/stderrlogs.txt?sv=2019-02-02&sr=b&sig=TyqYHoQIP8H9t%2Fq1kZ4LIR06%2FfkWOPnzJwKjqBCaDx0%3D&st=2021-02-09T12%3A11%3A28Z&se=2021-02-09T20%3A21%3A28Z&sp=r', 'logs/azureml/stdoutlogs.txt': 'https://amlstorageacc.blob.core.windows.net/azureml/ExperimentRun/dcid.20d38dd5-6149-494b-8244-5a8ac48681df/logs/azureml/stdoutlogs.txt?sv=2019-02-02&sr=b&sig=jnbBlCvwp%2F%2BRJxI5c7wNVpHI4Tl3x5s9aRVZqv9lhc0%3D&st=2021-02-09T12%3A11%3A28Z&se=2021-02-09T20%3A21%3A28Z&sp=r'}, 'submittedBy': 'Bjarne Meyn'}\n",
      "\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'Finished'"
      ]
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "pipeline_run = experiment.submit(pipeline)\n",
    "pipeline_run.wait_for_completion()"
   ]
  },
  {
   "source": [
    "## Multistep Pipeline"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "source": [
    "## Define another python execution script"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Overwriting src/sample_script_2.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile src/sample_script_2.py\n",
    "\n",
    "print(\"Sample_2\")"
   ]
  },
  {
   "source": [],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.pipeline.core import Pipeline, PipelineData \n",
    "from azureml.pipeline.steps import PythonScriptStep\n",
    "\n",
    "# Define the Python script to run\n",
    "step2 = PythonScriptStep(\n",
    "    name=\"step2\",\n",
    "    source_directory = \"src\",\n",
    "    script_name=\"sample_script_2.py\",\n",
    "    compute_target=compute_target)\n",
    "\n",
    "# create the Pipeline and add the steps\n",
    "multistep_pipe = Pipeline(workspace=ws, steps=[step1, step2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Submitted PipelineRun 0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/Pipeline_Tutorial/runs/0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a?wsid=/subscriptions/3a0172d3-ec0d-46bb-a88a-ff41a302711a/resourcegroups/Evonik/workspaces/AMLWorkspace\n",
      "PipelineRunId: 0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/experiments/Pipeline_Tutorial/runs/0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a?wsid=/subscriptions/3a0172d3-ec0d-46bb-a88a-ff41a302711a/resourcegroups/Evonik/workspaces/AMLWorkspace\n",
      "PipelineRun Status: NotStarted\n",
      "PipelineRun Status: Running\n",
      "Expected a StepRun object but received <class 'azureml.core.run.Run'> instead.\n",
      "This usually indicates a package conflict with one of the dependencies of azureml-core or azureml-pipeline-core.\n",
      "Please check for package conflicts in your python environment\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "PipelineRun Execution Summary\n",
      "==============================\n",
      "PipelineRun Status: Finished\n",
      "{'runId': '0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a', 'status': 'Completed', 'startTimeUtc': '2021-02-08T18:51:08.544468Z', 'endTimeUtc': '2021-02-08T18:51:14.092628Z', 'properties': {'azureml.runsource': 'azureml.PipelineRun', 'runSource': 'SDK', 'runType': 'SDK', 'azureml.parameters': '{}'}, 'inputDatasets': [], 'outputDatasets': [], 'logFiles': {'logs/azureml/executionlogs.txt': 'https://amlstorageacc.blob.core.windows.net/azureml/ExperimentRun/dcid.0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a/logs/azureml/executionlogs.txt?sv=2019-02-02&sr=b&sig=AtREIH%2Bpk3hG65IIBBcyg8cXn%2F5DNcMoC2Zcmd69SkU%3D&st=2021-02-08T18%3A41%3A17Z&se=2021-02-09T02%3A51%3A17Z&sp=r', 'logs/azureml/stderrlogs.txt': 'https://amlstorageacc.blob.core.windows.net/azureml/ExperimentRun/dcid.0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a/logs/azureml/stderrlogs.txt?sv=2019-02-02&sr=b&sig=d3V3Q0Ca7FyNgvTQeDXgVftoGmtqMesCr2GAV5XB9O0%3D&st=2021-02-08T18%3A41%3A18Z&se=2021-02-09T02%3A51%3A18Z&sp=r', 'logs/azureml/stdoutlogs.txt': 'https://amlstorageacc.blob.core.windows.net/azureml/ExperimentRun/dcid.0a7562a7-fc67-49f6-afb0-ee3a0e9f0b0a/logs/azureml/stdoutlogs.txt?sv=2019-02-02&sr=b&sig=cg%2BOtjKBWXOdVrHGb1YIqlWV3x0yyiwJi8nXW2Xz6vc%3D&st=2021-02-08T18%3A41%3A18Z&se=2021-02-09T02%3A51%3A18Z&sp=r'}, 'submittedBy': 'Bjarne Meyn'}\n",
      "\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'Finished'"
      ]
     },
     "metadata": {},
     "execution_count": 10
    }
   ],
   "source": [
    "# Run the Pipeline as an experiment\n",
    "pipeline_run = experiment.submit(pipeline)\n",
    "pipeline_run.wait_for_completion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}