Muhammad Ramzan Khan KavSkills AI Assignment 02.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6b6c19f5",
   "metadata": {},
   "source": [
    "### Using Pandas read dataset and print it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc9ac207",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "url=r\"C:\\Users\\Ramzan Khan\\Downloads\\Compressed\\Housing.csv\"\n",
    "df= pd.read_csv(url, header=None)\n",
    "df.head(10)\n",
    "df.tail(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b6fc567",
   "metadata": {},
   "source": [
    "### Perform Data Analysis using house price prediction dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d1f65f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf48e531",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2a586bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.describe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcafe178",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_json('dataset.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc610f8c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc17c7d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.info"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5aa84044",
   "metadata": {},
   "source": [
    "### Write 5 Numpy Programs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2437b411",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Create a 1D array\n",
    "arr1d = np.array([1, 2, 3, 4, 5])\n",
    "\n",
    "# Create a 2D array (matrix)\n",
    "arr2d = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])\n",
    "print(f\"first 1D array {arr1d}\")\n",
    "print(f\"second 2D array \\n{arr2d}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8243c629",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Addition\n",
    "arr1 = np.array([[1, 2, 3],[4,3,2]])\n",
    "arr2 = np.array([4, 5, 6])\n",
    "result = arr1 + arr2\n",
    "\n",
    "# Multiplication\n",
    "arr3 = np.array([2, 3, 4])\n",
    "result2 = arr1 * arr3\n",
    "\n",
    "print(result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d79af1f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "arr = np.array([1, 2, 3, 4, 5])\n",
    "\n",
    "# Slice elements from index 1 to 3 (exclusive)\n",
    "sliced_arr = arr[1:3]\n",
    "\n",
    "print(sliced_arr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "274f6e77",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "matrix1 = np.array([[1, 2], [3, 4]])\n",
    "matrix2 = np.array([[5, 6], [7, 8]])\n",
    "\n",
    "# Matrix multiplication\n",
    "result_matrix = np.dot(matrix1, matrix2)\n",
    "\n",
    "# Transpose of a matrix\n",
    "transposed_matrix = np.transpose(matrix1)\n",
    "\n",
    "print(transposed_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1edb794b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "arr = np.array([1, 3, 3, 4, 5])\n",
    "\n",
    "# Reshape the array\n",
    "reshaped_arr = arr.reshape(5, 1)\n",
    "\n",
    "# Append values to the array\n",
    "new_arr = np.append(arr, [6, 7, 8])\n",
    "\n",
    "# Find the unique elements in the array\n",
    "unique_values = np.unique(arr)\n",
    "\n",
    "print(unique_values)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53b3895b",
   "metadata": {},
   "source": [
    "### Display two Matplot and Searborn chart using any dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6d45651",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_iris\n",
    "\n",
    "# Load the Iris dataset\n",
    "iris = load_iris()\n",
    "data = iris.data\n",
    "target = iris.target\n",
    "\n",
    "# Create a scatter plot using Matplotlib\n",
    "plt.scatter(data[:, 0], data[:, 1], c=target, cmap=plt.cm.Set1, edgecolor='k')\n",
    "plt.xlabel('Sepal Length (cm)')\n",
    "plt.ylabel('Sepal Width (cm)')\n",
    "plt.title('Iris Dataset: Sepal Length vs Sepal Width')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5fab977",
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "\n",
    "# Load the Iris dataset as a pandas DataFrame\n",
    "iris = sns.load_dataset(\"iris\")\n",
    "\n",
    "# Create a pairplot using Seaborn\n",
    "sns.set(style=\"ticks\")\n",
    "sns.pairplot(iris, hue=\"species\", markers=[\"o\", \"s\", \"D\"])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e73e1f52",
   "metadata": {},
   "source": [
    "### Write 5 Scipy Programs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3aa0962",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.stats import linregress\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Sample data\n",
    "x = np.array([1, 2, 3, 4, 5])\n",
    "y = np.array([2, 4, 5, 4, 5])\n",
    "\n",
    "# Perform linear regression\n",
    "slope, intercept, r_value, p_value, std_err = linregress(x, y)\n",
    "\n",
    "# Predict y values\n",
    "predicted_y = slope * x + intercept\n",
    "\n",
    "# Plot the data and regression line\n",
    "plt.scatter(x, y, label='Data')\n",
    "plt.plot(x, predicted_y, label='Linear Regression', color='red')\n",
    "plt.legend()\n",
    "plt.title('Linear Regression with SciPy')\n",
    "plt.xlabel('X')\n",
    "plt.ylabel('Y')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fe944e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.optimize import curve_fit\n",
    "\n",
    "# Sample data\n",
    "x = np.linspace(0, 10, 50)\n",
    "y = 2 * np.sin(1.5 * x) + 0.5 * np.random.randn(50)\n",
    "\n",
    "# Define the function to fit\n",
    "def model_function(x, a, b):\n",
    "    return a * np.sin(b * x)\n",
    "\n",
    "# Perform curve fitting\n",
    "params, covariance = curve_fit(model_function, x, y)\n",
    "\n",
    "# Extract fitted parameters\n",
    "a, b = params\n",
    "\n",
    "# Generate fitted curve\n",
    "fitted_curve = model_function(x, a, b)\n",
    "\n",
    "# Plot the data and fitted curve\n",
    "plt.scatter(x, y, label='Data')\n",
    "plt.plot(x, fitted_curve, label='Fitted Curve', color='red')\n",
    "plt.legend()\n",
    "plt.title('Nonlinear Curve Fitting with SciPy')\n",
    "plt.xlabel('X')\n",
    "plt.ylabel('Y')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05419e44",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "\n",
    "# Objective function to minimize\n",
    "def objective_function(x):\n",
    "    return (x[0] - 2) ** 2 + (x[1] - 3) ** 2\n",
    "\n",
    "# Initial guess\n",
    "initial_guess = [0, 0]\n",
    "\n",
    "# Perform optimization\n",
    "result = minimize(objective_function, initial_guess, method='BFGS')\n",
    "\n",
    "# Extract optimized values\n",
    "optimized_values = result.x\n",
    "\n",
    "print(\"Optimized values:\", optimized_values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4b862171",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.interpolate import interp1d\n",
    "\n",
    "# Sample data\n",
    "x = np.linspace(0, 10, 10)\n",
    "y = np.sin(x)\n",
    "\n",
    "# Create an interpolation function\n",
    "interpolation_function = interp1d(x, y, kind='linear')\n",
    "\n",
    "# Generate points for the interpolated curve\n",
    "x_interp = np.linspace(0, 10, 100)\n",
    "y_interp = interpolation_function(x_interp)\n",
    "\n",
    "# Plot the original data and interpolated curve\n",
    "plt.plot(x, y, 'o', label='Data')\n",
    "plt.plot(x_interp, y_interp, label='Interpolated Curve', color='red')\n",
    "plt.legend()\n",
    "plt.title('Interpolation with SciPy')\n",
    "plt.xlabel('X')\n",
    "plt.ylabel('Y')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8454efb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.fft import fft\n",
    "\n",
    "# Sample data\n",
    "fs = 1000  # Sampling frequency\n",
    "t = np.linspace(0, 1, fs, endpoint=False)  # Time vector\n",
    "freq = 5  # Frequency of the signal\n",
    "signal = np.sin(2 * np.pi * freq * t)  # Signal with a frequency of 5 Hz\n",
    "\n",
    "# Perform Discrete Fourier Transform (DFT)\n",
    "dft = fft(signal)\n",
    "\n",
    "# Frequency domain representation\n",
    "frequencies = np.fft.fftfreq(len(dft), 1 / fs)\n",
    "\n",
    "# Plot the original signal and its frequency domain representation\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.subplot(2, 1, 1)\n",
    "plt.plot(t, signal)\n",
    "plt.title('Original Signal (Time Domain)')\n",
    "plt.xlabel('Time (s)')\n",
    "plt.ylabel('Amplitude')\n",
    "\n",
    "plt.subplot(2, 1, 2)\n",
    "plt.plot(frequencies, np.abs(dft))\n",
    "plt.title('DFT (Frequency Domain)')\n",
    "plt.xlabel('Frequency (Hz)')\n",
    "plt.ylabel('Amplitude')\n",
    "plt.xlim(0, 10)  # Display up to 10 Hz\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f54fe264",
   "metadata": {},
   "source": [
    "### Scrap data from OLX website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ccbd0d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "!conda install beautifulsoup4 requests lxml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58b473b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "title=[]\n",
    "price=[]\n",
    "location=[]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57cb59b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# URL of the Amazon search page\n",
    "url = \"https://www.olx.com.pk/items/q-laptop\"\n",
    "\n",
    "# Send an HTTP GET request to the URL\n",
    "response = requests.get(url)\n",
    "\n",
    "# Parse the HTML content of the page using BeautifulSoup\n",
    "soup = BeautifulSoup(response.text, \"lxml\")\n",
    "\n",
    "prices=soup.find_all('div', class_=\"_1075545d _52497c97 _96d4439a\")\n",
    "titles=soup.find_all('div', class_=\"a5112ca8 _5fdf4379\")\n",
    "locations=soup.find_all('span', class_=\"_2fc90438\")\n",
    "\n",
    "# title\n",
    "for t, p, l in zip (titles, prices, locations) :\n",
    "    title.append(t.get_text())\n",
    "    price.append(p.get_text())\n",
    "    location.append(l.get_text())\n",
    "\n",
    "data={\n",
    "        'title':title,\n",
    "        'price':price,\n",
    "        'location':location\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3bbde53",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38c9cf17",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea5b22c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b20cf92",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('olx_laptop.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b7fee78",
   "metadata": {},
   "source": [
    "### Scraping with selenium and Bueatiful Soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a85a866d",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40473b46",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from bs4 import BeautifulSoup\n",
    "from lxml import etree\n",
    "\n",
    "# Initialize Chrome WebDriver (make sure to specify the path to your WebDriver)\n",
    "driver = webdriver.Chrome()\n",
    "\n",
    "# URL of OLX search results page\n",
    "url = 'https://www.olx.com.pk/items/q-laptop'\n",
    "\n",
    "# Number of records to scrape\n",
    "num_records_to_scrape = 500\n",
    "\n",
    "# Counter to keep track of scraped records\n",
    "scraped_records = 0\n",
    "\n",
    "\n",
    "\n",
    "driver.get(url)\n",
    "time.sleep(5)  # Wait for the page to load\n",
    "title=[]\n",
    "price=[]\n",
    "location=[]\n",
    "\n",
    "while scraped_records < num_records_to_scrape:\n",
    " \n",
    "    # Scroll down to load more listings (adjust the number of scrolls as needed)\n",
    "    for _ in range(5):\n",
    "        driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
    "        time.sleep(2)\n",
    "\n",
    "    # Get the page source\n",
    "    page_source = driver.page_source\n",
    "\n",
    "    # Parse the page source with BeautifulSoup\n",
    "    soup = BeautifulSoup(page_source, 'lxml')\n",
    "    \n",
    "\n",
    "    prices=soup.find_all('div', class_=\"_1075545d _52497c97 _96d4439a\")\n",
    "    titles=soup.find_all('div', class_=\"a5112ca8 _5fdf4379\")\n",
    "    locations=soup.find_all('span', class_=\"_2fc90438\")\n",
    "    \n",
    "    \n",
    "        # Increment the counter\n",
    "    scraped_records += 20\n",
    "    driver.find_element(By.XPATH, '//*[@id=\"body-wrapper\"]/div[1]/header[2]/div/div/div/div[2]/div[2]/div[2]/div/a/button').click()\n",
    "\n",
    "    if scraped_records >= num_records_to_scrape:\n",
    "        for t, p, l in zip (titles, prices, locations) :\n",
    "            title.append(t.get_text())\n",
    "            price.append(p.get_text())\n",
    "            location.append(l.get_text())\n",
    "        data={\n",
    "                'title':title,\n",
    "                'price':price,\n",
    "                'location':location\n",
    "              }\n",
    "        break\n",
    "\n",
    "# Close the WebDriver\n",
    "driver.quit()\n",
    "\n",
    "# Print or save the scraped data as needed\n",
    "for item in data:\n",
    "    print(item)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc2eca50",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df=pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f4cb375",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['price'] = df['price'].str.replace('Rs ', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f8b1b91",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['location'] = df['location'].str.split(',').str[-1].str.replace('•','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4255db4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8163ecf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('olx_laptop_data.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ba6cc29",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}