Order_list.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled0.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNvkf2UBE9J2q8Xd3+s2Qj/",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TeamPrigge/MyProject/blob/master/Order_list.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2X4CzI7g_he2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import csv\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from scipy import stats\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xHdgTUmSFmS1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# You have to use the encoding as latin1 to read this file as there are some special character in this file...\n",
        "df = pd.read_csv (r'Order_list_lage.csv',encoding='latin1')\n",
        "df.head()\n",
        "# with open('order_list_Bestellung.csv', newline='') as f:\n",
        "#     reader = csv.reader(f)\n",
        "#     for column in reader:\n",
        "#         objects=(column[1])\n",
        "#         print(objects)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "sorted_by_product = df.sort_values(['Product'], ascending=False)\n",
        "\n",
        "b= sorted_by_product['Product'].values #.astype(str)\n",
        "c= b.astype(str)\n",
        "a = sorted_by_product['Product'].value_counts()\n",
        "# print(b[1])\n",
        "# print(c)\n",
        "type(c)\n",
        "print(c)\n",
        "# len(c)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "sorted_by_product[\"Product\"].head(10)\n",
        "\n",
        "plt.figure(figsize=(20,40))\n",
        "a.plot(kind='barh')\n",
        "\n",
        "sorted_by_product[\"Product\"].tail(10)\n",
        "\n",
        "#This package provides helpers for computing similarities between arbitrary sequences. \n",
        "#Included metrics are Levenshtein, Hamming, Jaccard, and Sorensen distance, \n",
        "#plus some bonuses. All distance computations are implemented in pure Python, \n",
        "#and most of them are also implemented in C.\n",
        "pip install Distance\n",
        "\n",
        "import re\n",
        "import pandas as pd\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "# Import your data to a Pandas.DataFrame\n",
        "df = pd.read_csv('Order_list_lage.csv',encoding='latin1')\n",
        "\n",
        "# Grab the column you'd like to group, filter out duplicate values\n",
        "# and make sure the values are Unicode\n",
        "# type(b)\n",
        "vals = df['Product'].astype('U')#unique().astype('U')\n",
        "# print(vals)\n",
        "\n",
        "\n",
        "# Write a function for cleaning strings and returning an array of ngrams\n",
        "def ngrams_analyzer(string):\n",
        "    string = re.sub(r'[,-./]', r'', string)\n",
        "    ngrams = zip(*[string[i:] for i in range(4)])  # N-Gram length is 5\n",
        "    return [''.join(ngram) for ngram in ngrams]\n",
        "\n",
        "# Construct your vectorizer for building the TF-IDF matrix\n",
        "vectorizer = TfidfVectorizer(analyzer=ngrams_analyzer)\n",
        "\n",
        "# Build the matrix!!!\n",
        "tfidf_matrix = vectorizer.fit_transform(vals)\n",
        "\n",
        "pip install sparse_dot_topn\n",
        "\n",
        "# Import IGN's awesome_cossim_topn module\n",
        "from sparse_dot_topn import awesome_cossim_topn\n",
        "\n",
        "# The arguments for awesome_cossim_topn are as follows:\n",
        "### 1. Our TF-IDF matrix\n",
        "### 2. Our TF-IDF matrix transposed (allowing us to build a pairwise cosine matrix)\n",
        "### 3. A top_n filter, which allows us to filter the number of matches returned, which isn't useful for our purposes\n",
        "### 4. This is our similarity threshold. Only values over 0.8 will be returned\n",
        "cosine_matrix = awesome_cossim_topn(\n",
        "  tfidf_matrix,\n",
        "  tfidf_matrix.transpose(),\n",
        "  vals.size,\n",
        "  0.35\n",
        ")\n",
        "\n",
        "\n",
        "# Build a coordinate matrix from a cosine matrix\n",
        "coo_matrix = cosine_matrix.tocoo()\n",
        "\n",
        "# Instaniate our lookup hash table\n",
        "group_lookup = {}\n",
        "\n",
        "\n",
        "def find_group(row, col):\n",
        "    # If either the row or the col string have already been given\n",
        "    # a group, return that group. Otherwise return none\n",
        "    if row in group_lookup:\n",
        "        return group_lookup[row]\n",
        "    elif col in group_lookup:\n",
        "        return group_lookup[col]\n",
        "    else:\n",
        "        return None\n",
        "\n",
        "\n",
        "def add_vals_to_lookup(group, row, col):\n",
        "    # Once we know the group name, set it as the value\n",
        "    # for both strings in the group_lookup\n",
        "    group_lookup[row] = group\n",
        "    group_lookup[col] = group\n",
        "\n",
        "\n",
        "def add_pair_to_lookup(row, col):\n",
        "    # in this function we'll add both the row and the col to the lookup\n",
        "    group = find_group(row, col)  # first, see if one has already been added\n",
        "    if group is not None:\n",
        "        # if we already know the group, make sure both row and col are in lookup\n",
        "        add_vals_to_lookup(group, row, col)\n",
        "    else:\n",
        "        # if we get here, we need to add a new group.\n",
        "        # The name is arbitrary, so just make it the row\n",
        "        add_vals_to_lookup(row, row, col)\n",
        "\n",
        "# for each row and column in coo_matrix\n",
        "# if they're not the same string add them to the group lookup\n",
        "for row, col in zip(coo_matrix.row, coo_matrix.col):\n",
        "    if row != col:\n",
        "        # Note that what is passed to add_pair_to_lookup is the string at each index\n",
        "        # (eg: the names in the legal_name column) not the indices themselves\n",
        "        add_pair_to_lookup(vals[row], vals[col])\n",
        "\n",
        "df['Group'] = df['Product'].map(group_lookup).fillna(df['Product'])\n",
        "\n",
        "df.to_csv('./dol-data-grouped.csv')\n",
        "\n",
        "# You have to use the encoding as latin1 to read this file as there are some special character in this file...\n",
        "df = pd.read_csv ('./dol-data-grouped.csv',encoding='latin1')\n",
        "df.head()\n",
        "\n",
        "sorted_by_product = df.sort_values(['Group'], ascending=False)\n",
        "\n",
        "# b= sorted_by_product['Group'].values #.astype(str)\n",
        "# c= b.astype(str)\n",
        "a = sorted_by_product['Group'].value_counts()\n",
        "sorted_by_product[\"Group\"].head(10)\n",
        "\n",
        "plt.figure(figsize=(10,20))\n",
        "a.plot(kind='barh')\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}