From f197873568e7b176b1bf9fa1f5c4a0ade641a2cb Mon Sep 17 00:00:00 2001 From: Diego Colombo Date: Tue, 10 Oct 2023 18:43:07 +0100 Subject: [PATCH] wip --- ...edding_Wikipedia_articles_for_search.ipynb | 1 - .../csharp/Get_embeddings_from_dataset.ipynb | 4 +- ...emantic_text_search_using_embeddings.ipynb | 254 ++++++++++++++++++ 3 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 Basic_Samples/Embeddings/dotnet/csharp/Semantic_text_search_using_embeddings.ipynb diff --git a/Basic_Samples/Embeddings/dotnet/csharp/Embedding_Wikipedia_articles_for_search.ipynb b/Basic_Samples/Embeddings/dotnet/csharp/Embedding_Wikipedia_articles_for_search.ipynb index 2269d138..6f01fc99 100644 --- a/Basic_Samples/Embeddings/dotnet/csharp/Embedding_Wikipedia_articles_for_search.ipynb +++ b/Basic_Samples/Embeddings/dotnet/csharp/Embedding_Wikipedia_articles_for_search.ipynb @@ -477,7 +477,6 @@ "using System.Text.Json;\n", "using System.IO;\n", "\n", - "\n", "var filePath = \"./wikipedia_embeddins.json\";\n", "\n", "var options = new JsonSerializerOptions\n", diff --git a/Basic_Samples/Embeddings/dotnet/csharp/Get_embeddings_from_dataset.ipynb b/Basic_Samples/Embeddings/dotnet/csharp/Get_embeddings_from_dataset.ipynb index 2d61f40a..d921237b 100644 --- a/Basic_Samples/Embeddings/dotnet/csharp/Get_embeddings_from_dataset.ipynb +++ b/Basic_Samples/Embeddings/dotnet/csharp/Get_embeddings_from_dataset.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": { "dotnet_interactive": { "language": "csharp" @@ -92,7 +92,7 @@ { "data": { "text/html": [ - "
Restore sources
  • https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
Installed Packages
  • Microsoft.DotNet.Interactive.AIUtilities, 1.0.0-beta.23505.6
" + "
Installed Packages
  • Microsoft.DotNet.Interactive.AIUtilities, 1.0.0-beta.23509.3
" ] }, "metadata": {}, diff --git a/Basic_Samples/Embeddings/dotnet/csharp/Semantic_text_search_using_embeddings.ipynb b/Basic_Samples/Embeddings/dotnet/csharp/Semantic_text_search_using_embeddings.ipynb new file mode 100644 index 00000000..4efe2817 --- /dev/null +++ b/Basic_Samples/Embeddings/dotnet/csharp/Semantic_text_search_using_embeddings.ipynb @@ -0,0 +1,254 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Semantic text search using embeddings\n", + "\n", + "We can search through all our reviews semantically in a very efficient manner and at very low cost, by embedding our search query, and then finding the most similar reviews. The dataset is created in the [Get_embeddings_from_dataset Notebook](Get_embeddings_from_dataset.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installation\n", + "Install the Azure Open AI SDK using the below command." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
Installed Packages
  • Azure.AI.OpenAI, 1.0.0-beta.8
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#r \"nuget: Azure.AI.OpenAI, *-*\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
Restore sources
  • https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#i \"nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
Installed Packages
  • Microsoft.DotNet.Interactive.AIUtilities, 1.0.0-beta.23509.3
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#r \"nuget:Microsoft.DotNet.Interactive.AIUtilities, *-*\"\n", + "\n", + "using Microsoft.DotNet.Interactive;\n", + "using Microsoft.DotNet.Interactive.AIUtilities;" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "var azureOpenAIKey = await Kernel.GetPasswordAsync(\"Provide your OPEN_AI_KEY\");\n", + "\n", + "// Your endpoint should look like the following https://YOUR_OPEN_AI_RESOURCE_NAME.openai.azure.com/\n", + "var azureOpenAIEndpoint = await Kernel.GetInputAsync(\"Provide the OPEN_AI_ENDPOINT\");\n", + "\n", + "// Enter the deployment name you chose when you deployed the model.\n", + "var deployment = await Kernel.GetInputAsync(\"Provide deployment name\");" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Import namesapaces and create an instance of `OpenAiClient` using the `azureOpenAIEndpoint` and the `azureOpenAIKey`" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "using Azure;\n", + "using Azure.AI.OpenAI;" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "OpenAIClient client = new (new Uri(azureOpenAIEndpoint), new AzureKeyCredential(azureOpenAIKey.GetClearTextPassword()));" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Loading `Microsoft.Data.Analysis` lastest package" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "#i \"nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-libraries/nuget/v3/index.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "#r \"nuget: Microsoft.Data.Analysis, *-*\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "dotnet_interactive": { + "language": "csharp" + }, + "polyglot_notebook": { + "kernelName": "csharp" + }, + "vscode": { + "languageId": "polyglot-notebook" + } + }, + "outputs": [], + "source": [ + "using Microsoft.ML.Data;" + ] + } + ], + "metadata": { + "language_info": { + "name": "csharp" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +}