New programming model

src/DependencyManagement/DependencyManager.cs

@@ -44,7 +44,7 @@ internal class DependencyManager : IDisposable
         #endregion
 
         public DependencyManager(
-            string requestMetadataDirectory = null,
+            string functionAppRootPath = null,
             IModuleProvider moduleProvider = null,
             IDependencyManagerStorage storage = null,
             IInstalledDependenciesLocator installedDependenciesLocator = null,
@@ -54,7 +54,7 @@ public DependencyManager(
             IBackgroundDependencySnapshotContentLogger currentSnapshotContentLogger = null,
             ILogger logger = null)
         {
-            _storage = storage ?? new DependencyManagerStorage(GetFunctionAppRootPath(requestMetadataDirectory));
+            _storage = storage ?? new DependencyManagerStorage(GetFunctionAppRootPath(functionAppRootPath));
             _installedDependenciesLocator = installedDependenciesLocator ?? new InstalledDependenciesLocator(_storage, logger);
             var snapshotContentLogger = new PowerShellModuleSnapshotLogger();
             _installer = installer ?? new DependencySnapshotInstaller(
@@ -252,14 +252,14 @@ private bool AreAcceptableDependenciesAlreadyInstalled()
             return _storage.SnapshotExists(_currentSnapshotPath);
         }
 
-        private static string GetFunctionAppRootPath(string requestMetadataDirectory)
+        private static string GetFunctionAppRootPath(string functionAppRootPath)
         {
-            if (string.IsNullOrWhiteSpace(requestMetadataDirectory))
+            if (string.IsNullOrWhiteSpace(functionAppRootPath))
             {
-                throw new ArgumentException("Empty request metadata directory path", nameof(requestMetadataDirectory));
+                throw new ArgumentException("Empty function app root path", nameof(functionAppRootPath));
             }
 
-            return Path.GetFullPath(Path.Join(requestMetadataDirectory, ".."));
+            return functionAppRootPath;
         }
 
         #endregion

src/RequestProcessor.cs

@@ -18,7 +18,10 @@
 
 namespace Microsoft.Azure.Functions.PowerShellWorker
 {
+    using Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing;
+    using Microsoft.PowerShell;
     using System.Diagnostics;
+    using System.Text.Json;
     using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level;
 
     internal class RequestProcessor
@@ -66,6 +69,8 @@ internal RequestProcessor(MessagingStream msgStream, System.Management.Automatio
             // If an invocation is cancelled, host will receive an invocation response with status cancelled.
             _requestHandlers.Add(StreamingMessage.ContentOneofCase.InvocationCancel, ProcessInvocationCancelRequest);
 
+            _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionsMetadataRequest, ProcessFunctionMetadataRequest);
+
             _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadRequest, ProcessFunctionEnvironmentReloadRequest);
         }
 
@@ -95,6 +100,9 @@ internal async Task ProcessRequestLoop()
 
         internal StreamingMessage ProcessWorkerInitRequest(StreamingMessage request)
         {
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
             var workerInitRequest = request.WorkerInitRequest;
             Environment.SetEnvironmentVariable("AZUREPS_HOST_ENVIRONMENT", $"AzureFunctions/{workerInitRequest.HostVersion}");
             Environment.SetEnvironmentVariable("POWERSHELL_DISTRIBUTION_CHANNEL", $"Azure-Functions:{workerInitRequest.HostVersion}");
@@ -117,6 +125,32 @@ internal StreamingMessage ProcessWorkerInitRequest(StreamingMessage request)
                 RemoteSessionNamedPipeServer.CreateCustomNamedPipeServer(pipeName);
             }
 
+            // Previously, this half of the dependency management would happen just prior to the dependency download in the 
+            // first function load request. Now that we have the FunctionAppDirectory in the WorkerInitRequest, 
+            // we can do the setup of these variables in the function load request. We need these variables initialized 
+            // for the FunctionMetadataRequest, should it be sent. 
+            try
+            {
+                var rpcLogger = new RpcLogger(_msgStream);
+                rpcLogger.SetContext(request.RequestId, null);
+
+                _dependencyManager = new DependencyManager(request.WorkerInitRequest.FunctionAppDirectory, logger: rpcLogger);
+
+                _powershellPool.Initialize(_firstPwshInstance);
+
+                rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.FirstFunctionLoadCompleted, stopwatch.ElapsedMilliseconds));
+            }
+            catch (Exception e)
+            {
+                // This is a terminating failure: we will need to return a failure response to
+                // all subsequent 'FunctionLoadRequest'. Cache the exception so we can reuse it in future calls.
+                _initTerminatingError = e;
+
+                status.Status = StatusResult.Types.Status.Failure;
+                status.Exception = e.ToRpcException();
+                return response;
+            }
+
             return response;
         }
 
@@ -189,26 +223,20 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request)
             {
                 try
                 {
-                    _isFunctionAppInitialized = true;
-
                     var rpcLogger = new RpcLogger(_msgStream);
                     rpcLogger.SetContext(request.RequestId, null);
 
-                    _dependencyManager = new DependencyManager(request.FunctionLoadRequest.Metadata.Directory, logger: rpcLogger);
-                    var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger);
-
-                    SetupAppRootPathAndModulePath(functionLoadRequest, managedDependenciesPath);
+                    _isFunctionAppInitialized = true;
 
-                    _powershellPool.Initialize(_firstPwshInstance);
+                    var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger);
 
+                    SetupAppRootPathAndModulePath(request.FunctionLoadRequest, managedDependenciesPath);
                     // Start the download asynchronously if needed.
                     _dependencyManager.StartDependencyInstallationIfNeeded(request, _firstPwshInstance, rpcLogger);
-
-                    rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.FirstFunctionLoadCompleted, stopwatch.ElapsedMilliseconds));
                 }
                 catch (Exception e)
                 {
-                    // Failure that happens during this step is terminating and we will need to return a failure response to
+                    // This is a terminating failure: we will need to return a failure response to
                     // all subsequent 'FunctionLoadRequest'. Cache the exception so we can reuse it in future calls.
                     _initTerminatingError = e;
 
@@ -341,6 +369,18 @@ internal StreamingMessage ProcessInvocationCancelRequest(StreamingMessage reques
             return null;
         }
 
+        private StreamingMessage ProcessFunctionMetadataRequest(StreamingMessage request)
+        {
+            StreamingMessage response = NewStreamingMessageTemplate(
+                request.RequestId,
+                StreamingMessage.ContentOneofCase.FunctionMetadataResponse,
+                out StatusResult status);
+
+            response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory));
+
+            return response;
+        }
+
         internal StreamingMessage ProcessFunctionEnvironmentReloadRequest(StreamingMessage request)
         {
             var stopwatch = new Stopwatch();
@@ -394,6 +434,9 @@ private StreamingMessage NewStreamingMessageTemplate(string requestId, Streaming
                 case StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadResponse:
                     response.FunctionEnvironmentReloadResponse = new FunctionEnvironmentReloadResponse() { Result = status };
                     break;
+                case StreamingMessage.ContentOneofCase.FunctionMetadataResponse:
+                    response.FunctionMetadataResponse = new FunctionMetadataResponse() { Result = status };
+                    break;
                 default:
                     throw new InvalidOperationException("Unreachable code.");
             }

src/WorkerIndexing/BindingInformation.cs

@@ -0,0 +1,59 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+
+namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing
+{
+    internal class BindingInformation
+    {
+        private const string BindingNameKey = "name";
+        private const string BindingDirectionKey = "direction";
+        private const string BindingTypeKey = "type";
+        public enum Directions
+        {
+            Unknown = -1,
+            In = 0,
+            Out = 1, 
+            Inout = 2
+        }
+
+        public Directions Direction { get; set; } = Directions.Unknown;
+        public string Type { get; set; } = "";
+        public string Name { get; set; } = "";
+        public Dictionary<string, Object> otherInformation { get; set; } = new Dictionary<string, Object>();
+
+        internal string ConvertToRpcRawBinding(out BindingInfo bindingInfo)
+        {
+            string rawBinding = string.Empty;
+            JObject rawBindingObject = new JObject();
+            rawBindingObject.Add(BindingNameKey, Name);
+            BindingInfo outInfo = new BindingInfo();
+
+
+            if (Direction == Directions.Unknown)
+            {
+                throw new Exception(string.Format(PowerShellWorkerStrings.InvalidBindingInfoDirection, Name));
+            }
+            outInfo.Direction = (BindingInfo.Types.Direction)Direction;
+            rawBindingObject.Add(BindingDirectionKey, Enum.GetName(typeof(BindingInfo.Types.Direction), outInfo.Direction).ToLower());
+            outInfo.Type = Type;
+            rawBindingObject.Add(BindingTypeKey, Type);
+
+            foreach (KeyValuePair<string, Object> pair in otherInformation)
+            {
+                rawBindingObject.Add(pair.Key, JToken.FromObject(pair.Value));
+            }
+
+            rawBinding = JsonConvert.SerializeObject(rawBindingObject);
+            bindingInfo = outInfo;
+            return rawBinding;
+        }
+    }
+}

src/WorkerIndexing/FunctionInformation.cs

@@ -0,0 +1,40 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
+using System.Collections.Generic;
+
+namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing
+{
+    internal class FunctionInformation
+    {
+        private const string FunctionLanguagePowerShell = "powershell";
+
+        public string Directory { get; set; } = "";
+        public string ScriptFile { get; set; } = "";
+        public string Name { get; set; } = "";
+        public string EntryPoint { get; set; } = "";
+        public string FunctionId { get; set; } = "";
+        public List<BindingInformation> Bindings { get; set; } = new List<BindingInformation>();
+
+        internal RpcFunctionMetadata ConvertToRpc()
+        {
+            RpcFunctionMetadata returnMetadata = new RpcFunctionMetadata();
+            returnMetadata.FunctionId = FunctionId;
+            returnMetadata.Directory = Directory;
+            returnMetadata.EntryPoint = EntryPoint;
+            returnMetadata.Name = Name;
+            returnMetadata.ScriptFile = ScriptFile;
+            returnMetadata.Language = FunctionLanguagePowerShell;
+            foreach(BindingInformation binding in Bindings)
+            {
+                string rawBinding = binding.ConvertToRpcRawBinding(out BindingInfo bindingInfo);
+                returnMetadata.Bindings.Add(binding.Name, bindingInfo);
+                returnMetadata.RawBindings.Add(rawBinding);
+            }
+            return returnMetadata;
+        }
+    }
+}

src/WorkerIndexing/WorkerIndexingHelper.cs

@@ -0,0 +1,73 @@
+//
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.Management.Automation;
+using System.Management.Automation.Runspaces;
+
+namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing
+{
+    internal class WorkerIndexingHelper
+    {
+        // TODO: Follow up with the PowerShell on why we get a CommandNotFoundException when using the module qualified cmdlet name.
+        //const string GetFunctionsMetadataCmdletName = "AzureFunctions.PowerShell.SDK\\Get-FunctionsMetadata";
+        const string GetFunctionsMetadataCmdletName = "Get-FunctionsMetadata";
+        internal static IEnumerable<RpcFunctionMetadata> IndexFunctions(string baseDir)
+        {
+            List<RpcFunctionMetadata> indexedFunctions = new List<RpcFunctionMetadata>();
+
+            // This is not the correct way to deal with getting a runspace for the cmdlet. 
+
+            // Firstly, creating a runspace is expensive. If we are going to generate a runspace, it should be done on 
+            // the function load request so that it can be created while the host is processing. 
+
+            // Secondly, this assumes that the AzureFunctions.PowerShell.SDK module is present on the machine/VM's 
+            // PSModulePath. On an Azure instance, it will not be. What we need to do here is move the call 
+            // to SetupAppRootPathAndModulePath in RequestProcessor to the init request, and then use the 
+            // _firstPwshInstance to invoke the Get-FunctionsMetadata command. The only issue with this is that
+            // SetupAppRootPathAndModulePath needs the initial function init request in order to know if managed
+            // dependencies are enabled in this function app.
+
+            // Proposed solutions: 
+            // 1. Pass ManagedDependencyEnabled flag in the worker init request
+            // 2. Change the flow, so that _firstPwshInstance is initialized in worker init with the PSModulePath 
+            //    assuming that managed dependencies are enabled, and then revert the PSModulePath in the first function
+            //    init request should the managed dependencies not be enabled. 
+            // 3. Continue using a new runspace for invoking Get-FunctionsMetadata, but initialize it in worker init and
+            //    point the PsModulePath to the module path bundled with the worker. 
+
+
+            InitialSessionState initial = InitialSessionState.CreateDefault();
+            Runspace runspace = RunspaceFactory.CreateRunspace(initial);
+            runspace.Open();
+            System.Management.Automation.PowerShell _powershell = System.Management.Automation.PowerShell.Create();
+            _powershell.Runspace = runspace;
+
+            _powershell.AddCommand(GetFunctionsMetadataCmdletName).AddArgument(baseDir);
+            string outputString = string.Empty;
+            foreach (PSObject rawMetadata in _powershell.Invoke())
+            {
+                if (outputString != string.Empty)
+                {
+                    throw new Exception(PowerShellWorkerStrings.GetFunctionsMetadataMultipleResultsError);
+                }
+                outputString = rawMetadata.ToString();
+            }
+            _powershell.Commands.Clear();
+
+            List<FunctionInformation> functionInformations = JsonConvert.DeserializeObject<List<FunctionInformation>>(outputString);
+
+            foreach(FunctionInformation fi in functionInformations)
+            {
+                indexedFunctions.Add(fi.ConvertToRpc());
+            }
+
+            return indexedFunctions;
+        }
+    }
+}

src/resources/PowerShellWorkerStrings.resx

@@ -352,4 +352,10 @@
   <data name="DependencySnapshotDoesNotContainAcceptableModuleVersions" xml:space="preserve">
     <value>Dependency snapshot '{0}' does not contain acceptable module versions.</value>
   </data>
+  <data name="GetFunctionsMetadataMultipleResultsError" xml:space="preserve">
+    <value>Multiple results from metadata cmdlet.</value>
+  </data>
+  <data name="InvalidBindingInfoDirection" xml:space="preserve">
+    <value>Invalid binding direction. Binding name: {0}</value>
+  </data>
 </root>

src/worker.config.json

@@ -1,11 +1,12 @@
 {
-    "description":{
-        "language":"powershell",
-        "extensions":[".ps1", ".psm1"],
-        "defaultExecutablePath":"dotnet",
-        "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll",
-        "supportedRuntimeVersions":["7", "7.2"],
+    "description": {
+        "language": "powershell",
+        "extensions": [ ".ps1", ".psm1" ],
+        "defaultExecutablePath": "dotnet",
+        "defaultWorkerPath": "%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll",
+        "supportedRuntimeVersions": [ "7", "7.2" ],
         "defaultRuntimeVersion": "7",
-        "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*"
+        "sanitizeRuntimeVersionRegex": "\\d+\\.?\\d*",
+        "workerIndexing": "true"
     }
 }