Managed dependency download on background thread (Azure#193)

SatishRanjan · daxian-dbw · commit 52e5c4b6a152 · 2019-04-24T16:51:36.000-07:00
diff --git a/src/DependencyManagement/DependencyManager.cs b/src/DependencyManagement/DependencyManager.cs
@@ -14,11 +14,14 @@
 using Microsoft.Azure.Functions.PowerShellWorker.PowerShell;
 using Microsoft.Azure.Functions.PowerShellWorker.Utility;
 using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level;
+using Microsoft.Azure.Functions.PowerShellWorker.Messaging;
 
 namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement
 {
     using System.Management.Automation;
     using System.Management.Automation.Language;
+    using System.Management.Automation.Runspaces;
+    using System.Threading.Tasks;
 
     internal class DependencyManager
     {
@@ -28,6 +31,11 @@ internal class DependencyManager
         // This is the location where the dependent modules will be installed.
         internal static string DependenciesPath { get; private set; }
 
+        internal Exception DependencyError => _dependencyError;
+
+        //The dependency download task
+        internal Task DependencyDownloadTask => _dependencyDownloadTask;
+
         // Az module name.
         private const string AzModuleName = "Az";
 
@@ -51,6 +59,12 @@ internal class DependencyManager
         // Managed Dependencies folder name.
         private const string ManagedDependenciesFolderName = "ManagedDependencies";
 
+        //Set when any error occurs while downloading dependencies
+        private Exception _dependencyError;
+
+        //Dependency download task
+        private Task _dependencyDownloadTask;
+
         // This flag is used to figure out if we need to install/reinstall all the function app dependencies.
         // If we do, we use it to clean up the module destination path.
         private bool _shouldUpdateFunctionAppDependencies;
@@ -60,6 +74,74 @@ internal DependencyManager()
             Dependencies = new List<DependencyInfo>();
         }
 
+        /// <summary>
+        /// Processes the dependency download request
+        /// </summary>
+        /// <param name="msgStream">The protobuf messaging stream</param>
+        /// <param name="request">The StreamingMessage request for function load</param>
+        internal void ProcessDependencyDownload(MessagingStream msgStream, StreamingMessage request)
+        {
+            if (request.FunctionLoadRequest.ManagedDependencyEnabled)
+            {
+                var rpcLogger = new RpcLogger(msgStream);
+                rpcLogger.SetContext(request.RequestId, null);
+                if (Dependencies.Count == 0)
+                {
+                    // If there are no dependencies to install, log and return.
+                    rpcLogger.Log(LogLevel.Trace, PowerShellWorkerStrings.FunctionAppDoesNotHaveDependentModulesToInstall, isUserLog: true);
+                    return;
+                }
+
+                if (!_shouldUpdateFunctionAppDependencies)
+                {
+                    // The function app already has the latest dependencies installed.
+                    rpcLogger.Log(LogLevel.Trace, PowerShellWorkerStrings.LatestFunctionAppDependenciesAlreadyInstalled, isUserLog: true);
+                    return;
+                }
+
+                //Start dependency download on a separate thread
+                _dependencyDownloadTask = Task.Run(() => ProcessDependencies(rpcLogger));
+            }
+        }
+
+        /// <summary>
+        /// Waits for the dependency download task to finish 
+        /// and sets it's reference to null to be picked for cleanup by next run of GC
+        /// </summary>
+        internal void WaitOnDependencyDownload()
+        {
+            if (_dependencyDownloadTask != null)
+            {
+                _dependencyDownloadTask.Wait();
+                _dependencyDownloadTask = null;
+            }
+        }
+
+        private void ProcessDependencies(RpcLogger rpcLogger)
+        {
+            try
+            {
+                _dependencyError = null;
+                var initialSessionState = InitialSessionState.CreateDefault();
+                initialSessionState.ThreadOptions = PSThreadOptions.UseCurrentThread;
+                initialSessionState.EnvironmentVariables.Add(new SessionStateVariableEntry("PSModulePath", FunctionLoader.FunctionModulePath, null));
+                // Setting the execution policy on macOS and Linux throws an exception so only update it on Windows
+                if (Platform.IsWindows)
+                {
+                    initialSessionState.ExecutionPolicy = Microsoft.PowerShell.ExecutionPolicy.Unrestricted;
+                }
+
+                using (PowerShell powerShellInstance = PowerShell.Create(initialSessionState))
+                {
+                    InstallFunctionAppDependencies(powerShellInstance, rpcLogger);
+                }
+            }
+            catch (Exception e)
+            {
+                _dependencyError = e;
+            }
+        }
+
         /// <summary>
         /// Initializes the dependency manger and performs the following:
         /// - Parse functionAppRoot\requirements.psd1 file and create a list of dependencies to install.
@@ -83,8 +165,8 @@ internal void Initialize(FunctionLoadRequest request)
                 foreach (DictionaryEntry entry in entries)
                 {
                     // A valid entry is of the form: 'ModuleName'='MajorVersion.*"
-                    string name = (string) entry.Key;
-                    string version = (string) entry.Value;
+                    string name = (string)entry.Key;
+                    string version = (string)entry.Value;
 
                     // Validates that the module name is a supported dependency.
                     ValidateModuleName(name);
@@ -124,20 +206,6 @@ internal void InstallFunctionAppDependencies(PowerShell pwsh, ILogger logger)
         {
             try
             {
-                if (Dependencies.Count == 0)
-                {
-                    // If there are no dependencies to install, log and return.
-                    logger.Log(LogLevel.Trace, PowerShellWorkerStrings.FunctionAppDoesNotHaveDependentModulesToInstall, isUserLog: true);
-                    return;
-                }
-
-                if (!_shouldUpdateFunctionAppDependencies)
-                {
-                    // The function app already has the latest dependencies installed.
-                    logger.Log(LogLevel.Trace, PowerShellWorkerStrings.LatestFunctionAppDependenciesAlreadyInstalled, isUserLog: true);
-                    return;
-                }
-
                 // Install the function dependencies.
                 logger.Log(LogLevel.Trace, PowerShellWorkerStrings.InstallingFunctionAppDependentModules, isUserLog: true);
 
diff --git a/src/PowerShell/PowerShellManager.cs b/src/PowerShell/PowerShellManager.cs
@@ -45,7 +45,7 @@ static PowerShellManager()
             addMethod.Invoke(null, new object[] { "HttpRequestContext", typeof(HttpRequestContext) });
         }
 
-        internal PowerShellManager(ILogger logger, Action<PowerShell, ILogger> initAction = null)
+        internal PowerShellManager(ILogger logger)
         {
             if (FunctionLoader.FunctionAppRootPath == null)
             {
@@ -77,9 +77,6 @@ internal PowerShellManager(ILogger logger, Action<PowerShell, ILogger> initActio
             _pwsh.Streams.Verbose.DataAdding += streamHandler.VerboseDataAdding;
             _pwsh.Streams.Warning.DataAdding += streamHandler.WarningDataAdding;
 
-            // Install function app dependent modules
-            initAction?.Invoke(_pwsh, logger);
-
             // Initialize the Runspace
             InvokeProfile(FunctionLoader.FunctionAppProfilePath);
         }
diff --git a/src/PowerShell/PowerShellManagerPool.cs b/src/PowerShell/PowerShellManagerPool.cs
@@ -46,26 +46,6 @@ internal PowerShellManagerPool(MessagingStream msgStream)
             RpcLogger.WriteSystemLog(string.Format(PowerShellWorkerStrings.LogConcurrencyUpperBound, _upperBound.ToString()));
         }
 
-        /// <summary>
-        /// Initialize the pool and populate it with PowerShellManager instances.
-        /// We instantiate PowerShellManager instances in a lazy way, starting from size 1 and increase the number of workers as needed.
-        /// </summary>
-        internal void Initialize(string requestId, Action<PowerShell, ILogger> initAction = null)
-        {
-            var logger = new RpcLogger(_msgStream);
-
-            try
-            {
-                logger.SetContext(requestId, invocationId: null);
-                _pool.Add(new PowerShellManager(logger, initAction));
-                _poolSize = 1;
-            }
-            finally
-            {
-                logger.ResetContext();
-            }
-        }
-
         /// <summary>
         /// Checkout an idle PowerShellManager instance in a non-blocking asynchronous way.
         /// </summary>
diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs
@@ -16,7 +16,7 @@
 using Microsoft.Azure.WebJobs.Script.Grpc.Messages;
 using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level;
 
-namespace  Microsoft.Azure.Functions.PowerShellWorker
+namespace Microsoft.Azure.Functions.PowerShellWorker
 {
     internal class RequestProcessor
     {
@@ -57,7 +57,7 @@ internal RequestProcessor(MessagingStream msgStream)
 
             // Host sends required metadata to worker to load function
             _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionLoadRequest, ProcessFunctionLoadRequest);
-            
+
             // Host requests a given invocation
             _requestHandlers.Add(StreamingMessage.ContentOneofCase.InvocationRequest, ProcessInvocationRequest);
 
@@ -167,7 +167,7 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request)
                 try
                 {
                     _isFunctionAppInitialized = true;
-                    InitializeForFunctionApp(request, response);
+                    InitializeForFunctionApp(request);
                 }
                 catch (Exception e)
                 {
@@ -183,7 +183,6 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request)
 
             try
             {
-                // Load the metadata of the function.
                 _functionLoader.LoadFunction(functionLoadRequest);
             }
             catch (Exception e)
@@ -206,6 +205,28 @@ internal StreamingMessage ProcessInvocationRequest(StreamingMessage request)
 
             try
             {
+                if (_dependencyManager.DependencyDownloadTask != null
+                    && (_dependencyManager.DependencyDownloadTask.Status != TaskStatus.Canceled
+                    || _dependencyManager.DependencyDownloadTask.Status != TaskStatus.Faulted
+                    || _dependencyManager.DependencyDownloadTask.Status != TaskStatus.RanToCompletion))
+                {
+                    var rpcLogger = new RpcLogger(_msgStream);
+                    rpcLogger.SetContext(request.RequestId, request.InvocationRequest?.InvocationId);
+                    rpcLogger.Log(LogLevel.Information, PowerShellWorkerStrings.DependencyDownloadInProgress, isUserLog: true);
+                    _dependencyManager.WaitOnDependencyDownload();
+                }
+
+                if (_dependencyManager.DependencyError != null)
+                {
+                    StreamingMessage response = NewStreamingMessageTemplate(request.RequestId,
+                        StreamingMessage.ContentOneofCase.InvocationResponse,
+                        out StatusResult status);
+                    status.Status = StatusResult.Types.Status.Failure;
+                    status.Exception = _dependencyManager.DependencyError.ToRpcException();
+                    response.InvocationResponse.InvocationId = request.InvocationRequest.InvocationId;
+                    return response;
+                }
+
                 functionInfo = _functionLoader.GetFunctionInfo(request.InvocationRequest.FunctionId);
                 psManager = _powershellPool.CheckoutIdleWorker(request, functionInfo);
 
@@ -296,31 +317,17 @@ internal StreamingMessage ProcessFunctionEnvironmentReloadRequest(StreamingMessa
         /// <summary>
         /// Initialize the worker based on the FunctionApp that the worker will deal with.
         /// </summary>
-        private void InitializeForFunctionApp(StreamingMessage request, StreamingMessage response)
+        private void InitializeForFunctionApp(StreamingMessage request)
         {
             var functionLoadRequest = request.FunctionLoadRequest;
-
-            // If 'ManagedDependencyEnabled' is true, process the function app dependencies as defined in FunctionAppRoot\requirements.psd1.
-            // These dependencies will be installed via 'Save-Module' when the first PowerShellManager instance is being created.
             if (functionLoadRequest.ManagedDependencyEnabled)
             {
                 _dependencyManager.Initialize(functionLoadRequest);
             }
 
             // Setup the FunctionApp root path and module path.
             FunctionLoader.SetupWellKnownPaths(functionLoadRequest);
-
-            // Constructing the first PowerShellManager instance for the Pool.
-            if (DependencyManager.Dependencies.Count > 0)
-            {
-                // Do extra work to install the specified dependencies.
-                _powershellPool.Initialize(request.RequestId, _dependencyManager.InstallFunctionAppDependencies);
-                response.FunctionLoadResponse.IsDependencyDownloaded = true;
-            }
-            else
-            {
-                _powershellPool.Initialize(request.RequestId);
-            }
+            _dependencyManager.ProcessDependencyDownload(_msgStream, request);
         }
 
         /// <summary>
@@ -425,7 +432,7 @@ private void BindOutputFromResult(InvocationResponse response, AzFunctionInfo fu
                         TypedData dataToUse = transformedValue.ToTypedData();
 
                         // if one of the bindings is '$return' we need to set the ReturnValue
-                        if(string.Equals(outBindingName, AzFunctionInfo.DollarReturn, StringComparison.OrdinalIgnoreCase))
+                        if (string.Equals(outBindingName, AzFunctionInfo.DollarReturn, StringComparison.OrdinalIgnoreCase))
                         {
                             response.ReturnValue = dataToUse;
                             continue;
diff --git a/src/resources/PowerShellWorkerStrings.resx b/src/resources/PowerShellWorkerStrings.resx

Original file line number	Diff line number	Diff line change
`@@ -45,7 +45,7 @@ static PowerShellManager()`
`45`	`45`	`addMethod.Invoke(null, new object[] { "HttpRequestContext", typeof(HttpRequestContext) });`
`46`	`46`	`}`
`47`	`47`
`48`		`- internal PowerShellManager(ILogger logger, Action<PowerShell, ILogger> initAction = null)`
	`48`	`+ internal PowerShellManager(ILogger logger)`
`49`	`49`	`{`
`50`	`50`	`if (FunctionLoader.FunctionAppRootPath == null)`
`51`	`51`	`{`
`@@ -77,9 +77,6 @@ internal PowerShellManager(ILogger logger, Action<PowerShell, ILogger> initActio`
`77`	`77`	`_pwsh.Streams.Verbose.DataAdding += streamHandler.VerboseDataAdding;`
`78`	`78`	`_pwsh.Streams.Warning.DataAdding += streamHandler.WarningDataAdding;`
`79`	`79`
`80`		`- // Install function app dependent modules`
`81`		`- initAction?.Invoke(_pwsh, logger);`
`82`		`-`
`83`	`80`	`// Initialize the Runspace`
`84`	`81`	`InvokeProfile(FunctionLoader.FunctionAppProfilePath);`
`85`	`82`	`}`