Take latest Master to Development

CHANGELOG.MD

@@ -1,3 +1,30 @@
+## Released v1.0.2 (June.24, 2015)
+###Features:
+- Perf improvements in the Engine to execute rules concurrently.
+
+
+###Rules:
+- New rule to validate the presence of deprecated module manifest fields.
+- Removed PSAvoidTrapStatement rule from the builtin set – since this is not deprecated and using trap is a better choice in certain scenarios.
+
+
+###Fixes:
+- Verbose Message rule applies to only DSC cmdlet based resources.
+- Multiple fixes to AvoidUninitializedVariable to work with non-mandatory parameters, fix in the flow graphs for throw statements;  support for missing preference variables; support for automatic variables.
+- PSAvoidUsingInternalsURLs to work with xPath expressions.
+- UseSingularNouns rule to raise warnings for plural phrases.
+- Added .gitignore to exclude certain files from being reported as untracked.
+- Revisited severity for DSC rules.
+- PSUseOutputTypeCorrectly rule not to get triggered for functions returning system.void type.
+- PSAvoidDefaultTrueValueSwitchParameter to work with switch attribute when supplied as fully qualified.
+- Ignore PSObject and PSCustomObject for UseOutputTypeCorrectly rule.
+- Only raise NullComparisonRule if the RHS is an array or has unknown type.
+- PSUseDeclaredVarsMoreThanAssignments rule to be raised for script variables and for setting the property of a variable.
+- Support for using PSUseCmdletCorrectly rule when mandatory parameters are supplied in a splatted hashtable.
+- AvoidUsingPlainTextForPassword rule to be raised only strings or object types.
+- Fix for PositionalParameterUsed method (Helper.cs) uses unsafe method to exclude ForEach-Object and Where-Object.
+
+
 ## Released v1.0.1 (May.8, 2015)
 ###Features:
 - Integrated with waffle.io for Project Management.
@@ -18,7 +45,6 @@
 
 
 ##Released v1.0.0 on Apr.24, 2015
-
 ###Features:
 - Finalized three levels of Severity - Error/Warning/Information. 
 - Improved PSScriptAnalyzer engine behavior: emits non-terminating errors (Ex: for failed ast parse) and continues rule application when running on multiple scripts.

Engine/Commands/InvokeScriptAnalyzerCommand.cs

@@ -12,6 +12,7 @@
 
 using System.Text.RegularExpressions;
 using System;
+using System.ComponentModel;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Globalization;
@@ -20,6 +21,9 @@
 using System.Management.Automation.Language;
 using System.IO;
 using Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic;
+using System.Threading.Tasks;
+using System.Collections.Concurrent;
+using System.Threading;
 
 namespace Microsoft.Windows.PowerShell.ScriptAnalyzer.Commands
 {
@@ -251,12 +255,12 @@ private void ProcessPath(string path)
                 }
             }
             else if (File.Exists(path))
-            {
-                WriteVerbose(string.Format(CultureInfo.CurrentCulture, Strings.VerboseFileMessage, path));
+            {                
                 if ((path.Length > ps1Suffix.Length && path.Substring(path.Length - ps1Suffix.Length).Equals(ps1Suffix, StringComparison.OrdinalIgnoreCase)) ||
                     (path.Length > psm1Suffix.Length && path.Substring(path.Length - psm1Suffix.Length).Equals(psm1Suffix, StringComparison.OrdinalIgnoreCase)) ||
                     (path.Length > psd1Suffix.Length && path.Substring(path.Length - psd1Suffix.Length).Equals(psd1Suffix, StringComparison.OrdinalIgnoreCase)))
                 {
+                    WriteVerbose(string.Format(CultureInfo.CurrentCulture, Strings.VerboseFileMessage, path));
                     AnalyzeFile(path);
                 }
             }
@@ -267,6 +271,13 @@ private void ProcessPath(string path)
 
         }
 
+        ConcurrentBag<DiagnosticRecord> diagnostics;
+        ConcurrentBag<SuppressedRecord> suppressed;
+        Dictionary<string, List<RuleSuppression>> ruleSuppressions;
+        List<Regex> includeRegexList;
+        List<Regex> excludeRegexList;
+        ConcurrentDictionary<string, List<object>> ruleDictionary;
+
         /// <summary>
         /// Analyzes a single script file.
         /// </summary>
@@ -275,15 +286,16 @@ private void AnalyzeFile(string filePath)
         {
             Token[] tokens = null;
             ParseError[] errors = null;
-            List<DiagnosticRecord> diagnostics = new List<DiagnosticRecord>();
-            List<SuppressedRecord> suppressed = new List<SuppressedRecord>();
+            ConcurrentBag<DiagnosticRecord> diagnostics = new ConcurrentBag<DiagnosticRecord>();
+            ConcurrentBag<SuppressedRecord> suppressed = new ConcurrentBag<SuppressedRecord>();
+            BlockingCollection<List<object>> verboseOrErrors = new BlockingCollection<List<object>>();
 
             // Use a List of KVP rather than dictionary, since for a script containing inline functions with same signature, keys clash
             List<KeyValuePair<CommandInfo, IScriptExtent>> cmdInfoTable = new List<KeyValuePair<CommandInfo, IScriptExtent>>();
 
             //Check wild card input for the Include/ExcludeRules and create regex match patterns
-            List<Regex> includeRegexList = new List<Regex>();
-            List<Regex> excludeRegexList = new List<Regex>();
+            includeRegexList = new List<Regex>();
+            excludeRegexList = new List<Regex>();
             if (includeRule != null)
             {
                 foreach (string rule in includeRule)
@@ -331,7 +343,7 @@ private void AnalyzeFile(string filePath)
                 return;
             }
 
-            Dictionary<string, List<RuleSuppression>> ruleSuppressions = Helper.Instance.GetRuleSuppression(ast);
+            ruleSuppressions = Helper.Instance.GetRuleSuppression(ast);
 
             foreach (List<RuleSuppression> ruleSuppressionsList in ruleSuppressions.Values)
             {
@@ -360,43 +372,75 @@ private void AnalyzeFile(string filePath)
 
             if (ScriptAnalyzer.Instance.ScriptRules != null)
             {
-                foreach (IScriptRule scriptRule in ScriptAnalyzer.Instance.ScriptRules)
-                {
-                    bool includeRegexMatch = false;
-                    bool excludeRegexMatch = false;
-                    foreach (Regex include in includeRegexList)
+                var tasks = ScriptAnalyzer.Instance.ScriptRules.Select(scriptRule => Task.Factory.StartNew(() =>
                     {
-                        if (include.IsMatch(scriptRule.GetName()))
+                        bool includeRegexMatch = false;
+                        bool excludeRegexMatch = false;
+
+                        foreach (Regex include in includeRegexList)
                         {
-                            includeRegexMatch = true;
-                            break;
+                            if (include.IsMatch(scriptRule.GetName()))
+                            {
+                                includeRegexMatch = true;
+                                break;
+                            }
                         }
-                    }
 
-                    foreach (Regex exclude in excludeRegexList)
-                    {
-                        if (exclude.IsMatch(scriptRule.GetName()))
+                        foreach (Regex exclude in excludeRegexList)
                         {
-                            excludeRegexMatch = true;
-                            break;
+                            if (exclude.IsMatch(scriptRule.GetName()))
+                            {
+                                excludeRegexMatch = true;
+                                break;
+                            }
                         }
-                    }
-
-                    if ((includeRule == null || includeRegexMatch) && (excludeRule == null || !excludeRegexMatch))
-                    {
-                        WriteVerbose(string.Format(CultureInfo.CurrentCulture, Strings.VerboseRunningMessage, scriptRule.GetName()));
 
-                        // Ensure that any unhandled errors from Rules are converted to non-terminating errors
-                        // We want the Engine to continue functioning even if one or more Rules throws an exception
-                        try
+                        if ((includeRule == null || includeRegexMatch) && (excludeRule == null || !excludeRegexMatch))
                         {
-                            var records = Helper.Instance.SuppressRule(scriptRule.GetName(), ruleSuppressions, scriptRule.AnalyzeScript(ast, filePath).ToList());
-                            diagnostics.AddRange(records.Item2);
-                            suppressed.AddRange(records.Item1);
+                            List<object> result = new List<object>();
+
+                            result.Add(string.Format(CultureInfo.CurrentCulture, Strings.VerboseRunningMessage, scriptRule.GetName()));
+
+                            // Ensure that any unhandled errors from Rules are converted to non-terminating errors
+                            // We want the Engine to continue functioning even if one or more Rules throws an exception
+                            try
+                            {
+                                var records = Helper.Instance.SuppressRule(scriptRule.GetName(), ruleSuppressions, scriptRule.AnalyzeScript(ast, ast.Extent.File).ToList());
+                                foreach (var record in records.Item2)
+                                {
+                                    diagnostics.Add(record);
+                                }
+                                foreach (var suppressedRec in records.Item1)
+                                {
+                                    suppressed.Add(suppressedRec);
+                                }
+                            }
+                            catch (Exception scriptRuleException)
+                            {
+                                result.Add(new ErrorRecord(scriptRuleException, Strings.RuleErrorMessage, ErrorCategory.InvalidOperation, ast.Extent.File));
+                            }
+
+                            verboseOrErrors.Add(result);
                         }
-                        catch (Exception scriptRuleException)
+                }));
+
+                Task.Factory.ContinueWhenAll(tasks.ToArray(), t => verboseOrErrors.CompleteAdding());
+
+                while (!verboseOrErrors.IsCompleted)
+                {
+                    List<object> data = null;
+                    try
+                    {
+                        data = verboseOrErrors.Take();
+                    }
+                    catch (InvalidOperationException) { }
+
+                    if (data != null)
+                    {
+                        WriteVerbose(data[0] as string);
+                        if (data.Count == 2)
                         {
-                            WriteError(new ErrorRecord(scriptRuleException, Strings.RuleErrorMessage, ErrorCategory.InvalidOperation, filePath));
+                            WriteError(data[1] as ErrorRecord);
                         }
                     }
                 }
@@ -437,8 +481,14 @@ private void AnalyzeFile(string filePath)
                         try
                         {
                             var records = Helper.Instance.SuppressRule(tokenRule.GetName(), ruleSuppressions, tokenRule.AnalyzeTokens(tokens, filePath).ToList());
-                            diagnostics.AddRange(records.Item2);
-                            suppressed.AddRange(records.Item1);
+                            foreach (var record in records.Item2)
+                            {
+                                diagnostics.Add(record);
+                            }
+                            foreach (var suppressedRec in records.Item1)
+                            {
+                                suppressed.Add(suppressedRec);
+                            }
                         }
                         catch (Exception tokenRuleException)
                         {
@@ -489,8 +539,14 @@ private void AnalyzeFile(string filePath)
                             try
                             {
                                 var records = Helper.Instance.SuppressRule(dscResourceRule.GetName(), ruleSuppressions, dscResourceRule.AnalyzeDSCClass(ast, filePath).ToList());
-                                diagnostics.AddRange(records.Item2);
-                                suppressed.AddRange(records.Item1);
+                                foreach (var record in records.Item2)
+                                {
+                                    diagnostics.Add(record);
+                                }
+                                foreach (var suppressedRec in records.Item1)
+                                {
+                                    suppressed.Add(suppressedRec);
+                                }
                             }
                             catch (Exception dscResourceRuleException)
                             {
@@ -532,8 +588,14 @@ private void AnalyzeFile(string filePath)
                             try
                             {
                                 var records = Helper.Instance.SuppressRule(dscResourceRule.GetName(), ruleSuppressions, dscResourceRule.AnalyzeDSCResource(ast, filePath).ToList());
-                                diagnostics.AddRange(records.Item2);
-                                suppressed.AddRange(records.Item1);
+                                foreach (var record in records.Item2)
+                                {
+                                    diagnostics.Add(record);
+                                }
+                                foreach (var suppressedRec in records.Item1)
+                                {
+                                    suppressed.Add(suppressedRec);
+                                }
                             }
                             catch (Exception dscResourceRuleException)
                             {
@@ -573,15 +635,20 @@ private void AnalyzeFile(string filePath)
                     }
                 }
 
-                diagnostics.AddRange(ScriptAnalyzer.Instance.GetExternalRecord(ast, tokens, exRules.ToArray(), this, fileName));
+                foreach (var record in ScriptAnalyzer.Instance.GetExternalRecord(ast, tokens, exRules.ToArray(), this, fileName))
+                {
+                    diagnostics.Add(record);
+                }
             }
 
             #endregion
 
+            IEnumerable<DiagnosticRecord> diagnosticsList = diagnostics;
+
             if (severity != null)
             {
                 var diagSeverity = severity.Select(item => Enum.Parse(typeof(DiagnosticSeverity), item, true));
-                diagnostics = diagnostics.Where(item => diagSeverity.Contains(item.Severity)).ToList();
+                diagnosticsList = diagnostics.Where(item => diagSeverity.Contains(item.Severity));
             }
 
             //Output through loggers
@@ -596,7 +663,7 @@ private void AnalyzeFile(string filePath)
                 }
                 else
                 {
-                    foreach (DiagnosticRecord diagnostic in diagnostics)
+                    foreach (DiagnosticRecord diagnostic in diagnosticsList)
                     {
                         logger.LogObject(diagnostic, this);
                     }

Engine/Helper.cs

@@ -219,6 +219,21 @@ item is TypeDefinitionAst
             return false;
         }
 
+        /// <summary>
+        /// Given a commandast, checks whether it uses splatted variable
+        /// </summary>
+        /// <param name="cmdAst"></param>
+        /// <returns></returns>
+        public bool HasSplattedVariable(CommandAst cmdAst)
+        {
+            if (cmdAst == null || cmdAst.CommandElements == null)
+            {
+                return false;
+            }
+
+            return cmdAst.CommandElements.Any(cmdElem => cmdElem is VariableExpressionAst && (cmdElem as VariableExpressionAst).Splatted);
+        }
+
         /// <summary>
         /// Given a commandast, checks whether positional parameters are used or not.
         /// </summary>
@@ -234,23 +249,17 @@ public bool PositionalParameterUsed(CommandAst cmdAst)
             CommandInfo commandInfo = GetCommandInfo(GetCmdletNameFromAlias(cmdAst.GetCommandName())) ?? GetCommandInfo(cmdAst.GetCommandName());
 
             IEnumerable<ParameterMetadata> switchParams = null;
-            IEnumerable<CommandParameterSetInfo> scriptBlocks = null;
-            bool hasScriptBlockSet = false;
+
+            if (HasSplattedVariable(cmdAst))
+            {
+                return false;
+            }
 
             if (commandInfo != null && commandInfo.CommandType == System.Management.Automation.CommandTypes.Cmdlet)
             {
                 try
                 {
                     switchParams = commandInfo.Parameters.Values.Where<ParameterMetadata>(pm => pm.SwitchParameter);
-                    scriptBlocks = commandInfo.ParameterSets;
-                    foreach (CommandParameterSetInfo cmdParaset in scriptBlocks)
-                    {
-                        if (String.Equals(cmdParaset.Name, "ScriptBlockSet", StringComparison.OrdinalIgnoreCase))
-                        {
-                            hasScriptBlockSet = true;
-                        }
-                    }
-
                 }
                 catch (Exception)
                 {
@@ -264,8 +273,6 @@ public bool PositionalParameterUsed(CommandAst cmdAst)
 
             foreach (CommandElementAst ceAst in cmdAst.CommandElements)
             {
-                if (!hasScriptBlockSet)
-                {
                     if (ceAst is CommandParameterAst)
                     {
                         // Skip if it's a switch parameter
@@ -286,17 +293,17 @@ public bool PositionalParameterUsed(CommandAst cmdAst)
                     }
                     else
                     {
-                        //Skip if splatting "@" is used
-                        if (ceAst is VariableExpressionAst)
-                        {
-                            if ((ceAst as VariableExpressionAst).Splatted)
-                            {
-                                continue;
-                            }
-                        }
                         arguments += 1;
                     }
-                }
+
+            }
+
+            // if not the first element in a pipeline, increase the number of arguments by 1
+            PipelineAst parent = cmdAst.Parent as PipelineAst;
+
+            if (parent != null && parent.PipelineElements.Count > 1 && parent.PipelineElements[0] != cmdAst)
+            {
+                arguments += 1;
             }
 
             return arguments > parameters;
@@ -672,7 +679,7 @@ internal string GetTypeFromMemberExpressionAstHelper(MemberExpressionAst memberA
         /// <param name="varAst"></param>
         /// <param name="ast"></param>
         /// <returns></returns>
-        internal Type GetTypeFromAnalysis(VariableExpressionAst varAst, Ast ast)
+        public Type GetTypeFromAnalysis(VariableExpressionAst varAst, Ast ast)
         {
             try
             {