From 1c6976f54562ae5cbc537597caf3e808efbf4941 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Fri, 8 Mar 2019 12:06:38 +0800 Subject: [PATCH 1/4] (GH-879) Add characterisation tests for Workspace.EnumeratePSFiles Previosly there were no tests for the Workspace.EnumeratePSFiles method. This commit adds tests for EnumeratePSFiles method using a set of static fixture test files. Note that the behaviour of EnumeratePSFiles changes depending on the .Net Framework edition --- .../Fixtures/Workspace/nested/donotfind.ps1 | 1 + .../Fixtures/Workspace/nested/donotfind.txt | 1 + .../Workspace/nested/nestedmodule.psd1 | 1 + .../Workspace/nested/nestedmodule.psm1 | 1 + .../Fixtures/Workspace/other/other.cdxml | 1 + .../Fixtures/Workspace/other/other.ps1xml | 1 + .../Fixtures/Workspace/other/other.psrc | 1 + .../Fixtures/Workspace/other/other.pssc | 1 + .../Fixtures/Workspace/rootfile.ps1 | 1 + .../PowerShellEditorServices.Test.csproj | 5 +++ .../Session/WorkspaceTests.cs | 38 +++++++++++++++++++ 11 files changed, 52 insertions(+) create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.ps1 create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.txt create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psd1 create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psm1 create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.cdxml create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.ps1xml create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.psrc create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.pssc create mode 100644 test/PowerShellEditorServices.Test/Fixtures/Workspace/rootfile.ps1 diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.ps1 b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.ps1 new file mode 100644 index 000000000..29b0e6fb9 --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.ps1 @@ -0,0 +1 @@ +# donotfind.ps1 diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.txt b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.txt new file mode 100644 index 000000000..c0070a904 --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/donotfind.txt @@ -0,0 +1 @@ +donotfind.txt diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psd1 b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psd1 new file mode 100644 index 000000000..6657524ae --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psd1 @@ -0,0 +1 @@ +# nestedmodule.psd1 diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psm1 b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psm1 new file mode 100644 index 000000000..437ba730e --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/nested/nestedmodule.psm1 @@ -0,0 +1 @@ +# nestedmodule.psm1 diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.cdxml b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.cdxml new file mode 100644 index 000000000..08cfdb60c --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.cdxml @@ -0,0 +1 @@ + diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.ps1xml b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.ps1xml new file mode 100644 index 000000000..951d51d34 --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.ps1xml @@ -0,0 +1 @@ + diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.psrc b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.psrc new file mode 100644 index 000000000..a53a4ddf8 --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.psrc @@ -0,0 +1 @@ +# other.psrc diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.pssc b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.pssc new file mode 100644 index 000000000..7d49b1093 --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/other/other.pssc @@ -0,0 +1 @@ +# other.pssc diff --git a/test/PowerShellEditorServices.Test/Fixtures/Workspace/rootfile.ps1 b/test/PowerShellEditorServices.Test/Fixtures/Workspace/rootfile.ps1 new file mode 100644 index 000000000..f61acd8a1 --- /dev/null +++ b/test/PowerShellEditorServices.Test/Fixtures/Workspace/rootfile.ps1 @@ -0,0 +1 @@ +# rootfile.ps1 diff --git a/test/PowerShellEditorServices.Test/PowerShellEditorServices.Test.csproj b/test/PowerShellEditorServices.Test/PowerShellEditorServices.Test.csproj index fdcd147fc..c363df61f 100644 --- a/test/PowerShellEditorServices.Test/PowerShellEditorServices.Test.csproj +++ b/test/PowerShellEditorServices.Test/PowerShellEditorServices.Test.csproj @@ -27,6 +27,11 @@ + + + PreserveNewest + + $(DefineConstants);CoreCLR diff --git a/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs b/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs index cfa965b65..ca61e44c4 100644 --- a/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs +++ b/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs @@ -24,6 +24,7 @@ public class WorkspaceTests : string.Empty; [Fact] + [Trait("Category", "Workspace")] public void CanResolveWorkspaceRelativePath() { string workspacePath = TestUtilities.NormalizePath("c:/Test/Workspace/"); @@ -47,6 +48,42 @@ public void CanResolveWorkspaceRelativePath() } [Fact] + [Trait("Category", "Workspace")] + public void CanRecurseDirectoryTree() + { + Workspace workspace = new Workspace(PowerShellVersion, Logging.NullLogger); + workspace.WorkspacePath = TestUtilities.NormalizePath("Fixtures/Workspace"); + + IEnumerable result = workspace.EnumeratePSFiles(); + List fileList = new List(); + foreach (string file in result) { fileList.Add(file); } + // Assume order is not important from EnumeratePSFiles and sort the array so we can use deterministic asserts + fileList.Sort(); + + if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Core")) + { + // .Net Core doesn't appear to use the same three letter pattern matching rule although the docs + // suggest it should be find the '.ps1xml' files because we search for the pattern '*.ps1' + // ref https://docs.microsoft.com/en-us/dotnet/api/system.io.directory.getfiles?view=netcore-2.1#System_IO_Directory_GetFiles_System_String_System_String_System_IO_EnumerationOptions_ + Assert.Equal(4, fileList.Count); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "donotfind.ps1"), fileList[0]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psd1"), fileList[1]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psm1"), fileList[2]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"rootfile.ps1"), fileList[3]); + } + else + { + Assert.Equal(5, fileList.Count); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "donotfind.ps1"), fileList[0]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psd1"), fileList[1]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psm1"), fileList[2]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"other", "other.ps1xml"), fileList[3]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"rootfile.ps1"), fileList[4]); + } + } + + [Fact] + [Trait("Category", "Workspace")] public void CanDetermineIsPathInMemory() { string tempDir = Path.GetTempPath(); @@ -84,6 +121,7 @@ public void CanDetermineIsPathInMemory() } [Theory()] + [Trait("Category", "Workspace")] [MemberData(nameof(PathsToResolve), parameters: 2)] public void CorrectlyResolvesPaths(string givenPath, string expectedPath) { From c508720c9759f5da93a77432ac883f0fe3947f9f Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Sun, 10 Mar 2019 13:20:35 +0800 Subject: [PATCH 2/4] (GH-879) Use globbing when enumerating workspace files Previously the Workspace.EnumeratePSFiles method could only filter files based on file extension (*.ps1, *.psm1, *.psd1). However editor settings tend to use file glob patterns, but Editor Services did not have a library that could parse them. This commit: * Updates Editor Services to use the Microsoft.Extensions.FileSystemGlobbing library * Updated the build process to include the new FileSystemGlobbing DLL * The FileSystemGlobbing library uses an abstract file system to search, not an actual System.IO.FileSystem object. So to implement the same error handling and maximum depth recursion, a WorkspaceFileSystemWrapperFactory is used to create the Directory and File objects needed for the globbing library The WorkspaceFileSystemWrapperFactory can filter on: - Maximum recursion depth - Reparse points (Note that these aren't strictly Symlinks on windows. There are many other types of filesystem items which are reparse points - File system extension - Gracefully ignores any file access errors * The EnumeratePSFiles has two method signatures. One with no arguments which uses the Workspace object's default values and another where all arguments must be specified when enumerating the files * Adds tests for the EnumeratePSFiles method to ensure that it filters on glob and recursion depth. --- PowerShellEditorServices.build.ps1 | 1 + .../PowerShellEditorServices.csproj | 1 + .../Workspace/Workspace.cs | 189 ++++----- .../Workspace/WorkspaceFileSystemWrapper.cs | 381 ++++++++++++++++++ .../Session/WorkspaceTests.cs | 85 +++- 5 files changed, 532 insertions(+), 125 deletions(-) create mode 100644 src/PowerShellEditorServices/Workspace/WorkspaceFileSystemWrapper.cs diff --git a/PowerShellEditorServices.build.ps1 b/PowerShellEditorServices.build.ps1 index 9991125b5..74a80db7a 100644 --- a/PowerShellEditorServices.build.ps1 +++ b/PowerShellEditorServices.build.ps1 @@ -56,6 +56,7 @@ $script:RequiredBuildAssets = @{ 'publish/Serilog.Sinks.Async.dll', 'publish/Serilog.Sinks.Console.dll', 'publish/Serilog.Sinks.File.dll', + 'publish/Microsoft.Extensions.FileSystemGlobbing.dll', 'Microsoft.PowerShell.EditorServices.dll', 'Microsoft.PowerShell.EditorServices.pdb' ) diff --git a/src/PowerShellEditorServices/PowerShellEditorServices.csproj b/src/PowerShellEditorServices/PowerShellEditorServices.csproj index df15c0555..fd282a96d 100644 --- a/src/PowerShellEditorServices/PowerShellEditorServices.csproj +++ b/src/PowerShellEditorServices/PowerShellEditorServices.csproj @@ -21,6 +21,7 @@ + diff --git a/src/PowerShellEditorServices/Workspace/Workspace.cs b/src/PowerShellEditorServices/Workspace/Workspace.cs index b9ca0185a..e62eafd63 100644 --- a/src/PowerShellEditorServices/Workspace/Workspace.cs +++ b/src/PowerShellEditorServices/Workspace/Workspace.cs @@ -11,6 +11,8 @@ using System.Security; using System.Text; using System.Runtime.InteropServices; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; namespace Microsoft.PowerShell.EditorServices { @@ -22,11 +24,29 @@ public class Workspace { #region Private Fields - private static readonly string[] s_psFilePatterns = new [] + // List of all file extensions considered PowerShell files in the .Net Core Framework. + private static readonly string[] s_psFileExtensionsCoreFramework = { - "*.ps1", - "*.psm1", - "*.psd1" + ".ps1", + ".psm1", + ".psd1" + }; + + // .Net Core doesn't appear to use the same three letter pattern matching rule although the docs + // suggest it should be find the '.ps1xml' files because we search for the pattern '*.ps1'. + // ref https://docs.microsoft.com/en-us/dotnet/api/system.io.directory.getfiles?view=netcore-2.1#System_IO_Directory_GetFiles_System_String_System_String_System_IO_EnumerationOptions_ + private static readonly string[] s_psFileExtensionsFullFramework = + { + ".ps1", + ".psm1", + ".psd1", + ".ps1xml" + }; + + // An array of globs which includes everything. + private static readonly string[] s_psIncludeAllGlob = new [] + { + "**/*" }; private ILogger logger; @@ -42,6 +62,16 @@ public class Workspace /// public string WorkspacePath { get; set; } + /// + /// Gets or sets the default list of file globs to exclude during workspace searches. + /// + public List ExcludeFilesGlob { get; set; } + + /// + /// Gets or sets whether the workspace should follow symlinks in search operations. + /// + public bool FollowSymlinks { get; set; } + #endregion #region Constructors @@ -55,6 +85,8 @@ public Workspace(Version powerShellVersion, ILogger logger) { this.powerShellVersion = powerShellVersion; this.logger = logger; + this.ExcludeFilesGlob = new List(); + this.FollowSymlinks = true; } #endregion @@ -282,135 +314,56 @@ public string GetRelativePath(string filePath) } /// - /// Enumerate all the PowerShell (ps1, psm1, psd1) files in the workspace in a recursive manner + /// Enumerate all the PowerShell (ps1, psm1, psd1) files in the workspace in a recursive manner, using default values. /// - /// An enumerator over the PowerShell files found in the workspace + /// An enumerator over the PowerShell files found in the workspace. public IEnumerable EnumeratePSFiles() { - if (WorkspacePath == null || !Directory.Exists(WorkspacePath)) - { - return Enumerable.Empty(); - } - - var foundFiles = new List(); - this.RecursivelyEnumerateFiles(WorkspacePath, ref foundFiles); - return foundFiles; + return EnumeratePSFiles( + ExcludeFilesGlob.ToArray(), + s_psIncludeAllGlob, + maxDepth: 64, + ignoreReparsePoints: !FollowSymlinks + ); } - #endregion - - #region Private Methods - /// - /// Find PowerShell files recursively down from a given directory path. - /// Currently collects files in depth-first order. - /// Directory.GetFiles(folderPath, pattern, SearchOption.AllDirectories) would provide this, - /// but a cycle in the filesystem will cause that to enter an infinite loop. + /// Enumerate all the PowerShell (ps1, psm1, psd1) files in the workspace in a recursive manner. /// - /// The path of the current directory to find files in - /// The accumulator for files found so far. - /// The current depth of the recursion from the original base directory. - private void RecursivelyEnumerateFiles(string folderPath, ref List foundFiles, int currDepth = 0) + /// An enumerator over the PowerShell files found in the workspace. + public IEnumerable EnumeratePSFiles( + string[] excludeGlobs, + string[] includeGlobs, + int maxDepth, + bool ignoreReparsePoints + ) { - const int recursionDepthLimit = 64; - - // Look for any PowerShell files in the current directory - foreach (string pattern in s_psFilePatterns) - { - string[] psFiles; - try - { - psFiles = Directory.GetFiles(folderPath, pattern, SearchOption.TopDirectoryOnly); - } - catch (DirectoryNotFoundException e) - { - this.logger.WriteHandledException( - $"Could not enumerate files in the path '{folderPath}' due to it being an invalid path", - e); - - continue; - } - catch (PathTooLongException e) - { - this.logger.WriteHandledException( - $"Could not enumerate files in the path '{folderPath}' due to the path being too long", - e); - - continue; - } - catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException) - { - this.logger.WriteHandledException( - $"Could not enumerate files in the path '{folderPath}' due to the path not being accessible", - e); - - continue; - } - catch (Exception e) - { - this.logger.WriteHandledException( - $"Could not enumerate files in the path '{folderPath}' due to an exception", - e); - - continue; - } - - foundFiles.AddRange(psFiles); - } - - // Prevent unbounded recursion here - if (currDepth >= recursionDepthLimit) - { - this.logger.Write(LogLevel.Warning, $"Recursion depth limit hit for path {folderPath}"); - return; - } - - // Add the recursive directories to search next - string[] subDirs; - try - { - subDirs = Directory.GetDirectories(folderPath); - } - catch (DirectoryNotFoundException e) - { - this.logger.WriteHandledException( - $"Could not enumerate directories in the path '{folderPath}' due to it being an invalid path", - e); - - return; - } - catch (PathTooLongException e) - { - this.logger.WriteHandledException( - $"Could not enumerate directories in the path '{folderPath}' due to the path being too long", - e); - - return; - } - catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException) - { - this.logger.WriteHandledException( - $"Could not enumerate directories in the path '{folderPath}' due to the path not being accessible", - e); - - return; - } - catch (Exception e) + if (WorkspacePath == null || !Directory.Exists(WorkspacePath)) { - this.logger.WriteHandledException( - $"Could not enumerate directories in the path '{folderPath}' due to an exception", - e); - - return; + yield break; } - - foreach (string subDir in subDirs) + var matcher = new Microsoft.Extensions.FileSystemGlobbing.Matcher(); + foreach (string pattern in includeGlobs) { matcher.AddInclude(pattern); } + foreach (string pattern in excludeGlobs) { matcher.AddExclude(pattern); } + + var fsFactory = new WorkspaceFileSystemWrapperFactory( + WorkspacePath, + maxDepth, + Utils.IsNetCore ? s_psFileExtensionsCoreFramework : s_psFileExtensionsFullFramework, + ignoreReparsePoints, + logger + ); + var fileMatchResult = matcher.Execute(fsFactory.RootDirectory); + foreach (FilePatternMatch item in fileMatchResult.Files) { - RecursivelyEnumerateFiles(subDir, ref foundFiles, currDepth: currDepth + 1); + yield return Path.Combine(WorkspacePath, item.Path.Replace('/', Path.DirectorySeparatorChar)); } } + #endregion + + #region Private Methods /// /// Recusrively searches through referencedFiles in scriptFiles /// and builds a Dictonary of the file references diff --git a/src/PowerShellEditorServices/Workspace/WorkspaceFileSystemWrapper.cs b/src/PowerShellEditorServices/Workspace/WorkspaceFileSystemWrapper.cs new file mode 100644 index 000000000..d90d0b4ec --- /dev/null +++ b/src/PowerShellEditorServices/Workspace/WorkspaceFileSystemWrapper.cs @@ -0,0 +1,381 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.PowerShell.EditorServices.Utility; +using System; +using System.Collections.Generic; +using System.IO; +using System.Security; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; + +namespace Microsoft.PowerShell.EditorServices +{ + + /// + /// A FileSystem wrapper class which only returns files and directories that the consumer is interested in, + /// with a maximum recursion depth and silently ignores most file system errors. Typically this is used by the + /// Microsoft.Extensions.FileSystemGlobbing library. + /// + public class WorkspaceFileSystemWrapperFactory + { + private readonly DirectoryInfoBase _rootDirectory; + private readonly string[] _allowedExtensions; + private readonly bool _ignoreReparsePoints; + + /// + /// Gets the maximum depth of the directories that will be searched + /// + internal int MaxRecursionDepth { get; } + + /// + /// Gets the logging facility + /// + internal ILogger Logger { get; } + + /// + /// Gets the directory where the factory is rooted. Only files and directories at this level, or deeper, will be visible + /// by the wrapper + /// + public DirectoryInfoBase RootDirectory + { + get { return _rootDirectory; } + } + + /// + /// Creates a new FileWrapper Factory + /// + /// The path to the root directory for the factory. + /// The maximum directory depth. + /// An array of file extensions that will be visible from the factory. For example [".ps1", ".psm1"] + /// Whether objects which are Reparse Points should be ignored. https://docs.microsoft.com/en-us/windows/desktop/fileio/reparse-points + /// An ILogger implementation used for writing log messages. + public WorkspaceFileSystemWrapperFactory(String rootPath, int recursionDepthLimit, string[] allowedExtensions, bool ignoreReparsePoints, ILogger logger) + { + MaxRecursionDepth = recursionDepthLimit; + _rootDirectory = new WorkspaceFileSystemDirectoryWrapper(this, new DirectoryInfo(rootPath), 0); + _allowedExtensions = allowedExtensions; + _ignoreReparsePoints = ignoreReparsePoints; + Logger = logger; + } + + /// + /// Creates a wrapped object from . + /// + internal DirectoryInfoBase CreateDirectoryInfoWrapper(DirectoryInfo dirInfo, int depth) => + new WorkspaceFileSystemDirectoryWrapper(this, dirInfo, depth >= 0 ? depth : 0); + + /// + /// Creates a wrapped object from . + /// + internal FileInfoBase CreateFileInfoWrapper(FileInfo fileInfo, int depth) => + new WorkspaceFileSystemFileInfoWrapper(this, fileInfo, depth >= 0 ? depth : 0); + + /// + /// Enumerates all objects in the specified directory and ignores most errors + /// + internal IEnumerable SafeEnumerateFileSystemInfos(DirectoryInfo dirInfo) + { + // Find the subdirectories + string[] subDirs; + try + { + subDirs = Directory.GetDirectories(dirInfo.FullName, "*", SearchOption.TopDirectoryOnly); + } + catch (DirectoryNotFoundException e) + { + Logger.WriteHandledException( + $"Could not enumerate directories in the path '{dirInfo.FullName}' due to it being an invalid path", + e); + + yield break; + } + catch (PathTooLongException e) + { + Logger.WriteHandledException( + $"Could not enumerate directories in the path '{dirInfo.FullName}' due to the path being too long", + e); + + yield break; + } + catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException) + { + Logger.WriteHandledException( + $"Could not enumerate directories in the path '{dirInfo.FullName}' due to the path not being accessible", + e); + + yield break; + } + catch (Exception e) + { + Logger.WriteHandledException( + $"Could not enumerate directories in the path '{dirInfo.FullName}' due to an exception", + e); + + yield break; + } + foreach (string dirPath in subDirs) + { + var subDirInfo = new DirectoryInfo(dirPath); + if (_ignoreReparsePoints && (subDirInfo.Attributes & FileAttributes.ReparsePoint) != 0) { continue; } + yield return subDirInfo; + } + + // Find the files + string[] filePaths; + try + { + filePaths = Directory.GetFiles(dirInfo.FullName, "*", SearchOption.TopDirectoryOnly); + } + catch (DirectoryNotFoundException e) + { + Logger.WriteHandledException( + $"Could not enumerate files in the path '{dirInfo.FullName}' due to it being an invalid path", + e); + + yield break; + } + catch (PathTooLongException e) + { + Logger.WriteHandledException( + $"Could not enumerate files in the path '{dirInfo.FullName}' due to the path being too long", + e); + + yield break; + } + catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException) + { + Logger.WriteHandledException( + $"Could not enumerate files in the path '{dirInfo.FullName}' due to the path not being accessible", + e); + + yield break; + } + catch (Exception e) + { + Logger.WriteHandledException( + $"Could not enumerate files in the path '{dirInfo.FullName}' due to an exception", + e); + + yield break; + } + foreach (string filePath in filePaths) + { + var fileInfo = new FileInfo(filePath); + if (_allowedExtensions == null || _allowedExtensions.Length == 0) { yield return fileInfo; continue; } + if (_ignoreReparsePoints && (fileInfo.Attributes & FileAttributes.ReparsePoint) != 0) { continue; } + foreach (string extension in _allowedExtensions) + { + if (fileInfo.Extension == extension) { yield return fileInfo; break; } + } + } + } + } + + /// + /// Wraps an instance of and provides implementation of + /// . + /// Based on https://github.com/aspnet/Extensions/blob/c087cadf1dfdbd2b8785ef764e5ef58a1a7e5ed0/src/FileSystemGlobbing/src/Abstractions/DirectoryInfoWrapper.cs + /// + public class WorkspaceFileSystemDirectoryWrapper : DirectoryInfoBase + { + private readonly DirectoryInfo _concreteDirectoryInfo; + private readonly bool _isParentPath; + private readonly WorkspaceFileSystemWrapperFactory _fsWrapperFactory; + private readonly int _depth; + + /// + /// Initializes an instance of . + /// + public WorkspaceFileSystemDirectoryWrapper(WorkspaceFileSystemWrapperFactory factory, DirectoryInfo directoryInfo, int depth) + { + _concreteDirectoryInfo = directoryInfo; + _isParentPath = (depth == 0); + _fsWrapperFactory = factory; + _depth = depth; + } + + /// + public override IEnumerable EnumerateFileSystemInfos() + { + if (!_concreteDirectoryInfo.Exists || _depth >= _fsWrapperFactory.MaxRecursionDepth) { yield break; } + foreach (FileSystemInfo fileSystemInfo in _fsWrapperFactory.SafeEnumerateFileSystemInfos(_concreteDirectoryInfo)) + { + switch (fileSystemInfo) + { + case DirectoryInfo dirInfo: + yield return _fsWrapperFactory.CreateDirectoryInfoWrapper(dirInfo, _depth + 1); + break; + case FileInfo fileInfo: + yield return _fsWrapperFactory.CreateFileInfoWrapper(fileInfo, _depth); + break; + default: + // We should NEVER get here, but if we do just continue on + break; + } + } + } + + /// + /// Returns an instance of that represents a subdirectory. + /// + /// + /// If equals '..', this returns the parent directory. + /// + /// The directory name. + /// The directory + public override DirectoryInfoBase GetDirectory(string name) + { + bool isParentPath = string.Equals(name, "..", StringComparison.Ordinal); + + if (isParentPath) { return ParentDirectory; } + + var dirs = _concreteDirectoryInfo.GetDirectories(name); + + if (dirs.Length == 1) { return _fsWrapperFactory.CreateDirectoryInfoWrapper(dirs[0], _depth + 1); } + if (dirs.Length == 0) { return null; } + // This shouldn't happen. The parameter name isn't supposed to contain wild card. + throw new InvalidOperationException( + string.Format( + System.Globalization.CultureInfo.CurrentCulture, + "More than one sub directories are found under {0} with name {1}.", + _concreteDirectoryInfo.FullName, name)); + } + + /// + public override FileInfoBase GetFile(string name) => _fsWrapperFactory.CreateFileInfoWrapper(new FileInfo(Path.Combine(_concreteDirectoryInfo.FullName, name)), _depth); + + /// + public override string Name => _isParentPath ? ".." : _concreteDirectoryInfo.Name; + + /// + /// Returns the full path to the directory. + /// + public override string FullName => _concreteDirectoryInfo.FullName; + + /// + /// Safely calculates the parent of this directory, swallowing most errors. + /// + private DirectoryInfoBase SafeParentDirectory() + { + try + { + return _fsWrapperFactory.CreateDirectoryInfoWrapper(_concreteDirectoryInfo.Parent, _depth - 1); + } + catch (DirectoryNotFoundException e) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteDirectoryInfo.FullName}' due to it being an invalid path", + e); + } + catch (PathTooLongException e) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteDirectoryInfo.FullName}' due to the path being too long", + e); + } + catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteDirectoryInfo.FullName}' due to the path not being accessible", + e); + } + catch (Exception e) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteDirectoryInfo.FullName}' due to an exception", + e); + } + return null; + } + + /// + /// Returns the parent directory. (Overrides ). + /// + public override DirectoryInfoBase ParentDirectory + { + get + { + return SafeParentDirectory(); + } + } + } + + /// + /// Wraps an instance of to provide implementation of . + /// + public class WorkspaceFileSystemFileInfoWrapper : FileInfoBase + { + private readonly FileInfo _concreteFileInfo; + private readonly WorkspaceFileSystemWrapperFactory _fsWrapperFactory; + private readonly int _depth; + + /// + /// Initializes instance of to wrap the specified object . + /// + public WorkspaceFileSystemFileInfoWrapper(WorkspaceFileSystemWrapperFactory factory, FileInfo fileInfo, int depth) + { + _fsWrapperFactory = factory; + _concreteFileInfo = fileInfo; + _depth = depth; + } + + /// + /// The file name. (Overrides ). + /// + public override string Name => _concreteFileInfo.Name; + + /// + /// The full path of the file. (Overrides ). + /// + public override string FullName => _concreteFileInfo.FullName; + + /// + /// Safely calculates the parent of this file, swallowing most errors. + /// + private DirectoryInfoBase SafeParentDirectory() + { + try + { + return _fsWrapperFactory.CreateDirectoryInfoWrapper(_concreteFileInfo.Directory, _depth); + } + catch (DirectoryNotFoundException e) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteFileInfo.FullName}' due to it being an invalid path", + e); + } + catch (PathTooLongException e) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteFileInfo.FullName}' due to the path being too long", + e); + } + catch (Exception e) when (e is SecurityException || e is UnauthorizedAccessException) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteFileInfo.FullName}' due to the path not being accessible", + e); + } + catch (Exception e) + { + _fsWrapperFactory.Logger.WriteHandledException( + $"Could not get parent of '{_concreteFileInfo.FullName}' due to an exception", + e); + } + return null; + } + + /// + /// The directory containing the file. (Overrides ). + /// + public override DirectoryInfoBase ParentDirectory + { + get + { + return SafeParentDirectory(); + } + } + } +} diff --git a/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs b/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs index ca61e44c4..f20407405 100644 --- a/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs +++ b/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs @@ -47,19 +47,55 @@ public void CanResolveWorkspaceRelativePath() Assert.Equal(testPathAnotherDrive, workspace.GetRelativePath(testPathAnotherDrive)); } - [Fact] - [Trait("Category", "Workspace")] - public void CanRecurseDirectoryTree() + public static Workspace FixturesWorkspace() { - Workspace workspace = new Workspace(PowerShellVersion, Logging.NullLogger); - workspace.WorkspacePath = TestUtilities.NormalizePath("Fixtures/Workspace"); + return new Workspace(PowerShellVersion, Logging.NullLogger) { + WorkspacePath = TestUtilities.NormalizePath("Fixtures/Workspace") + }; + } - IEnumerable result = workspace.EnumeratePSFiles(); - List fileList = new List(); + // These are the default values for the EnumeratePSFiles() method + // in Microsoft.PowerShell.EditorServices.Workspace class + private static string[] s_defaultExcludeGlobs = new string[0]; + private static string[] s_defaultIncludeGlobs = new [] { "**/*" }; + private static int s_defaultMaxDepth = 64; + private static bool s_defaultIgnoreReparsePoints = false; + + public static List ExecuteEnumeratePSFiles( + Workspace workspace, + string[] excludeGlobs, + string[] includeGlobs, + int maxDepth, + bool ignoreReparsePoints + ) + { + var result = workspace.EnumeratePSFiles( + excludeGlobs: excludeGlobs, + includeGlobs: includeGlobs, + maxDepth: maxDepth, + ignoreReparsePoints: ignoreReparsePoints + ); + var fileList = new List(); foreach (string file in result) { fileList.Add(file); } // Assume order is not important from EnumeratePSFiles and sort the array so we can use deterministic asserts fileList.Sort(); + return fileList; + } + + [Fact] + [Trait("Category", "Workspace")] + public void CanRecurseDirectoryTree() + { + var workspace = FixturesWorkspace(); + var fileList = ExecuteEnumeratePSFiles( + workspace: workspace, + excludeGlobs: s_defaultExcludeGlobs, + includeGlobs: s_defaultIncludeGlobs, + maxDepth: s_defaultMaxDepth, + ignoreReparsePoints: s_defaultIgnoreReparsePoints + ); + if (RuntimeInformation.FrameworkDescription.StartsWith(".NET Core")) { // .Net Core doesn't appear to use the same three letter pattern matching rule although the docs @@ -82,6 +118,41 @@ public void CanRecurseDirectoryTree() } } + [Fact] + [Trait("Category", "Workspace")] + public void CanRecurseDirectoryTreeWithLimit() + { + var workspace = FixturesWorkspace(); + var fileList = ExecuteEnumeratePSFiles( + workspace: workspace, + excludeGlobs: s_defaultExcludeGlobs, + includeGlobs: s_defaultIncludeGlobs, + maxDepth: 1, + ignoreReparsePoints: s_defaultIgnoreReparsePoints + ); + + Assert.Equal(1, fileList.Count); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"rootfile.ps1"), fileList[0]); + } + + [Fact] + [Trait("Category", "Workspace")] + public void CanRecurseDirectoryTreeWithGlobs() + { + var workspace = FixturesWorkspace(); + var fileList = ExecuteEnumeratePSFiles( + workspace: workspace, + excludeGlobs: new [] {"**/donotfind*"}, // Exclude any files starting with donotfind + includeGlobs: new [] {"**/*.ps1", "**/*.psd1"}, // Only include PS1 and PSD1 files + maxDepth: s_defaultMaxDepth, + ignoreReparsePoints: s_defaultIgnoreReparsePoints + ); + + Assert.Equal(2, fileList.Count); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psd1"), fileList[0]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"rootfile.ps1"), fileList[1]); + } + [Fact] [Trait("Category", "Workspace")] public void CanDetermineIsPathInMemory() From ed335757814a89933ef83f1d92f7a7ed88316c62 Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Sat, 16 Mar 2019 21:42:27 +0800 Subject: [PATCH 3/4] (GH-879) Capture the editor settings and enforce file exclusions Previously the EnumeratePSFiles method was modified to be able to use globbing patterns to filter workspace files. This commit * Modifies the LanguageServerSettings class to capture the 'files' and 'search' Settings in order to determine the correct list of glob patterns to use when searching. Currently the 'files.exclude' and 'search.exclude' are merged together to generate the list of globs and then set the Workspace settings appropriately * Uses the 'search.followSymlinks' setting to determine whether to ignore reparse points Note that the LanguageClient must be configured to send these settings during the didChangeConfiguration events otherwise it will default to include everything. --- .../Server/LanguageServer.cs | 30 +++++++++++ .../Server/LanguageServerSettings.cs | 54 ++++++++++++++++--- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/src/PowerShellEditorServices.Protocol/Server/LanguageServer.cs b/src/PowerShellEditorServices.Protocol/Server/LanguageServer.cs index 37d791198..045a84936 100644 --- a/src/PowerShellEditorServices.Protocol/Server/LanguageServer.cs +++ b/src/PowerShellEditorServices.Protocol/Server/LanguageServer.cs @@ -727,6 +727,36 @@ await this.RunScriptDiagnosticsAsync( this.editorSession, eventContext); } + + // Convert the editor file glob patterns into an array for the Workspace + // Both the files.exclude and search.exclude hash tables look like (glob-text, is-enabled): + // "files.exclude" : { + // "Makefile": true, + // "*.html": true, + // "build/*": true + // } + var excludeFilePatterns = new List(); + if (configChangeParams.Settings.Files?.Exclude != null) + { + foreach(KeyValuePair patternEntry in configChangeParams.Settings.Files.Exclude) + { + if (patternEntry.Value) { excludeFilePatterns.Add(patternEntry.Key); } + } + } + if (configChangeParams.Settings.Search?.Exclude != null) + { + foreach(KeyValuePair patternEntry in configChangeParams.Settings.Files.Exclude) + { + if (patternEntry.Value && !excludeFilePatterns.Contains(patternEntry.Key)) { excludeFilePatterns.Add(patternEntry.Key); } + } + } + editorSession.Workspace.ExcludeFilesGlob = excludeFilePatterns; + + // Convert the editor file search options to Workspace properties + if (configChangeParams.Settings.Search?.FollowSymlinks != null) + { + editorSession.Workspace.FollowSymlinks = configChangeParams.Settings.Search.FollowSymlinks; + } } protected async Task HandleDefinitionRequestAsync( diff --git a/src/PowerShellEditorServices.Protocol/Server/LanguageServerSettings.cs b/src/PowerShellEditorServices.Protocol/Server/LanguageServerSettings.cs index 8cbf953b9..8ce27bcfb 100644 --- a/src/PowerShellEditorServices.Protocol/Server/LanguageServerSettings.cs +++ b/src/PowerShellEditorServices.Protocol/Server/LanguageServerSettings.cs @@ -6,6 +6,7 @@ using Microsoft.PowerShell.EditorServices.Utility; using System; using System.Collections; +using System.Collections.Generic; using System.IO; using System.Reflection; using System.Security; @@ -331,12 +332,51 @@ public void Update( } } - public class LanguageServerSettingsWrapper - { - // NOTE: This property is capitalized as 'Powershell' because the - // mode name sent from the client is written as 'powershell' and - // JSON.net is using camelCasing. + /// + /// Additional settings from the Language Client that affect Language Server operations but + /// do not exist under the 'powershell' section + /// + public class EditorFileSettings + { + /// + /// Exclude files globs consists of hashtable with the key as the glob and a boolean value to indicate if the + /// the glob is in effect. + /// + public Dictionary Exclude { get; set; } + } - public LanguageServerSettings Powershell { get; set; } - } + /// + /// Additional settings from the Language Client that affect Language Server operations but + /// do not exist under the 'powershell' section + /// + public class EditorSearchSettings + { + /// + /// Exclude files globs consists of hashtable with the key as the glob and a boolean value to indicate if the + /// the glob is in effect. + /// + public Dictionary Exclude { get; set; } + /// + /// Whether to follow symlinks when searching + /// + public bool FollowSymlinks { get; set; } = true; + } + + public class LanguageServerSettingsWrapper + { + // NOTE: This property is capitalized as 'Powershell' because the + // mode name sent from the client is written as 'powershell' and + // JSON.net is using camelCasing. + public LanguageServerSettings Powershell { get; set; } + + // NOTE: This property is capitalized as 'Files' because the + // mode name sent from the client is written as 'files' and + // JSON.net is using camelCasing. + public EditorFileSettings Files { get; set; } + + // NOTE: This property is capitalized as 'Search' because the + // mode name sent from the client is written as 'search' and + // JSON.net is using camelCasing. + public EditorSearchSettings Search { get; set; } } +} From 19b2703bf52017ad2e35d3478c70313f1b99355f Mon Sep 17 00:00:00 2001 From: Glenn Sarti Date: Thu, 2 May 2019 11:41:09 -0700 Subject: [PATCH 4/4] (GH-879) Do not normalise paths from EnumeratePSFiles Previously the paths emitted by `EnumeratePSFiles` were normalised to use the directory path separator appropriate for the platform. In particular on Windows the paths emitted by the Microsoft.Extensions.FileSystemGlobbing library contained both forward and backward slashes. However on inspection this is not required as all the paths are converted to URIs when communicating over LSP, so the normalisation is no longer required. This commit removes the normalisation and updates the tests to reflect the new paths. --- .../Workspace/Workspace.cs | 2 +- .../Session/WorkspaceTests.cs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/PowerShellEditorServices/Workspace/Workspace.cs b/src/PowerShellEditorServices/Workspace/Workspace.cs index e62eafd63..34cb0d641 100644 --- a/src/PowerShellEditorServices/Workspace/Workspace.cs +++ b/src/PowerShellEditorServices/Workspace/Workspace.cs @@ -357,7 +357,7 @@ bool ignoreReparsePoints var fileMatchResult = matcher.Execute(fsFactory.RootDirectory); foreach (FilePatternMatch item in fileMatchResult.Files) { - yield return Path.Combine(WorkspacePath, item.Path.Replace('/', Path.DirectorySeparatorChar)); + yield return Path.Combine(WorkspacePath, item.Path); } } diff --git a/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs b/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs index f20407405..2b8919cc1 100644 --- a/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs +++ b/test/PowerShellEditorServices.Test/Session/WorkspaceTests.cs @@ -102,18 +102,18 @@ public void CanRecurseDirectoryTree() // suggest it should be find the '.ps1xml' files because we search for the pattern '*.ps1' // ref https://docs.microsoft.com/en-us/dotnet/api/system.io.directory.getfiles?view=netcore-2.1#System_IO_Directory_GetFiles_System_String_System_String_System_IO_EnumerationOptions_ Assert.Equal(4, fileList.Count); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "donotfind.ps1"), fileList[0]); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psd1"), fileList[1]); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psm1"), fileList[2]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested") + "/donotfind.ps1", fileList[0]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested") + "/nestedmodule.psd1", fileList[1]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested") + "/nestedmodule.psm1", fileList[2]); Assert.Equal(Path.Combine(workspace.WorkspacePath,"rootfile.ps1"), fileList[3]); } else { Assert.Equal(5, fileList.Count); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "donotfind.ps1"), fileList[0]); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psd1"), fileList[1]); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psm1"), fileList[2]); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"other", "other.ps1xml"), fileList[3]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested") + "/donotfind.ps1", fileList[0]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested") + "/nestedmodule.psd1", fileList[1]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested") + "/nestedmodule.psm1", fileList[2]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"other") + "/other.ps1xml", fileList[3]); Assert.Equal(Path.Combine(workspace.WorkspacePath,"rootfile.ps1"), fileList[4]); } } @@ -149,7 +149,7 @@ public void CanRecurseDirectoryTreeWithGlobs() ); Assert.Equal(2, fileList.Count); - Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested", "nestedmodule.psd1"), fileList[0]); + Assert.Equal(Path.Combine(workspace.WorkspacePath,"nested") + "/nestedmodule.psd1", fileList[0]); Assert.Equal(Path.Combine(workspace.WorkspacePath,"rootfile.ps1"), fileList[1]); }