From 714a48d296f160d0546b89a6e3efc4c673a41419 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Feb 2022 17:56:45 +0100 Subject: [PATCH 01/13] Moved ReadDir and ReadDirRecursive in their own file --- paths.go | 41 ------------------------------ readdir.go | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 41 deletions(-) create mode 100644 readdir.go diff --git a/paths.go b/paths.go index 6021208..b969606 100644 --- a/paths.go +++ b/paths.go @@ -306,47 +306,6 @@ func (p *Path) IsDirCheck() (bool, error) { return false, err } -// ReadDir returns a PathList containing the content of the directory -// pointed by the current Path -func (p *Path) ReadDir() (PathList, error) { - infos, err := ioutil.ReadDir(p.path) - if err != nil { - return nil, err - } - paths := PathList{} - for _, info := range infos { - path := p.Clone().Join(info.Name()) - paths.Add(path) - } - return paths, nil -} - -// ReadDirRecursive returns a PathList containing the content of the directory -// and its subdirectories pointed by the current Path -func (p *Path) ReadDirRecursive() (PathList, error) { - infos, err := ioutil.ReadDir(p.path) - if err != nil { - return nil, err - } - paths := PathList{} - for _, info := range infos { - path := p.Clone().Join(info.Name()) - paths.Add(path) - - if isDir, err := path.IsDirCheck(); err != nil { - return nil, err - } else if isDir { - subPaths, err := path.ReadDirRecursive() - if err != nil { - return nil, err - } - paths.AddAll(subPaths) - } - - } - return paths, nil -} - // CopyTo copies the contents of the file named src to the file named // by dst. The file will be created if it does not already exist. If the // destination file exists, all it's contents will be replaced by the contents diff --git a/readdir.go b/readdir.go new file mode 100644 index 0000000..a2dae00 --- /dev/null +++ b/readdir.go @@ -0,0 +1,73 @@ +/* + * This file is part of PathsHelper library. + * + * Copyright 2018-2022 Arduino AG (http://www.arduino.cc/) + * + * PathsHelper library is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + */ + +package paths + +import "io/ioutil" + +// ReadDir returns a PathList containing the content of the directory +// pointed by the current Path +func (p *Path) ReadDir() (PathList, error) { + infos, err := ioutil.ReadDir(p.path) + if err != nil { + return nil, err + } + paths := PathList{} + for _, info := range infos { + path := p.Clone().Join(info.Name()) + paths.Add(path) + } + return paths, nil +} + +// ReadDirRecursive returns a PathList containing the content of the directory +// and its subdirectories pointed by the current Path +func (p *Path) ReadDirRecursive() (PathList, error) { + infos, err := ioutil.ReadDir(p.path) + if err != nil { + return nil, err + } + paths := PathList{} + for _, info := range infos { + path := p.Clone().Join(info.Name()) + paths.Add(path) + + if isDir, err := path.IsDirCheck(); err != nil { + return nil, err + } else if isDir { + subPaths, err := path.ReadDirRecursive() + if err != nil { + return nil, err + } + paths.AddAll(subPaths) + } + + } + return paths, nil +} From d348d46afa917a880c834e0ee560a0478c26c507 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Feb 2022 16:14:23 +0100 Subject: [PATCH 02/13] Put SymLinkLoop test into his own test case --- paths_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paths_test.go b/paths_test.go index da28432..a814ef5 100644 --- a/paths_test.go +++ b/paths_test.go @@ -30,6 +30,7 @@ package paths import ( + "fmt" "os" "path/filepath" "runtime" @@ -278,7 +279,9 @@ func TestReadDirRecursive(t *testing.T) { pathEqualsTo(t, "_testdata/symlinktofolder/subfolder/file4", list[13]) pathEqualsTo(t, "_testdata/test.txt", list[14]) pathEqualsTo(t, "_testdata/test.txt.gz", list[15]) +} +func TestReadDirRecursiveSymLinkLoop(t *testing.T) { // Test symlink loop tmp, err := MkTempDir("", "") require.NoError(t, err) @@ -290,6 +293,7 @@ func TestReadDirRecursive(t *testing.T) { l, err := tmp.ReadDirRecursive() require.Error(t, err) + fmt.Println(err) require.Nil(t, l) } From 87555a7f6e15902e838567de861b5b004f042edb Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Feb 2022 16:15:28 +0100 Subject: [PATCH 03/13] Moved ReadDir* tests in their own file --- paths_test.go | 43 --------------------------- readdir_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 43 deletions(-) create mode 100644 readdir_test.go diff --git a/paths_test.go b/paths_test.go index a814ef5..4e7817a 100644 --- a/paths_test.go +++ b/paths_test.go @@ -30,8 +30,6 @@ package paths import ( - "fmt" - "os" "path/filepath" "runtime" "strings" @@ -256,47 +254,6 @@ func TestParents(t *testing.T) { pathEqualsTo(t, ".", parents2[4]) } -func TestReadDirRecursive(t *testing.T) { - testPath := New("_testdata") - - list, err := testPath.ReadDirRecursive() - require.NoError(t, err) - require.Len(t, list, 16) - - pathEqualsTo(t, "_testdata/anotherFile", list[0]) - pathEqualsTo(t, "_testdata/file", list[1]) - pathEqualsTo(t, "_testdata/folder", list[2]) - pathEqualsTo(t, "_testdata/folder/.hidden", list[3]) - pathEqualsTo(t, "_testdata/folder/file2", list[4]) - pathEqualsTo(t, "_testdata/folder/file3", list[5]) - pathEqualsTo(t, "_testdata/folder/subfolder", list[6]) - pathEqualsTo(t, "_testdata/folder/subfolder/file4", list[7]) - pathEqualsTo(t, "_testdata/symlinktofolder", list[8]) - pathEqualsTo(t, "_testdata/symlinktofolder/.hidden", list[9]) - pathEqualsTo(t, "_testdata/symlinktofolder/file2", list[10]) - pathEqualsTo(t, "_testdata/symlinktofolder/file3", list[11]) - pathEqualsTo(t, "_testdata/symlinktofolder/subfolder", list[12]) - pathEqualsTo(t, "_testdata/symlinktofolder/subfolder/file4", list[13]) - pathEqualsTo(t, "_testdata/test.txt", list[14]) - pathEqualsTo(t, "_testdata/test.txt.gz", list[15]) -} - -func TestReadDirRecursiveSymLinkLoop(t *testing.T) { - // Test symlink loop - tmp, err := MkTempDir("", "") - require.NoError(t, err) - defer tmp.RemoveAll() - - folder := tmp.Join("folder") - err = os.Symlink(tmp.String(), folder.String()) - require.NoError(t, err) - - l, err := tmp.ReadDirRecursive() - require.Error(t, err) - fmt.Println(err) - require.Nil(t, l) -} - func TestFilterDirs(t *testing.T) { testPath := New("_testdata") diff --git a/readdir_test.go b/readdir_test.go new file mode 100644 index 0000000..a7ad2a9 --- /dev/null +++ b/readdir_test.go @@ -0,0 +1,79 @@ +/* + * This file is part of PathsHelper library. + * + * Copyright 2018-2022 Arduino AG (http://www.arduino.cc/) + * + * PathsHelper library is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As a special exception, you may use this file as part of a free software + * library without restriction. Specifically, if other files instantiate + * templates or use macros or inline functions from this file, or you compile + * this file and link it with other files to produce an executable, this + * file does not by itself cause the resulting executable to be covered by + * the GNU General Public License. This exception does not however + * invalidate any other reasons why the executable file might be covered by + * the GNU General Public License. + */ + +package paths + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestReadDirRecursive(t *testing.T) { + testPath := New("_testdata") + + list, err := testPath.ReadDirRecursive() + require.NoError(t, err) + require.Len(t, list, 16) + + pathEqualsTo(t, "_testdata/anotherFile", list[0]) + pathEqualsTo(t, "_testdata/file", list[1]) + pathEqualsTo(t, "_testdata/folder", list[2]) + pathEqualsTo(t, "_testdata/folder/.hidden", list[3]) + pathEqualsTo(t, "_testdata/folder/file2", list[4]) + pathEqualsTo(t, "_testdata/folder/file3", list[5]) + pathEqualsTo(t, "_testdata/folder/subfolder", list[6]) + pathEqualsTo(t, "_testdata/folder/subfolder/file4", list[7]) + pathEqualsTo(t, "_testdata/symlinktofolder", list[8]) + pathEqualsTo(t, "_testdata/symlinktofolder/.hidden", list[9]) + pathEqualsTo(t, "_testdata/symlinktofolder/file2", list[10]) + pathEqualsTo(t, "_testdata/symlinktofolder/file3", list[11]) + pathEqualsTo(t, "_testdata/symlinktofolder/subfolder", list[12]) + pathEqualsTo(t, "_testdata/symlinktofolder/subfolder/file4", list[13]) + pathEqualsTo(t, "_testdata/test.txt", list[14]) + pathEqualsTo(t, "_testdata/test.txt.gz", list[15]) +} + +func TestReadDirRecursiveSymLinkLoop(t *testing.T) { + // Test symlink loop + tmp, err := MkTempDir("", "") + require.NoError(t, err) + defer tmp.RemoveAll() + + folder := tmp.Join("folder") + err = os.Symlink(tmp.String(), folder.String()) + require.NoError(t, err) + + l, err := tmp.ReadDirRecursive() + require.Error(t, err) + fmt.Println(err) + require.Nil(t, l) +} From 202137f9364c85c4abdd2492b45ec1a22029c0b0 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Feb 2022 18:07:40 +0100 Subject: [PATCH 04/13] Removed useless path Clone-ing Join already creates a copy of the path. --- readdir.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readdir.go b/readdir.go index a2dae00..34aba3c 100644 --- a/readdir.go +++ b/readdir.go @@ -40,7 +40,7 @@ func (p *Path) ReadDir() (PathList, error) { } paths := PathList{} for _, info := range infos { - path := p.Clone().Join(info.Name()) + path := p.Join(info.Name()) paths.Add(path) } return paths, nil @@ -55,7 +55,7 @@ func (p *Path) ReadDirRecursive() (PathList, error) { } paths := PathList{} for _, info := range infos { - path := p.Clone().Join(info.Name()) + path := p.Join(info.Name()) paths.Add(path) if isDir, err := path.IsDirCheck(); err != nil { From 3009dd612b1818bc6a713218224d1944d6622b37 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 3 Feb 2022 18:07:03 +0100 Subject: [PATCH 05/13] Added ReadDirFilter feature in ReadDir --- readdir.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/readdir.go b/readdir.go index 34aba3c..dded5be 100644 --- a/readdir.go +++ b/readdir.go @@ -29,18 +29,30 @@ package paths -import "io/ioutil" +import ( + "io/ioutil" +) + +// ReadDirFilter is a filter for Path.ReadDir and Path.ReadDirRecursive methods. +// The filter should return true to accept a file or false to reject it. +type ReadDirFilter func(file *Path) bool // ReadDir returns a PathList containing the content of the directory -// pointed by the current Path -func (p *Path) ReadDir() (PathList, error) { +// pointed by the current Path. The resulting list is filtered by the given filters chained. +func (p *Path) ReadDir(filters ...ReadDirFilter) (PathList, error) { infos, err := ioutil.ReadDir(p.path) if err != nil { return nil, err } paths := PathList{} +fileLoop: for _, info := range infos { path := p.Join(info.Name()) + for _, filter := range filters { + if !filter(path) { + continue fileLoop + } + } paths.Add(path) } return paths, nil From 8d03be013ff014727a659698665cda5207445541 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Feb 2022 14:29:35 +0100 Subject: [PATCH 06/13] Added basic filters --- readdir.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/readdir.go b/readdir.go index dded5be..d8c32af 100644 --- a/readdir.go +++ b/readdir.go @@ -83,3 +83,38 @@ func (p *Path) ReadDirRecursive() (PathList, error) { } return paths, nil } + +// FilterDirectories is a ReadDirFilter that accepts only directories +func FilterDirectories() ReadDirFilter { + return func(path *Path) bool { + return path.IsDir() + } +} + +// FilterOutDirectories is a ReadDirFilter that rejects all directories +func FilterOutDirectories() ReadDirFilter { + return func(path *Path) bool { + return !path.IsDir() + } +} + +// OrFilter creates a ReadDirFilter that accepts all items that are accepted by x or by y +func OrFilter(x, y ReadDirFilter) ReadDirFilter { + return func(path *Path) bool { + return x(path) || y(path) + } +} + +// AndFilter creates a ReadDirFilter that accepts all items that are accepted by both x and y +func AndFilter(x, y ReadDirFilter) ReadDirFilter { + return func(path *Path) bool { + return x(path) && y(path) + } +} + +// NotFilter creates a ReadDifFilter that accepts all items rejected by x and viceversa +func NotFilter(x ReadDirFilter) ReadDirFilter { + return func(path *Path) bool { + return !x(path) + } +} From 86d74cc936f33bc7fc744909b4b4874e9da7737a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Feb 2022 14:29:47 +0100 Subject: [PATCH 07/13] Added ReadDirRecursiveFiltered function --- readdir.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/readdir.go b/readdir.go index d8c32af..42fc7b2 100644 --- a/readdir.go +++ b/readdir.go @@ -84,6 +84,49 @@ func (p *Path) ReadDirRecursive() (PathList, error) { return paths, nil } +// ReadDirRecursiveFiltered returns a PathList containing the content of the directory +// and its subdirectories pointed by the current Path, filtered by the given skipFilter +// and filters: +// - `recursionFilter` is a filter that is checked to determine if the subdirectory must +// by visited recursively (if the filter rejects the entry, the entry is not visited +// but can still be added to the result) +// - `filters` are the filters that are checked to determine if the entry should be +// added to the resulting PathList +func (p *Path) ReadDirRecursiveFiltered(recursionFilter ReadDirFilter, filters ...ReadDirFilter) (PathList, error) { + infos, err := ioutil.ReadDir(p.path) + if err != nil { + return nil, err + } + paths := PathList{} + for _, info := range infos { + path := p.Join(info.Name()) + + accept := true + for _, filter := range filters { + if !filter(path) { + accept = false + break + } + } + if accept { + paths.Add(path) + } + + if recursionFilter == nil || recursionFilter(path) { + if isDir, err := path.IsDirCheck(); err != nil { + return nil, err + } else if isDir { + subPaths, err := path.ReadDirRecursiveFiltered(recursionFilter, filters...) + if err != nil { + return nil, err + } + paths.AddAll(subPaths) + } + } + } + return paths, nil +} + // FilterDirectories is a ReadDirFilter that accepts only directories func FilterDirectories() ReadDirFilter { return func(path *Path) bool { From 89e8607ae00e269cc75781a3fb6bfb880a34966f Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Feb 2022 16:06:19 +0100 Subject: [PATCH 08/13] Added tests --- readdir_test.go | 92 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/readdir_test.go b/readdir_test.go index a7ad2a9..7863c5d 100644 --- a/readdir_test.go +++ b/readdir_test.go @@ -76,4 +76,96 @@ func TestReadDirRecursiveSymLinkLoop(t *testing.T) { require.Error(t, err) fmt.Println(err) require.Nil(t, l) + + l, err = tmp.ReadDirRecursiveFiltered(nil) + require.Error(t, err) + fmt.Println(err) + require.Nil(t, l) +} + +func TestReadDirRecursiveFiltered(t *testing.T) { + testdata := New("_testdata") + l, err := testdata.ReadDirRecursiveFiltered(nil) + require.NoError(t, err) + l.Sort() + require.Equal(t, []string{ + "_testdata/anotherFile", + "_testdata/file", + "_testdata/folder", + "_testdata/folder/.hidden", + "_testdata/folder/file2", + "_testdata/folder/file3", + "_testdata/folder/subfolder", + "_testdata/folder/subfolder/file4", + "_testdata/symlinktofolder", + "_testdata/symlinktofolder/.hidden", + "_testdata/symlinktofolder/file2", + "_testdata/symlinktofolder/file3", + "_testdata/symlinktofolder/subfolder", + "_testdata/symlinktofolder/subfolder/file4", + "_testdata/test.txt", + "_testdata/test.txt.gz"}, l.AsStrings()) + + l, err = testdata.ReadDirRecursiveFiltered(FilterOutDirectories()) + require.NoError(t, err) + l.Sort() + require.Equal(t, []string{ + "_testdata/anotherFile", + "_testdata/file", + "_testdata/folder", // <- this is listed but not traversed + // "_testdata/folder/.hidden", + // "_testdata/folder/file2", + // "_testdata/folder/file3", + // "_testdata/folder/subfolder", + // "_testdata/folder/subfolder/file4", + "_testdata/symlinktofolder", // <- this is listed but not traversed + // "_testdata/symlinktofolder/.hidden", + // "_testdata/symlinktofolder/file2", + // "_testdata/symlinktofolder/file3", + // "_testdata/symlinktofolder/subfolder", + // "_testdata/symlinktofolder/subfolder/file4", + "_testdata/test.txt", + "_testdata/test.txt.gz"}, l.AsStrings()) + + l, err = testdata.ReadDirRecursiveFiltered(nil, FilterOutDirectories()) + require.NoError(t, err) + l.Sort() + require.Equal(t, []string{ + "_testdata/anotherFile", + "_testdata/file", + // "_testdata/folder", <- this is filtered but still traversed + "_testdata/folder/.hidden", + "_testdata/folder/file2", + "_testdata/folder/file3", + // "_testdata/folder/subfolder", <- this is filtered but still traversed + "_testdata/folder/subfolder/file4", + // "_testdata/symlinktofolder", <- this is filtered but still traversed + "_testdata/symlinktofolder/.hidden", + "_testdata/symlinktofolder/file2", + "_testdata/symlinktofolder/file3", + // "_testdata/symlinktofolder/subfolder", <- this is filtered but still traversed + "_testdata/symlinktofolder/subfolder/file4", + "_testdata/test.txt", + "_testdata/test.txt.gz"}, l.AsStrings()) + + l, err = testdata.ReadDirRecursiveFiltered(FilterOutDirectories(), FilterOutDirectories()) + require.NoError(t, err) + l.Sort() + require.Equal(t, []string{ + "_testdata/anotherFile", + "_testdata/file", + // "_testdata/folder", + // "_testdata/folder/.hidden", + // "_testdata/folder/file2", + // "_testdata/folder/file3", + // "_testdata/folder/subfolder", + // "_testdata/folder/subfolder/file4", + // "_testdata/symlinktofolder", + // "_testdata/symlinktofolder/.hidden", + // "_testdata/symlinktofolder/file2", + // "_testdata/symlinktofolder/file3", + // "_testdata/symlinktofolder/subfolder", + // "_testdata/symlinktofolder/subfolder/file4", + "_testdata/test.txt", + "_testdata/test.txt.gz"}, l.AsStrings()) } From 82c68d623459323755d9b4bc19b2c91bcebb6efd Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Feb 2022 17:16:44 +0100 Subject: [PATCH 09/13] Added filters for prefix and suffix --- readdir.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ readdir_test.go | 21 +++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/readdir.go b/readdir.go index 42fc7b2..9833e1d 100644 --- a/readdir.go +++ b/readdir.go @@ -31,6 +31,7 @@ package paths import ( "io/ioutil" + "strings" ) // ReadDirFilter is a filter for Path.ReadDir and Path.ReadDirRecursive methods. @@ -141,6 +142,60 @@ func FilterOutDirectories() ReadDirFilter { } } +// FilterSuffixes creates a ReadDirFilter that accepts only the given +// filename suffixes +func FilterSuffixes(allowedSuffixes ...string) ReadDirFilter { + return func(file *Path) bool { + for _, suffix := range allowedSuffixes { + if strings.HasSuffix(file.String(), suffix) { + return true + } + } + return false + } +} + +// FilterOutSuffixes creates a ReadDirFilter that rejects all the given +// filename suffixes +func FilterOutSuffixes(rejectedSuffixes ...string) ReadDirFilter { + return func(file *Path) bool { + for _, suffix := range rejectedSuffixes { + if strings.HasSuffix(file.String(), suffix) { + return false + } + } + return true + } +} + +// FilterPrefixes creates a ReadDirFilter that accepts only the given +// filename prefixes +func FilterPrefixes(allowedPrefixes ...string) ReadDirFilter { + return func(file *Path) bool { + name := file.Base() + for _, prefix := range allowedPrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false + } +} + +// FilterOutPrefixes creates a ReadDirFilter that rejects all the given +// filename prefixes +func FilterOutPrefixes(rejectedPrefixes ...string) ReadDirFilter { + return func(file *Path) bool { + name := file.Base() + for _, prefix := range rejectedPrefixes { + if strings.HasPrefix(name, prefix) { + return false + } + } + return true + } +} + // OrFilter creates a ReadDirFilter that accepts all items that are accepted by x or by y func OrFilter(x, y ReadDirFilter) ReadDirFilter { return func(path *Path) bool { diff --git a/readdir_test.go b/readdir_test.go index 7863c5d..a25c591 100644 --- a/readdir_test.go +++ b/readdir_test.go @@ -168,4 +168,25 @@ func TestReadDirRecursiveFiltered(t *testing.T) { // "_testdata/symlinktofolder/subfolder/file4", "_testdata/test.txt", "_testdata/test.txt.gz"}, l.AsStrings()) + + l, err = testdata.ReadDirRecursiveFiltered(FilterOutPrefixes("sub"), FilterOutSuffixes("3")) + require.NoError(t, err) + l.Sort() + require.Equal(t, []string{ + "_testdata/anotherFile", + "_testdata/file", + "_testdata/folder", + "_testdata/folder/.hidden", + "_testdata/folder/file2", + // "_testdata/folder/file3", <- filtered by Suffix("3") + "_testdata/folder/subfolder", // <- subfolder skipped by Prefix("sub") + // "_testdata/folder/subfolder/file4", + "_testdata/symlinktofolder", + "_testdata/symlinktofolder/.hidden", + "_testdata/symlinktofolder/file2", + // "_testdata/symlinktofolder/file3", <- filtered by Suffix("3") + "_testdata/symlinktofolder/subfolder", // <- subfolder skipped by Prefix("sub") + // "_testdata/symlinktofolder/subfolder/file4", + "_testdata/test.txt", + "_testdata/test.txt.gz"}, l.AsStrings()) } From 531a9e181eaaa084b98e4d6994fb102c2ddf0bad Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 4 Feb 2022 17:30:07 +0100 Subject: [PATCH 10/13] Fixing tests for Windows... --- readdir_test.go | 165 +++++++++++++++++++++++------------------------- 1 file changed, 80 insertions(+), 85 deletions(-) diff --git a/readdir_test.go b/readdir_test.go index a25c591..ae0f294 100644 --- a/readdir_test.go +++ b/readdir_test.go @@ -88,105 +88,100 @@ func TestReadDirRecursiveFiltered(t *testing.T) { l, err := testdata.ReadDirRecursiveFiltered(nil) require.NoError(t, err) l.Sort() - require.Equal(t, []string{ - "_testdata/anotherFile", - "_testdata/file", - "_testdata/folder", - "_testdata/folder/.hidden", - "_testdata/folder/file2", - "_testdata/folder/file3", - "_testdata/folder/subfolder", - "_testdata/folder/subfolder/file4", - "_testdata/symlinktofolder", - "_testdata/symlinktofolder/.hidden", - "_testdata/symlinktofolder/file2", - "_testdata/symlinktofolder/file3", - "_testdata/symlinktofolder/subfolder", - "_testdata/symlinktofolder/subfolder/file4", - "_testdata/test.txt", - "_testdata/test.txt.gz"}, l.AsStrings()) + pathEqualsTo(t, "_testdata/anotherFile", l[0]) + pathEqualsTo(t, "_testdata/file", l[1]) + pathEqualsTo(t, "_testdata/folder", l[2]) + pathEqualsTo(t, "_testdata/folder/.hidden", l[3]) + pathEqualsTo(t, "_testdata/folder/file2", l[4]) + pathEqualsTo(t, "_testdata/folder/file3", l[5]) + pathEqualsTo(t, "_testdata/folder/subfolder", l[6]) + pathEqualsTo(t, "_testdata/folder/subfolder/file4", l[7]) + pathEqualsTo(t, "_testdata/symlinktofolder", l[8]) + pathEqualsTo(t, "_testdata/symlinktofolder/.hidden", l[9]) + pathEqualsTo(t, "_testdata/symlinktofolder/file2", l[10]) + pathEqualsTo(t, "_testdata/symlinktofolder/file3", l[11]) + pathEqualsTo(t, "_testdata/symlinktofolder/subfolder", l[12]) + pathEqualsTo(t, "_testdata/symlinktofolder/subfolder/file4", l[13]) + pathEqualsTo(t, "_testdata/test.txt", l[14]) + pathEqualsTo(t, "_testdata/test.txt.gz", l[15]) l, err = testdata.ReadDirRecursiveFiltered(FilterOutDirectories()) require.NoError(t, err) l.Sort() - require.Equal(t, []string{ - "_testdata/anotherFile", - "_testdata/file", - "_testdata/folder", // <- this is listed but not traversed - // "_testdata/folder/.hidden", - // "_testdata/folder/file2", - // "_testdata/folder/file3", - // "_testdata/folder/subfolder", - // "_testdata/folder/subfolder/file4", - "_testdata/symlinktofolder", // <- this is listed but not traversed - // "_testdata/symlinktofolder/.hidden", - // "_testdata/symlinktofolder/file2", - // "_testdata/symlinktofolder/file3", - // "_testdata/symlinktofolder/subfolder", - // "_testdata/symlinktofolder/subfolder/file4", - "_testdata/test.txt", - "_testdata/test.txt.gz"}, l.AsStrings()) + pathEqualsTo(t, "_testdata/anotherFile", l[0]) + pathEqualsTo(t, "_testdata/file", l[1]) + pathEqualsTo(t, "_testdata/folder", l[2]) // <- this is listed but not traversed + // pathEqualsTo(t,"_testdata/folder/.hidden", l[]) + // pathEqualsTo(t,"_testdata/folder/file2", l[]) + // pathEqualsTo(t,"_testdata/folder/file3", l[]) + // pathEqualsTo(t,"_testdata/folder/subfolder", l[]) + // pathEqualsTo(t,"_testdata/folder/subfolder/file4", l[]) + pathEqualsTo(t, "_testdata/symlinktofolder", l[3]) // <- this is listed but not traversed + // pathEqualsTo(t,"_testdata/symlinktofolder/.hidden", l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/file2", l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/file3", l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder", l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder/file4", l[]) + pathEqualsTo(t, "_testdata/test.txt", l[4]) + pathEqualsTo(t, "_testdata/test.txt.gz", l[5]) l, err = testdata.ReadDirRecursiveFiltered(nil, FilterOutDirectories()) require.NoError(t, err) l.Sort() - require.Equal(t, []string{ - "_testdata/anotherFile", - "_testdata/file", - // "_testdata/folder", <- this is filtered but still traversed - "_testdata/folder/.hidden", - "_testdata/folder/file2", - "_testdata/folder/file3", - // "_testdata/folder/subfolder", <- this is filtered but still traversed - "_testdata/folder/subfolder/file4", - // "_testdata/symlinktofolder", <- this is filtered but still traversed - "_testdata/symlinktofolder/.hidden", - "_testdata/symlinktofolder/file2", - "_testdata/symlinktofolder/file3", - // "_testdata/symlinktofolder/subfolder", <- this is filtered but still traversed - "_testdata/symlinktofolder/subfolder/file4", - "_testdata/test.txt", - "_testdata/test.txt.gz"}, l.AsStrings()) + pathEqualsTo(t, "_testdata/anotherFile", l[0]) + pathEqualsTo(t, "_testdata/file", l[1]) + // pathEqualsTo(t,"_testdata/folder", l[]) <- this is filtered but still traversed + pathEqualsTo(t, "_testdata/folder/.hidden", l[2]) + pathEqualsTo(t, "_testdata/folder/file2", l[3]) + pathEqualsTo(t, "_testdata/folder/file3", l[4]) + // pathEqualsTo(t,"_testdata/folder/subfolder", l[]) <- this is filtered but still traversed + pathEqualsTo(t, "_testdata/folder/subfolder/file4", l[5]) + // pathEqualsTo(t,"_testdata/symlinktofolder", l[]) <- this is filtered but still traversed + pathEqualsTo(t, "_testdata/symlinktofolder/.hidden", l[6]) + pathEqualsTo(t, "_testdata/symlinktofolder/file2", l[7]) + pathEqualsTo(t, "_testdata/symlinktofolder/file3", l[8]) + // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder", l[]) <- this is filtered but still traversed + pathEqualsTo(t, "_testdata/symlinktofolder/subfolder/file4", l[9]) + pathEqualsTo(t, "_testdata/test.txt", l[10]) + pathEqualsTo(t, "_testdata/test.txt.gz", l[11]) l, err = testdata.ReadDirRecursiveFiltered(FilterOutDirectories(), FilterOutDirectories()) require.NoError(t, err) l.Sort() - require.Equal(t, []string{ - "_testdata/anotherFile", - "_testdata/file", - // "_testdata/folder", - // "_testdata/folder/.hidden", - // "_testdata/folder/file2", - // "_testdata/folder/file3", - // "_testdata/folder/subfolder", - // "_testdata/folder/subfolder/file4", - // "_testdata/symlinktofolder", - // "_testdata/symlinktofolder/.hidden", - // "_testdata/symlinktofolder/file2", - // "_testdata/symlinktofolder/file3", - // "_testdata/symlinktofolder/subfolder", - // "_testdata/symlinktofolder/subfolder/file4", - "_testdata/test.txt", - "_testdata/test.txt.gz"}, l.AsStrings()) + pathEqualsTo(t, "_testdata/anotherFile", l[0]) + pathEqualsTo(t, "_testdata/file", l[1]) + // pathEqualsTo(t,"_testdata/folder" ,l[]) + // pathEqualsTo(t,"_testdata/folder/.hidden" ,l[]) + // pathEqualsTo(t,"_testdata/folder/file2" ,l[]) + // pathEqualsTo(t,"_testdata/folder/file3" ,l[]) + // pathEqualsTo(t,"_testdata/folder/subfolder" ,l[]) + // pathEqualsTo(t,"_testdata/folder/subfolder/file4" ,l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder" ,l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/.hidden" ,l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/file2" ,l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/file3" ,l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder" ,l[]) + // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder/file4" ,l[]) + pathEqualsTo(t, "_testdata/test.txt", l[2]) + pathEqualsTo(t, "_testdata/test.txt.gz", l[3]) l, err = testdata.ReadDirRecursiveFiltered(FilterOutPrefixes("sub"), FilterOutSuffixes("3")) require.NoError(t, err) l.Sort() - require.Equal(t, []string{ - "_testdata/anotherFile", - "_testdata/file", - "_testdata/folder", - "_testdata/folder/.hidden", - "_testdata/folder/file2", - // "_testdata/folder/file3", <- filtered by Suffix("3") - "_testdata/folder/subfolder", // <- subfolder skipped by Prefix("sub") - // "_testdata/folder/subfolder/file4", - "_testdata/symlinktofolder", - "_testdata/symlinktofolder/.hidden", - "_testdata/symlinktofolder/file2", - // "_testdata/symlinktofolder/file3", <- filtered by Suffix("3") - "_testdata/symlinktofolder/subfolder", // <- subfolder skipped by Prefix("sub") - // "_testdata/symlinktofolder/subfolder/file4", - "_testdata/test.txt", - "_testdata/test.txt.gz"}, l.AsStrings()) + pathEqualsTo(t, "_testdata/anotherFile", l[0]) + pathEqualsTo(t, "_testdata/file", l[1]) + pathEqualsTo(t, "_testdata/folder", l[2]) + pathEqualsTo(t, "_testdata/folder/.hidden", l[3]) + pathEqualsTo(t, "_testdata/folder/file2", l[4]) + // pathEqualsTo(t,"_testdata/folder/file3", l[]) <- filtered by Suffix("3") + pathEqualsTo(t, "_testdata/folder/subfolder", l[5]) // <- subfolder skipped by Prefix("sub") + // pathEqualsTo(t,"_testdata/folder/subfolder/file4",l[]) + pathEqualsTo(t, "_testdata/symlinktofolder", l[6]) + pathEqualsTo(t, "_testdata/symlinktofolder/.hidden", l[7]) + pathEqualsTo(t, "_testdata/symlinktofolder/file2", l[8]) + // pathEqualsTo(t,"_testdata/symlinktofolder/file3", l[]) <- filtered by Suffix("3") + pathEqualsTo(t, "_testdata/symlinktofolder/subfolder", l[9]) // <- subfolder skipped by Prefix("sub") + // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder/file4", l[]) + pathEqualsTo(t, "_testdata/test.txt", l[10]) + pathEqualsTo(t, "_testdata/test.txt.gz", l[11]) } From 8d528721cff798bf407f3c06345f870df6fedb49 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 11 Feb 2022 17:19:41 +0100 Subject: [PATCH 11/13] Added test for ReadDir with filters --- readdir_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/readdir_test.go b/readdir_test.go index ae0f294..30d6053 100644 --- a/readdir_test.go +++ b/readdir_test.go @@ -83,6 +83,33 @@ func TestReadDirRecursiveSymLinkLoop(t *testing.T) { require.Nil(t, l) } +func TestReadDirFiltered(t *testing.T) { + folderPath := New("_testdata/folder") + list, err := folderPath.ReadDir() + require.NoError(t, err) + require.Len(t, list, 4) + pathEqualsTo(t, "_testdata/folder/.hidden", list[0]) + pathEqualsTo(t, "_testdata/folder/file2", list[1]) + pathEqualsTo(t, "_testdata/folder/file3", list[2]) + pathEqualsTo(t, "_testdata/folder/subfolder", list[3]) + + list, err = folderPath.ReadDir(FilterDirectories()) + require.NoError(t, err) + require.Len(t, list, 1) + // pathEqualsTo(t, "_testdata/folder/.hidden", list[]) + // pathEqualsTo(t, "_testdata/folder/file2", list[]) + // pathEqualsTo(t, "_testdata/folder/file3", list[]) + pathEqualsTo(t, "_testdata/folder/subfolder", list[0]) + + list, err = folderPath.ReadDir(FilterOutPrefixes("file")) + require.NoError(t, err) + require.Len(t, list, 2) + pathEqualsTo(t, "_testdata/folder/.hidden", list[0]) + // pathEqualsTo(t, "_testdata/folder/file2", list[]) + // pathEqualsTo(t, "_testdata/folder/file3", list[]) + pathEqualsTo(t, "_testdata/folder/subfolder", list[1]) +} + func TestReadDirRecursiveFiltered(t *testing.T) { testdata := New("_testdata") l, err := testdata.ReadDirRecursiveFiltered(nil) From 4373094d4e16268a6f548b33fb598ec580287de3 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 11 Feb 2022 17:27:52 +0100 Subject: [PATCH 12/13] Use internal anonymous function to simplify alogirthms --- readdir.go | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/readdir.go b/readdir.go index 9833e1d..dc7ab19 100644 --- a/readdir.go +++ b/readdir.go @@ -45,14 +45,21 @@ func (p *Path) ReadDir(filters ...ReadDirFilter) (PathList, error) { if err != nil { return nil, err } + + accept := func(p *Path) bool { + for _, filter := range filters { + if !filter(p) { + return false + } + } + return true + } + paths := PathList{} -fileLoop: for _, info := range infos { path := p.Join(info.Name()) - for _, filter := range filters { - if !filter(path) { - continue fileLoop - } + if !accept(path) { + continue } paths.Add(path) } @@ -98,18 +105,21 @@ func (p *Path) ReadDirRecursiveFiltered(recursionFilter ReadDirFilter, filters . if err != nil { return nil, err } - paths := PathList{} - for _, info := range infos { - path := p.Join(info.Name()) - accept := true + accept := func(p *Path) bool { for _, filter := range filters { - if !filter(path) { - accept = false - break + if !filter(p) { + return false } } - if accept { + return true + } + + paths := PathList{} + for _, info := range infos { + path := p.Join(info.Name()) + + if accept(path) { paths.Add(path) } From 4ad8389fa089e765b7a3fb2a98cfc69cb9b39c86 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 11 Feb 2022 17:39:01 +0100 Subject: [PATCH 13/13] Removed useless comments --- readdir_test.go | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/readdir_test.go b/readdir_test.go index 30d6053..14316e7 100644 --- a/readdir_test.go +++ b/readdir_test.go @@ -96,17 +96,12 @@ func TestReadDirFiltered(t *testing.T) { list, err = folderPath.ReadDir(FilterDirectories()) require.NoError(t, err) require.Len(t, list, 1) - // pathEqualsTo(t, "_testdata/folder/.hidden", list[]) - // pathEqualsTo(t, "_testdata/folder/file2", list[]) - // pathEqualsTo(t, "_testdata/folder/file3", list[]) pathEqualsTo(t, "_testdata/folder/subfolder", list[0]) list, err = folderPath.ReadDir(FilterOutPrefixes("file")) require.NoError(t, err) require.Len(t, list, 2) pathEqualsTo(t, "_testdata/folder/.hidden", list[0]) - // pathEqualsTo(t, "_testdata/folder/file2", list[]) - // pathEqualsTo(t, "_testdata/folder/file3", list[]) pathEqualsTo(t, "_testdata/folder/subfolder", list[1]) } @@ -137,18 +132,8 @@ func TestReadDirRecursiveFiltered(t *testing.T) { l.Sort() pathEqualsTo(t, "_testdata/anotherFile", l[0]) pathEqualsTo(t, "_testdata/file", l[1]) - pathEqualsTo(t, "_testdata/folder", l[2]) // <- this is listed but not traversed - // pathEqualsTo(t,"_testdata/folder/.hidden", l[]) - // pathEqualsTo(t,"_testdata/folder/file2", l[]) - // pathEqualsTo(t,"_testdata/folder/file3", l[]) - // pathEqualsTo(t,"_testdata/folder/subfolder", l[]) - // pathEqualsTo(t,"_testdata/folder/subfolder/file4", l[]) + pathEqualsTo(t, "_testdata/folder", l[2]) // <- this is listed but not traversed pathEqualsTo(t, "_testdata/symlinktofolder", l[3]) // <- this is listed but not traversed - // pathEqualsTo(t,"_testdata/symlinktofolder/.hidden", l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/file2", l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/file3", l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder", l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder/file4", l[]) pathEqualsTo(t, "_testdata/test.txt", l[4]) pathEqualsTo(t, "_testdata/test.txt.gz", l[5]) @@ -157,17 +142,13 @@ func TestReadDirRecursiveFiltered(t *testing.T) { l.Sort() pathEqualsTo(t, "_testdata/anotherFile", l[0]) pathEqualsTo(t, "_testdata/file", l[1]) - // pathEqualsTo(t,"_testdata/folder", l[]) <- this is filtered but still traversed pathEqualsTo(t, "_testdata/folder/.hidden", l[2]) pathEqualsTo(t, "_testdata/folder/file2", l[3]) pathEqualsTo(t, "_testdata/folder/file3", l[4]) - // pathEqualsTo(t,"_testdata/folder/subfolder", l[]) <- this is filtered but still traversed pathEqualsTo(t, "_testdata/folder/subfolder/file4", l[5]) - // pathEqualsTo(t,"_testdata/symlinktofolder", l[]) <- this is filtered but still traversed pathEqualsTo(t, "_testdata/symlinktofolder/.hidden", l[6]) pathEqualsTo(t, "_testdata/symlinktofolder/file2", l[7]) pathEqualsTo(t, "_testdata/symlinktofolder/file3", l[8]) - // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder", l[]) <- this is filtered but still traversed pathEqualsTo(t, "_testdata/symlinktofolder/subfolder/file4", l[9]) pathEqualsTo(t, "_testdata/test.txt", l[10]) pathEqualsTo(t, "_testdata/test.txt.gz", l[11]) @@ -177,18 +158,6 @@ func TestReadDirRecursiveFiltered(t *testing.T) { l.Sort() pathEqualsTo(t, "_testdata/anotherFile", l[0]) pathEqualsTo(t, "_testdata/file", l[1]) - // pathEqualsTo(t,"_testdata/folder" ,l[]) - // pathEqualsTo(t,"_testdata/folder/.hidden" ,l[]) - // pathEqualsTo(t,"_testdata/folder/file2" ,l[]) - // pathEqualsTo(t,"_testdata/folder/file3" ,l[]) - // pathEqualsTo(t,"_testdata/folder/subfolder" ,l[]) - // pathEqualsTo(t,"_testdata/folder/subfolder/file4" ,l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder" ,l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/.hidden" ,l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/file2" ,l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/file3" ,l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder" ,l[]) - // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder/file4" ,l[]) pathEqualsTo(t, "_testdata/test.txt", l[2]) pathEqualsTo(t, "_testdata/test.txt.gz", l[3]) @@ -200,15 +169,11 @@ func TestReadDirRecursiveFiltered(t *testing.T) { pathEqualsTo(t, "_testdata/folder", l[2]) pathEqualsTo(t, "_testdata/folder/.hidden", l[3]) pathEqualsTo(t, "_testdata/folder/file2", l[4]) - // pathEqualsTo(t,"_testdata/folder/file3", l[]) <- filtered by Suffix("3") pathEqualsTo(t, "_testdata/folder/subfolder", l[5]) // <- subfolder skipped by Prefix("sub") - // pathEqualsTo(t,"_testdata/folder/subfolder/file4",l[]) pathEqualsTo(t, "_testdata/symlinktofolder", l[6]) pathEqualsTo(t, "_testdata/symlinktofolder/.hidden", l[7]) pathEqualsTo(t, "_testdata/symlinktofolder/file2", l[8]) - // pathEqualsTo(t,"_testdata/symlinktofolder/file3", l[]) <- filtered by Suffix("3") pathEqualsTo(t, "_testdata/symlinktofolder/subfolder", l[9]) // <- subfolder skipped by Prefix("sub") - // pathEqualsTo(t,"_testdata/symlinktofolder/subfolder/file4", l[]) pathEqualsTo(t, "_testdata/test.txt", l[10]) pathEqualsTo(t, "_testdata/test.txt.gz", l[11]) }