Change core and lib search commands to use fuzzy search

commands/core/search.go

@@ -23,19 +23,16 @@ import (
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
+	"github.com/lithammer/fuzzysearch/fuzzy"
 )
 
-func match(line, searchArgs string) bool {
-	return strings.Contains(strings.ToLower(line), strings.ToLower(searchArgs))
-}
-
-func exactMatch(line, searchArgs string) bool {
-	return strings.Compare(strings.ToLower(line), strings.ToLower(searchArgs)) == 0
-}
+// maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
+// This value is completely arbitrary and picked randomly.
+const maximumSearchDistance = 20
 
 // PlatformSearch FIXMEDOC
 func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error) {
-	searchArgs := req.SearchArgs
+	searchArgs := strings.Trim(req.SearchArgs, " ")
 	allVersions := req.AllVersions
 	pm := commands.GetPackageManager(req.Instance.Id)
 	if pm == nil {
@@ -63,29 +60,54 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error)
 					continue
 				}
 
-				// platform has a valid release, check if it matches the search arguments
-				if match(platform.Name, searchArgs) || match(platform.Architecture, searchArgs) ||
-					exactMatch(platform.String(), searchArgs) || match(targetPackage.Name, searchArgs) ||
-					match(targetPackage.Maintainer, searchArgs) || match(targetPackage.WebsiteURL, searchArgs) {
+				if searchArgs == "" {
 					if allVersions {
 						res = append(res, platform.GetAllReleases()...)
 					} else {
 						res = append(res, platformRelease)
 					}
-				} else {
-					// if we didn't find a match in the platform data, search for
-					// a match in the boards manifest
-					for _, board := range platformRelease.BoardsManifest {
-						if match(board.Name, searchArgs) {
+					continue
+				}
+
+				// Gather all strings that can be used for searching
+				toTest := []string{
+					platform.String(),
+					platform.Name,
+					platform.Architecture,
+					targetPackage.Name,
+					targetPackage.Maintainer,
+					targetPackage.WebsiteURL,
+				}
+				for _, board := range platformRelease.BoardsManifest {
+					toTest = append(toTest, board.Name)
+				}
+
+				// Removes some chars from query strings to enhance results
+				cleanSearchArgs := strings.Map(func(r rune) rune {
+					switch r {
+					case '_':
+					case '-':
+					case ' ':
+						return -1
+					}
+					return r
+				}, searchArgs)
+
+				// Fuzzy search
+				for _, arg := range []string{searchArgs, cleanSearchArgs} {
+					for _, rank := range fuzzy.RankFindNormalizedFold(arg, toTest) {
+						// Accepts only results that close to the searched terms
+						if rank.Distance < maximumSearchDistance {
 							if allVersions {
 								res = append(res, platform.GetAllReleases()...)
 							} else {
 								res = append(res, platformRelease)
 							}
-							break
+							goto nextPlatform
 						}
 					}
 				}
+			nextPlatform:
 			}
 		}
 	}

commands/core/search_test.go

@@ -24,17 +24,9 @@ import (
 	"github.com/arduino/arduino-cli/rpc/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
 	"github.com/arduino/go-paths-helper"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestMatch(t *testing.T) {
-	assert.True(t, match("this is platform Foo", "foo"))
-	assert.True(t, match("this is platform Foo", "FOO"))
-	assert.True(t, match("this is platform Foo", ""))
-	assert.False(t, match("this is platform Foo", "Bar"))
-}
-
 func TestPlatformSearch(t *testing.T) {
 
 	dataDir := paths.TempDir().Join("test", "data_dir")
@@ -238,4 +230,50 @@ func TestPlatformSearch(t *testing.T) {
 			{Name: "Linino One"},
 		},
 	})
+
+	res, err = PlatformSearch(&rpc.PlatformSearchReq{
+		Instance:    inst,
+		SearchArgs:  "yun",
+		AllVersions: true,
+	})
+	require.Nil(t, err)
+	require.NotNil(t, res)
+	require.Len(t, res.SearchOutput, 1)
+	require.Contains(t, res.SearchOutput, &commands.Platform{
+		ID:         "arduino:avr",
+		Installed:  "",
+		Latest:     "1.8.3",
+		Name:       "Arduino AVR Boards",
+		Maintainer: "Arduino",
+		Website:    "https://www.arduino.cc/",
+		Email:      "packages@arduino.cc",
+		Boards: []*commands.Board{
+			{Name: "Arduino Yún"},
+			{Name: "Arduino Uno"},
+			{Name: "Arduino Uno WiFi"},
+			{Name: "Arduino Diecimila"},
+			{Name: "Arduino Nano"},
+			{Name: "Arduino Mega"},
+			{Name: "Arduino MegaADK"},
+			{Name: "Arduino Leonardo"},
+			{Name: "Arduino Leonardo Ethernet"},
+			{Name: "Arduino Micro"},
+			{Name: "Arduino Esplora"},
+			{Name: "Arduino Mini"},
+			{Name: "Arduino Ethernet"},
+			{Name: "Arduino Fio"},
+			{Name: "Arduino BT"},
+			{Name: "Arduino LilyPadUSB"},
+			{Name: "Arduino Lilypad"},
+			{Name: "Arduino Pro"},
+			{Name: "Arduino ATMegaNG"},
+			{Name: "Arduino Robot Control"},
+			{Name: "Arduino Robot Motor"},
+			{Name: "Arduino Gemma"},
+			{Name: "Adafruit Circuit Playground"},
+			{Name: "Arduino Yún Mini"},
+			{Name: "Arduino Industrial 101"},
+			{Name: "Linino One"},
+		},
+	})
 }

commands/lib/search.go

@@ -24,12 +24,10 @@ import (
 	"github.com/arduino/arduino-cli/arduino/libraries/librariesmanager"
 	"github.com/arduino/arduino-cli/commands"
 	rpc "github.com/arduino/arduino-cli/rpc/commands"
-	"github.com/imjasonmiller/godice"
+	"github.com/lithammer/fuzzysearch/fuzzy"
 	semver "go.bug.st/relaxed-semver"
 )
 
-var similarityThreshold = 0.7
-
 // LibrarySearch FIXMEDOC
 func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchReq) (*rpc.LibrarySearchResp, error) {
 	lm := commands.GetLibraryManager(req.GetInstance().GetId())
@@ -41,45 +39,70 @@ func LibrarySearch(ctx context.Context, req *rpc.LibrarySearchReq) (*rpc.Library
 }
 
 func searchLibrary(req *rpc.LibrarySearchReq, lm *librariesmanager.LibrariesManager) (*rpc.LibrarySearchResp, error) {
+	query := req.GetQuery()
 	res := []*rpc.SearchedLibrary{}
 	status := rpc.LibrarySearchStatus_success
 
-	for _, lib := range lm.Index.Libraries {
-		qry := strings.ToLower(req.GetQuery())
-		if strings.Contains(strings.ToLower(lib.Name), qry) ||
-			strings.Contains(strings.ToLower(lib.Latest.Paragraph), qry) ||
-			strings.Contains(strings.ToLower(lib.Latest.Sentence), qry) {
-			releases := map[string]*rpc.LibraryRelease{}
-			for str, rel := range lib.Releases {
-				releases[str] = GetLibraryParameters(rel)
-			}
-			latest := GetLibraryParameters(lib.Latest)
-
-			searchedLib := &rpc.SearchedLibrary{
-				Name:     lib.Name,
-				Releases: releases,
-				Latest:   latest,
-			}
-			res = append(res, searchedLib)
+	// If the query is empty all libraries are returned
+	if strings.Trim(query, " ") == "" {
+		for _, lib := range lm.Index.Libraries {
+			res = append(res, indexLibraryToRPCSearchLibrary(lib))
 		}
+		return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil
 	}
 
-	if len(res) == 0 {
-		status = rpc.LibrarySearchStatus_failed
-		for _, lib := range lm.Index.Libraries {
-			if godice.CompareString(req.GetQuery(), lib.Name) > similarityThreshold {
-				res = append(res, &rpc.SearchedLibrary{
-					Name: lib.Name,
-				})
+	// maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search.
+	// This value is completely arbitrary and picked randomly.
+	maximumSearchDistance := 150
+	// Use a lower distance for shorter query or the user might be flooded with unrelated results
+	if len(query) <= 4 {
+		maximumSearchDistance = 40
+	}
+
+	// Removes some chars from query strings to enhance results
+	cleanQuery := strings.Map(func(r rune) rune {
+		switch r {
+		case '_':
+		case '-':
+		case ' ':
+			return -1
+		}
+		return r
+	}, query)
+	for _, lib := range lm.Index.Libraries {
+		// Use both uncleaned and cleaned query
+		for _, q := range []string{query, cleanQuery} {
+			toTest := []string{lib.Name, lib.Latest.Paragraph, lib.Latest.Sentence}
+			for _, rank := range fuzzy.RankFindNormalizedFold(q, toTest) {
+				if rank.Distance < maximumSearchDistance {
+					res = append(res, indexLibraryToRPCSearchLibrary(lib))
+					goto nextLib
+				}
 			}
 		}
+	nextLib:
 	}
 
 	return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil
 }
 
-// GetLibraryParameters FIXMEDOC
-func GetLibraryParameters(rel *librariesindex.Release) *rpc.LibraryRelease {
+// indexLibraryToRPCSearchLibrary converts a librariindex.Library to rpc.SearchLibrary
+func indexLibraryToRPCSearchLibrary(lib *librariesindex.Library) *rpc.SearchedLibrary {
+	releases := map[string]*rpc.LibraryRelease{}
+	for str, rel := range lib.Releases {
+		releases[str] = getLibraryParameters(rel)
+	}
+	latest := getLibraryParameters(lib.Latest)
+
+	return &rpc.SearchedLibrary{
+		Name:     lib.Name,
+		Releases: releases,
+		Latest:   latest,
+	}
+}
+
+// getLibraryParameters FIXMEDOC
+func getLibraryParameters(rel *librariesindex.Release) *rpc.LibraryRelease {
 	return &rpc.LibraryRelease{
 		Author:           rel.Author,
 		Version:          rel.Version.String(),

commands/lib/search_test.go

@@ -48,7 +48,12 @@ func TestSearchLibrarySimilar(t *testing.T) {
 	}
 
 	assert := assert.New(t)
-	assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_failed)
-	assert.Equal(len(resp.GetLibraries()), 1)
-	assert.Equal(resp.GetLibraries()[0].Name, "Arduino")
+	assert.Equal(resp.GetStatus(), rpc.LibrarySearchStatus_success)
+	assert.Equal(len(resp.GetLibraries()), 2)
+	libs := map[string]*rpc.SearchedLibrary{}
+	for _, l := range resp.GetLibraries() {
+		libs[l.Name] = l
+	}
+	assert.Contains(libs, "ArduinoTestPackage")
+	assert.Contains(libs, "Arduino")
 }

go.mod

@@ -19,10 +19,10 @@ require (
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/golang/protobuf v1.4.2
 	github.com/h2non/filetype v1.0.8 // indirect
-	github.com/imjasonmiller/godice v0.1.2
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/leonelquinteros/gotext v1.4.0
+	github.com/lithammer/fuzzysearch v1.1.1
 	github.com/mattn/go-colorable v0.1.2
 	github.com/mattn/go-isatty v0.0.8
 	github.com/mattn/go-runewidth v0.0.9 // indirect

go.sum

@@ -1,7 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/jt0CW30vsg=
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
@@ -117,8 +116,6 @@ github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14=
 github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/imjasonmiller/godice v0.1.2 h1:T1/sW/HoDzFeuwzOOuQjmeMELz9CzZ53I2CnD+08zD4=
-github.com/imjasonmiller/godice v0.1.2/go.mod h1:8cTkdnVI+NglU2d6sv+ilYcNaJ5VSTBwvMbFULJd/QQ=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
@@ -155,6 +152,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leonelquinteros/gotext v1.4.0 h1:2NHPCto5IoMXbrT0bldPrxj0qM5asOCwtb1aUQZ1tys=
 github.com/leonelquinteros/gotext v1.4.0/go.mod h1:yZGXREmoGTtBvZHNcc+Yfug49G/2spuF/i/Qlsvz1Us=
+github.com/lithammer/fuzzysearch v1.1.1 h1:8F9OAV2xPuYblToVohjanztdnPjbtA0MLgMvDKQ0Z08=
+github.com/lithammer/fuzzysearch v1.1.1/go.mod h1:H2bng+w5gsR7NlfIJM8ElGZI0sX6C/9uzGqicVXGU6c=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -215,7 +214,6 @@ github.com/segmentio/objconv v1.0.1 h1:QjfLzwriJj40JibCV3MGSEiAoXixbp4ybhwfTB8RX
 github.com/segmentio/objconv v1.0.1/go.mod h1:auayaH5k3137Cl4SoXTgrzQcuQDmvuVtZgS0fb1Ahys=
 github.com/segmentio/stats/v4 v4.5.3 h1:Y/DSUWZ4c8ICgqJ9rQohzKvGqGWbLPWad5zmxVoKN+Y=
 github.com/segmentio/stats/v4 v4.5.3/go.mod h1:LsaahUJR7iiSs8mnkvQvdQ/RLHAS5adGLxuntg0ydGo=
-github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
@@ -281,7 +279,6 @@ golang.org/x/crypto v0.0.0-20180214000028-650f4a345ab4/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y=
 golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -334,7 +331,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a h1:mEQZbbaBjWyLNy0tmZmgEuQAR8XOQ3hL8GYi3J/NG64=
 golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=