diff --git a/commands/board/listall.go b/commands/board/listall.go index a3d3eb075a9..ecf25e6e8d7 100644 --- a/commands/board/listall.go +++ b/commands/board/listall.go @@ -20,9 +20,9 @@ import ( "errors" "strings" + "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/commands" - "github.com/lithammer/fuzzysearch/fuzzy" ) // maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search. @@ -36,18 +36,26 @@ func ListAll(ctx context.Context, req *rpc.BoardListAllReq) (*rpc.BoardListAllRe return nil, errors.New("invalid instance") } - searchArgs := strings.Join(req.SearchArgs, " ") + searchArgs := []string{} + for _, s := range req.SearchArgs { + searchArgs = append(searchArgs, strings.Trim(s, " ")) + } - match := func(toTest []string) bool { + match := func(toTest []string) (bool, error) { if len(searchArgs) == 0 { - return true + return true, nil } - for _, rank := range fuzzy.RankFindNormalizedFold(searchArgs, toTest) { - if rank.Distance < maximumSearchDistance { - return true + + for _, t := range toTest { + matches, err := utils.Match(t, searchArgs) + if err != nil { + return false, err + } + if matches { + return matches, nil } } - return false + return false, nil } list := &rpc.BoardListAllResp{Boards: []*rpc.BoardListItem{}} @@ -90,10 +98,11 @@ func ListAll(ctx context.Context, req *rpc.BoardListAllReq) (*rpc.BoardListAllRe continue } - toTest := toTest - toTest = append(toTest, strings.Split(board.Name(), " ")...) + toTest := append(toTest, board.Name()) toTest = append(toTest, board.FQBN()) - if !match(toTest) { + if ok, err := match(toTest); err != nil { + return nil, err + } else if !ok { continue } diff --git a/commands/core/search.go b/commands/core/search.go index 46b55e548ca..b12c10d3b05 100644 --- a/commands/core/search.go +++ b/commands/core/search.go @@ -21,9 +21,9 @@ import ( "strings" "github.com/arduino/arduino-cli/arduino/cores" + "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/commands" - "github.com/lithammer/fuzzysearch/fuzzy" ) // maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search. @@ -44,6 +44,26 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error) vid, pid := searchArgs[:4], searchArgs[5:] res = pm.FindPlatformReleaseProvidingBoardsWithVidPid(vid, pid) } else { + + searchArgs := strings.Split(searchArgs, " ") + + match := func(toTest []string) (bool, error) { + if len(searchArgs) == 0 { + return true, nil + } + + for _, t := range toTest { + matches, err := utils.Match(t, searchArgs) + if err != nil { + return false, err + } + if matches { + return matches, nil + } + } + return false, nil + } + for _, targetPackage := range pm.Packages { for _, platform := range targetPackage.Platforms { // discard invalid platforms @@ -60,15 +80,6 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error) continue } - if searchArgs == "" { - if allVersions { - res = append(res, platform.GetAllReleases()...) - } else { - res = append(res, platformRelease) - } - continue - } - // Gather all strings that can be used for searching toTest := []string{ platform.String(), @@ -82,32 +93,18 @@ func PlatformSearch(req *rpc.PlatformSearchReq) (*rpc.PlatformSearchResp, error) toTest = append(toTest, board.Name) } - // Removes some chars from query strings to enhance results - cleanSearchArgs := strings.Map(func(r rune) rune { - switch r { - case '_': - case '-': - case ' ': - return -1 - } - return r - }, searchArgs) + // Search + if ok, err := match(toTest); err != nil { + return nil, err + } else if !ok { + continue + } - // Fuzzy search - for _, arg := range []string{searchArgs, cleanSearchArgs} { - for _, rank := range fuzzy.RankFindNormalizedFold(arg, toTest) { - // Accepts only results that close to the searched terms - if rank.Distance < maximumSearchDistance { - if allVersions { - res = append(res, platform.GetAllReleases()...) - } else { - res = append(res, platformRelease) - } - goto nextPlatform - } - } + if allVersions { + res = append(res, platform.GetAllReleases()...) + } else { + res = append(res, platformRelease) } - nextPlatform: } } } diff --git a/commands/lib/search.go b/commands/lib/search.go index 5bb55063f10..376bab9b21f 100644 --- a/commands/lib/search.go +++ b/commands/lib/search.go @@ -22,9 +22,9 @@ import ( "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" + "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/commands" - "github.com/lithammer/fuzzysearch/fuzzy" semver "go.bug.st/relaxed-semver" ) @@ -43,44 +43,33 @@ func searchLibrary(req *rpc.LibrarySearchReq, lm *librariesmanager.LibrariesMana res := []*rpc.SearchedLibrary{} status := rpc.LibrarySearchStatus_success - // If the query is empty all libraries are returned - if strings.Trim(query, " ") == "" { - for _, lib := range lm.Index.Libraries { - res = append(res, indexLibraryToRPCSearchLibrary(lib)) + searchArgs := strings.Split(strings.Trim(query, " "), " ") + + match := func(toTest []string) (bool, error) { + if len(searchArgs) == 0 { + return true, nil } - return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil - } - // maximumSearchDistance is the maximum Levenshtein distance accepted when using fuzzy search. - // This value is completely arbitrary and picked randomly. - maximumSearchDistance := 150 - // Use a lower distance for shorter query or the user might be flooded with unrelated results - if len(query) <= 4 { - maximumSearchDistance = 40 + for _, t := range toTest { + matches, err := utils.Match(t, searchArgs) + if err != nil { + return false, err + } + if matches { + return matches, nil + } + } + return false, nil } - // Removes some chars from query strings to enhance results - cleanQuery := strings.Map(func(r rune) rune { - switch r { - case '_': - case '-': - case ' ': - return -1 - } - return r - }, query) for _, lib := range lm.Index.Libraries { - // Use both uncleaned and cleaned query - for _, q := range []string{query, cleanQuery} { - toTest := []string{lib.Name, lib.Latest.Paragraph, lib.Latest.Sentence} - for _, rank := range fuzzy.RankFindNormalizedFold(q, toTest) { - if rank.Distance < maximumSearchDistance { - res = append(res, indexLibraryToRPCSearchLibrary(lib)) - goto nextLib - } - } + toTest := []string{lib.Name, lib.Latest.Paragraph, lib.Latest.Sentence} + if ok, err := match(toTest); err != nil { + return nil, err + } else if !ok { + continue } - nextLib: + res = append(res, indexLibraryToRPCSearchLibrary(lib)) } return &rpc.LibrarySearchResp{Libraries: res, Status: status}, nil diff --git a/commands/lib/search_test.go b/commands/lib/search_test.go index 2fd8088d867..c0191229463 100644 --- a/commands/lib/search_test.go +++ b/commands/lib/search_test.go @@ -39,7 +39,7 @@ func TestSearchLibrarySimilar(t *testing.T) { req := &rpc.LibrarySearchReq{ Instance: &rpc.Instance{Id: 1}, - Query: "ardino", + Query: "arduino", } resp, err := searchLibrary(req, lm) diff --git a/go.mod b/go.mod index ad259be746e..d9e7e87844b 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect github.com/kr/text v0.2.0 // indirect github.com/leonelquinteros/gotext v1.4.0 - github.com/lithammer/fuzzysearch v1.1.1 github.com/marcinbor85/gohex v0.0.0-20210308104911-55fb1c624d84 github.com/mattn/go-colorable v0.1.2 github.com/mattn/go-isatty v0.0.8 diff --git a/go.sum b/go.sum index 8335ed4bb9d..232f81baed5 100644 --- a/go.sum +++ b/go.sum @@ -150,8 +150,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leonelquinteros/gotext v1.4.0 h1:2NHPCto5IoMXbrT0bldPrxj0qM5asOCwtb1aUQZ1tys= github.com/leonelquinteros/gotext v1.4.0/go.mod h1:yZGXREmoGTtBvZHNcc+Yfug49G/2spuF/i/Qlsvz1Us= -github.com/lithammer/fuzzysearch v1.1.1 h1:8F9OAV2xPuYblToVohjanztdnPjbtA0MLgMvDKQ0Z08= -github.com/lithammer/fuzzysearch v1.1.1/go.mod h1:H2bng+w5gsR7NlfIJM8ElGZI0sX6C/9uzGqicVXGU6c= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= diff --git a/test/test_board.py b/test/test_board.py index 526b5453214..32c7152e783 100644 --- a/test/test_board.py +++ b/test/test_board.py @@ -455,34 +455,6 @@ def test_board_listall_with_manually_installed_platform(run_command, data_dir): assert "Arduino SAMD (32-bits ARM Cortex-M0+) Boards" == platform["Name"] -def test_board_listall_fuzzy_search(run_command, data_dir): - assert run_command("update") - - # Install from platform manager - assert run_command("core install arduino:avr@1.8.3") - - # Manually installs a core in sketchbooks hardware folder - git_url = "https://github.com/arduino/ArduinoCore-samd.git" - repo_dir = Path(data_dir, "hardware", "arduino-beta-development", "samd") - assert Repo.clone_from(git_url, repo_dir, multi_options=["-b 1.8.11"]) - - res = run_command("board listall --format json samd") - assert res.ok - data = json.loads(res.stdout) - boards = {b["FQBN"]: b for b in data["boards"]} - assert len(boards) == 17 - assert "arduino-beta-development:samd:mkr1000" in boards - assert "arduino:avr:uno" not in boards - - res = run_command("board listall --format json avr") - assert res.ok - data = json.loads(res.stdout) - boards = {b["FQBN"]: b for b in data["boards"]} - assert len(boards) == 26 - assert "arduino:avr:uno" in boards - assert "arduino-beta-development:samd:mkr1000" not in boards - - def test_board_details(run_command): run_command("core update-index") # Download samd core pinned to 1.8.6 diff --git a/test/test_core.py b/test/test_core.py index e62aaa881c4..e046f902330 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -76,12 +76,33 @@ def get_platforms(stdout): assert "1.0.5" in platforms["Retrokits-RK002:arm"] assert "1.0.6" in platforms["Retrokits-RK002:arm"] - # Search using a board name + # Search using board names result = run_command(f"core search myboard --all --additional-urls={url} --format json") assert result.ok platforms = get_platforms(result.stdout) assert "1.2.3" in platforms["Package:x86"] + def run_search(search_args, expected_ids): + res = run_command(f"core search --format json {search_args}") + assert res.ok + data = json.loads(res.stdout) + platform_ids = [p["ID"] for p in data] + for platform_id in expected_ids: + assert platform_id in platform_ids + + run_search("mkr1000", ["arduino:samd"]) + run_search("mkr 1000", ["arduino:samd"]) + + run_search("yún", ["arduino:avr"]) + run_search("yùn", ["arduino:avr"]) + run_search("yun", ["arduino:avr"]) + + run_search("nano", ["arduino:avr", "arduino:megaavr", "arduino:samd", "arduino:mbed"]) + run_search("nano 33", ["arduino:samd", "arduino:mbed"]) + run_search("nano ble", ["arduino:mbed"]) + run_search("ble", ["arduino:mbed"]) + run_search("ble nano", ["arduino:mbed"]) + def test_core_search_no_args(run_command, httpserver): """ @@ -146,32 +167,6 @@ def test_core_search_no_args(run_command, httpserver): assert len(platforms) == num_platforms -def test_core_search_fuzzy(run_command): - assert run_command("update") - - def run_fuzzy_search(search_args, expected_ids): - res = run_command(f"core search --format json {search_args}") - assert res.ok - data = json.loads(res.stdout) - platform_ids = [p["ID"] for p in data] - for platform_id in expected_ids: - assert platform_id in platform_ids - - run_fuzzy_search("mkr1000", ["arduino:samd"]) - run_fuzzy_search("mkr 1000", ["arduino:samd"]) - - run_fuzzy_search("yún", ["arduino:avr"]) - run_fuzzy_search("yùn", ["arduino:avr"]) - run_fuzzy_search("yun", ["arduino:avr"]) - - run_fuzzy_search("nano", ["arduino:avr", "arduino:megaavr", "arduino:samd", "arduino:mbed"]) - run_fuzzy_search("nano33", ["arduino:samd", "arduino:mbed"]) - run_fuzzy_search("nano 33", ["arduino:samd", "arduino:mbed"]) - run_fuzzy_search("nano ble", ["arduino:mbed"]) - run_fuzzy_search("ble", ["arduino:mbed"]) - run_fuzzy_search("ble nano", []) - - def test_core_updateindex_url_not_found(run_command, httpserver): assert run_command("core update-index") diff --git a/test/test_lib.py b/test/test_lib.py index 873bf66d831..7264c09500f 100644 --- a/test/test_lib.py +++ b/test/test_lib.py @@ -414,7 +414,7 @@ def test_lib_ops_caseinsensitive(run_command): def test_search(run_command): - assert run_command("lib update-index") + assert run_command("update") result = run_command("lib search --names") assert result.ok @@ -433,29 +433,6 @@ def test_search(run_command): result = run_command("lib search --names") assert result.ok - # Search for a specific target - result = run_command("lib search --names ArduinoJson --format json") - assert result.ok - libs_json = json.loads(result.stdout) - assert len(libs_json.get("libraries")) >= 1 - - -def test_search_paragraph(run_command): - """ - Search for a string that's only present in the `paragraph` field - within the index file. - """ - assert run_command("lib update-index") - result = run_command('lib search "A simple and efficient JSON library" --names --format json') - assert result.ok - data = json.loads(result.stdout) - libraries = [l["name"] for l in data["libraries"]] - assert "ArduinoJson" in libraries - - -def test_lib_search_fuzzy(run_command): - run_command("update") - def run_search(search_args, expected_libraries): res = run_command(f"lib search --names --format json {search_args}") assert res.ok @@ -468,7 +445,6 @@ def run_search(search_args, expected_libraries): run_search("Arduino mkr iot carrier", ["Arduino_MKRIoTCarrier"]) run_search("mkr iot carrier", ["Arduino_MKRIoTCarrier"]) run_search("mkriotcarrier", ["Arduino_MKRIoTCarrier"]) - run_search("Arduinomkriotcarrier", ["Arduino_MKRIoTCarrier"]) run_search( "dht", @@ -481,10 +457,23 @@ def run_search(search_args, expected_libraries): run_search("sensor dht", []) run_search("arduino json", ["ArduinoJson", "Arduino_JSON"]) - run_search("arduinojson", ["ArduinoJson", "Arduino_JSON"]) + run_search("arduinojson", ["ArduinoJson"]) run_search("json", ["ArduinoJson", "Arduino_JSON"]) +def test_search_paragraph(run_command): + """ + Search for a string that's only present in the `paragraph` field + within the index file. + """ + assert run_command("lib update-index") + result = run_command('lib search "A simple and efficient JSON library" --names --format json') + assert result.ok + data = json.loads(result.stdout) + libraries = [l["name"] for l in data["libraries"]] + assert "ArduinoJson" in libraries + + def test_lib_list_with_updatable_flag(run_command): # Init the environment explicitly run_command("lib update-index")