Fix library prioritization

Author: silvanocerza

arduino/libraries/libraries_location.go

@@ -35,6 +35,9 @@ const (
 	ReferencedPlatformBuiltIn
 	// User are user installed libraries
 	User
+	// Unmanaged is for libraries set manually by the user in the CLI command or from the gRPC function.
+	// Ideally it's used for `libraries` outside folders managed by the CLI.
+	Unmanaged
 )
 
 func (d *LibraryLocation) String() string {
@@ -47,6 +50,8 @@ func (d *LibraryLocation) String() string {
 		return "ref-platform"
 	case User:
 		return "user"
+	case Unmanaged:
+		return "unmanaged"
 	}
 	panic(fmt.Sprintf("invalid LibraryLocation value %d", *d))
 }
@@ -62,6 +67,8 @@ func (d *LibraryLocation) MarshalJSON() ([]byte, error) {
 		return json.Marshal("ref-platform")
 	case User:
 		return json.Marshal("user")
+	case Unmanaged:
+		return json.Marshal("unmanaged")
 	}
 	return nil, fmt.Errorf("invalid library location value: %d", *d)
 }
@@ -81,6 +88,8 @@ func (d *LibraryLocation) UnmarshalJSON(b []byte) error {
 		*d = ReferencedPlatformBuiltIn
 	case "user":
 		*d = User
+	case "unmanaged":
+		*d = Unmanaged
 	}
 	return fmt.Errorf("invalid library location: %s", s)
 }
@@ -96,6 +105,8 @@ func (d *LibraryLocation) ToRPCLibraryLocation() rpc.LibraryLocation {
 		return rpc.LibraryLocation_LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN
 	case User:
 		return rpc.LibraryLocation_LIBRARY_LOCATION_USER
+	case Unmanaged:
+		return rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED
 	}
 	panic(fmt.Sprintf("invalid LibraryLocation value %d", *d))
 }
@@ -111,6 +122,8 @@ func FromRPCLibraryLocation(l rpc.LibraryLocation) LibraryLocation {
 		return ReferencedPlatformBuiltIn
 	case rpc.LibraryLocation_LIBRARY_LOCATION_USER:
 		return User
+	case rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED:
+		return Unmanaged
 	}
 	panic(fmt.Sprintf("invalid rpc.LibraryLocation value %d", l))
 }

arduino/libraries/librariesresolver/cpp.go

@@ -158,6 +158,8 @@ func computePriority(lib *libraries.Library, header, arch string) int {
 		priority += 2
 	case libraries.User:
 		priority += 3
+	case libraries.Unmanaged:
+		priority += 4
 	default:
 		panic(fmt.Sprintf("Invalid library location: %d", lib.Location))
 	}

cli/compile/compile.go

@@ -98,7 +98,7 @@ func NewCommand() *cobra.Command {
 	command.Flags().BoolVarP(&verify, "verify", "t", false, "Verify uploaded binary after the upload.")
 	command.Flags().StringVar(&vidPid, "vid-pid", "", "When specified, VID/PID specific build properties are used, if board supports them.")
 	command.Flags().StringSliceVar(&library, "library", []string{},
-		"List of paths to libraries root folders. Can be used multiple times for different libraries.")
+		"List of paths to libraries root folders. Libraries set this way have top priority in case of conflicts. Can be used multiple times for different libraries.")
 	command.Flags().StringSliceVar(&libraries, "libraries", []string{},
 		"List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries dir paths.")
 	command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debugging, rather than for release.")

docs/sketch-build-process.md

@@ -102,6 +102,8 @@ The "folder name priority" is determined as follows (in order of highest to lowe
 
 The "location priority" is determined as follows (in order of highest to lowest priority):
 
+1. The library is specified using the [`--library` option](commands/arduino-cli_compile.md#options) of
+   `arduino-cli compile`
 1. The library is under a custom libraries path specified via the
    [`--libraries` option](commands/arduino-cli_compile.md#options) of `arduino-cli compile` (in decreasing order of
    priority when multiple custom paths are defined)

legacy/builder/libraries_loader.go

@@ -62,7 +62,8 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error {
 	}
 
 	for _, dir := range ctx.LibrariesDirs {
-		if err := lm.LoadLibraryFromDir(dir, libraries.User); err != nil {
+		// Libraries specified this way have top priority
+		if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil {
 			return err
 		}
 	}

rpc/cc/arduino/cli/commands/v1/lib.pb.go

@@ -297,6 +297,8 @@ enum LibraryLocation {
   // this indicates the library is in the `libraries` subdirectory of a
   // platform referenced by the board's platform.
   LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN = 3;
+  // Outside the `libraries` folders managed by the CLI.
+  LIBRARY_LOCATION_UNMANAGED = 4;
 }
 
 message ZipLibraryInstallRequest {

rpc/cc/arduino/cli/commands/v1/lib.proto

@@ -928,24 +928,30 @@ def test_compile_with_library_priority(run_command, data_dir):
 
     # Manually installs the same library and add a new custom header to it
     git_url = "https://github.com/arduino-libraries/WiFi101.git"
-    lib_path = Path(data_dir, "my-libraries", "WiFi101")
-    assert Repo.clone_from(git_url, lib_path, multi_options=["-b 0.16.1"])
-    with open(lib_path / "src" / "WiFiSomething.h", "x") as f:
-        f.writelines(["#ifndef WIFI_SOMETHING\n", "#define WIFI_SOMETHING\n", "#endif\n"])
+    manually_install_lib_path = Path(data_dir, "my-libraries", "WiFi101")
+    assert Repo.clone_from(git_url, manually_install_lib_path, multi_options=["-b 0.16.1"])
+    # with open(manually_install_lib_path / "src" / "WiFiSomething.h", "x") as f:
+    #     f.writelines(["#ifndef WIFI_SOMETHING\n", "#define WIFI_SOMETHING\n", "#endif\n"])
 
     # Install the same library we installed manually
     assert run_command("lib install WiFi101")
 
-    # Create new sketch and add custom header include
+    # Create new sketch and add library include
     assert run_command(f"sketch new {sketch_path}")
     sketch_file = sketch_path / f"{sketch_name}.ino"
     lines = []
     with open(sketch_file, "r") as f:
         lines = f.readlines()
-    lines = ["#include <WiFiSomething.h>"] + lines
+    lines = ["#include <WiFi101.h>"] + lines
     with open(sketch_file, "w") as f:
         f.writelines(lines)
 
-    res = run_command(f"compile -b {fqbn} {sketch_path} --library {lib_path} -v")
+    res = run_command(f"compile -b {fqbn} {sketch_path} --library {manually_install_lib_path} -v")
     assert res.ok
-    assert "WiFi101" in res.stdout
+    cli_installed_lib_path = Path(data_dir, "libraries", "WiFi101")
+    expected_output = [
+        'Multiple libraries were found for "WiFi101.h"',
+        f" Used: {manually_install_lib_path}",
+        f" Not used: {cli_installed_lib_path}",
+    ]
+    assert "\n".join(expected_output) in res.stdout