From 23eec2c763f97026528e8c82e91bd3f2b6190ecd Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Wed, 7 Apr 2021 17:14:57 +0200 Subject: [PATCH 1/5] Add --library flag to compile command --- .../librariesmanager/librariesmanager.go | 23 +++++ cli/compile/compile.go | 11 ++- commands/compile/compile.go | 2 + legacy/builder/libraries_loader.go | 6 ++ legacy/builder/types/context.go | 1 + rpc/cc/arduino/cli/commands/v1/board.pb.go | 2 +- rpc/cc/arduino/cli/commands/v1/commands.pb.go | 2 +- rpc/cc/arduino/cli/commands/v1/common.pb.go | 2 +- rpc/cc/arduino/cli/commands/v1/compile.pb.go | 93 +++++++++++-------- rpc/cc/arduino/cli/commands/v1/compile.proto | 4 +- rpc/cc/arduino/cli/commands/v1/core.pb.go | 2 +- rpc/cc/arduino/cli/commands/v1/lib.pb.go | 2 +- rpc/cc/arduino/cli/commands/v1/upload.pb.go | 2 +- rpc/cc/arduino/cli/debug/v1/debug.pb.go | 2 +- rpc/cc/arduino/cli/monitor/v1/monitor.pb.go | 10 +- rpc/cc/arduino/cli/settings/v1/settings.pb.go | 2 +- test/test_compile.py | 62 +++++++++++++ 17 files changed, 171 insertions(+), 57 deletions(-) diff --git a/arduino/libraries/librariesmanager/librariesmanager.go b/arduino/libraries/librariesmanager/librariesmanager.go index b8d170f6820..e01c8d681ae 100644 --- a/arduino/libraries/librariesmanager/librariesmanager.go +++ b/arduino/libraries/librariesmanager/librariesmanager.go @@ -204,6 +204,29 @@ func (sc *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) err return nil } +// LoadLibraryFromDir loads one single library from the libRootDir. +// libRootDir must point to the root of a valid library. +// An error is returned if the path doesn't exist or loading of the library fails. +func (lm *LibrariesManager) LoadLibraryFromDir(libRootDir *paths.Path, location libraries.LibraryLocation) error { + if libRootDir.NotExist() { + return fmt.Errorf("library path does not exist: %s", libRootDir) + } + + library, err := libraries.Load(libRootDir, location) + if err != nil { + return fmt.Errorf("loading library from %s: %s", libRootDir, err) + } + + alternatives, ok := lm.Libraries[library.Name] + if !ok { + alternatives = &LibraryAlternatives{} + lm.Libraries[library.Name] = alternatives + } + alternatives.Add(library) + + return nil +} + // FindByReference return the installed library matching the Reference // name and version or, if the version is nil, the library installed // in the User folder. diff --git a/cli/compile/compile.go b/cli/compile/compile.go index d6d0566f052..4a14908b0c6 100644 --- a/cli/compile/compile.go +++ b/cli/compile/compile.go @@ -51,12 +51,16 @@ var ( port string // Upload port, e.g.: COM10 or /dev/ttyACM0. verify bool // Upload, verify uploaded binary after the upload. exportDir string // The compiled binary is written to this file - libraries []string // List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths. optimizeForDebug bool // Optimize compile output for debug, not for release programmer string // Use the specified programmer to upload clean bool // Cleanup the build folder and do not use any cached build compilationDatabaseOnly bool // Only create compilation database without actually compiling sourceOverrides string // Path to a .json file that contains a set of replacements of the sketch source code. + // library and libraries sound similar but they're actually different. + // library expects a path to the root folder of one single library. + // libraries expects a path to a directory containing multiple libraries, similarly to the /libraries path. + library []string // List of paths to libraries root folders. Can be used multiple times for different libraries + libraries []string // List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries paths. ) // NewCommand created a new `compile` command @@ -93,8 +97,10 @@ func NewCommand() *cobra.Command { command.Flags().StringVarP(&port, "port", "p", "", "Upload port, e.g.: COM10 or /dev/ttyACM0") command.Flags().BoolVarP(&verify, "verify", "t", false, "Verify uploaded binary after the upload.") command.Flags().StringVar(&vidPid, "vid-pid", "", "When specified, VID/PID specific build properties are used, if board supports them.") + command.Flags().StringSliceVar(&library, "library", []string{}, + "List of paths to libraries root folders. Can be used multiple times for different libraries.") command.Flags().StringSliceVar(&libraries, "libraries", []string{}, - "List of custom libraries paths separated by commas. Or can be used multiple times for multiple libraries paths.") + "List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries dir paths.") command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debugging, rather than for release.") command.Flags().StringVarP(&programmer, "programmer", "P", "", "Optional, use the specified programmer to upload.") command.Flags().BoolVar(&compilationDatabaseOnly, "only-compilation-database", false, "Just produce the compilation database, without actually compiling.") @@ -171,6 +177,7 @@ func run(cmd *cobra.Command, args []string) { Clean: clean, CreateCompilationDatabaseOnly: compilationDatabaseOnly, SourceOverride: overrides, + Library: library, } compileOut := new(bytes.Buffer) compileErr := new(bytes.Buffer) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 03bba9a4926..b3964d58fc8 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -137,6 +137,8 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.OtherLibrariesDirs = paths.NewPathList(req.GetLibraries()...) builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings)) + builderCtx.LibrariesDirs = paths.NewPathList(req.Library...) + if req.GetBuildPath() == "" { builderCtx.BuildPath = bldr.GenBuildPath(sketch.FullPath) } else { diff --git a/legacy/builder/libraries_loader.go b/legacy/builder/libraries_loader.go index 5abcdec5451..abbbeb8c797 100644 --- a/legacy/builder/libraries_loader.go +++ b/legacy/builder/libraries_loader.go @@ -61,6 +61,12 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error { return errors.WithStack(err) } + for _, dir := range ctx.LibrariesDirs { + if err := lm.LoadLibraryFromDir(dir, libraries.User); err != nil { + return err + } + } + if debugLevel > 0 { for _, lib := range lm.Libraries { for _, libAlt := range lib.Alternatives { diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 1aa99dc8a52..e32b5b44105 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -67,6 +67,7 @@ type Context struct { BuiltInToolsDirs paths.PathList BuiltInLibrariesDirs paths.PathList OtherLibrariesDirs paths.PathList + LibrariesDirs paths.PathList // List of paths pointing to individual libraries root folders SketchLocation *paths.Path WatchedLocations paths.PathList ArduinoAPIVersion string diff --git a/rpc/cc/arduino/cli/commands/v1/board.pb.go b/rpc/cc/arduino/cli/commands/v1/board.pb.go index 854de42c301..edc6aa13182 100644 --- a/rpc/cc/arduino/cli/commands/v1/board.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/board.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/commands/v1/board.proto package commands diff --git a/rpc/cc/arduino/cli/commands/v1/commands.pb.go b/rpc/cc/arduino/cli/commands/v1/commands.pb.go index 28d4382cbf7..73c2b099858 100644 --- a/rpc/cc/arduino/cli/commands/v1/commands.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/commands.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/commands/v1/commands.proto package commands diff --git a/rpc/cc/arduino/cli/commands/v1/common.pb.go b/rpc/cc/arduino/cli/commands/v1/common.pb.go index de08ee9aec4..1d957e126b3 100644 --- a/rpc/cc/arduino/cli/commands/v1/common.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/common.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/commands/v1/common.proto package commands diff --git a/rpc/cc/arduino/cli/commands/v1/compile.pb.go b/rpc/cc/arduino/cli/commands/v1/compile.pb.go index 8bc8d5307c4..275223759f4 100644 --- a/rpc/cc/arduino/cli/commands/v1/compile.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/compile.pb.go @@ -16,16 +16,16 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/commands/v1/compile.proto package commands import ( proto "github.com/golang/protobuf/proto" - wrappers "github.com/golang/protobuf/ptypes/wrappers" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + wrapperspb "google.golang.org/protobuf/types/known/wrapperspb" reflect "reflect" sync "sync" ) @@ -79,7 +79,7 @@ type CompileRequest struct { // If jobs is set to 0, it will use the number of available CPUs as the // maximum. Jobs int32 `protobuf:"varint,14,opt,name=jobs,proto3" json:"jobs,omitempty"` - // List of custom libraries paths separated by commas. + // List of custom libraries dir paths. Libraries []string `protobuf:"bytes,15,rep,name=libraries,proto3" json:"libraries,omitempty"` // Optimize compile output for debug, not for release. OptimizeForDebug bool `protobuf:"varint,16,opt,name=optimize_for_debug,json=optimizeForDebug,proto3" json:"optimize_for_debug,omitempty"` @@ -99,7 +99,9 @@ type CompileRequest struct { SourceOverride map[string]string `protobuf:"bytes,22,rep,name=source_override,json=sourceOverride,proto3" json:"source_override,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` // When set to `true` the compiled binary will be copied to the export // directory. - ExportBinaries *wrappers.BoolValue `protobuf:"bytes,23,opt,name=export_binaries,json=exportBinaries,proto3" json:"export_binaries,omitempty"` + ExportBinaries *wrapperspb.BoolValue `protobuf:"bytes,23,opt,name=export_binaries,json=exportBinaries,proto3" json:"export_binaries,omitempty"` + // List of paths to libraries root folders + Library []string `protobuf:"bytes,24,rep,name=library,proto3" json:"library,omitempty"` } func (x *CompileRequest) Reset() { @@ -267,13 +269,20 @@ func (x *CompileRequest) GetSourceOverride() map[string]string { return nil } -func (x *CompileRequest) GetExportBinaries() *wrappers.BoolValue { +func (x *CompileRequest) GetExportBinaries() *wrapperspb.BoolValue { if x != nil { return x.ExportBinaries } return nil } +func (x *CompileRequest) GetLibrary() []string { + if x != nil { + return x.Library + } + return nil +} + type CompileResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -435,7 +444,7 @@ var file_cc_arduino_cli_commands_v1_compile_proto_rawDesc = []byte{ 0x76, 0x31, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x24, 0x63, 0x63, 0x2f, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x2f, 0x76, 0x31, 0x2f, 0x6c, 0x69, 0x62, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf8, 0x06, 0x0a, 0x0e, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x92, 0x07, 0x0a, 0x0e, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x63, 0x63, 0x2e, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2e, 0x63, 0x6c, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, @@ -486,41 +495,43 @@ var file_cc_arduino_cli_commands_v1_compile_proto_rawDesc = []byte{ 0x72, 0x74, 0x5f, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x17, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x42, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0e, 0x65, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x1a, 0x41, 0x0a, - 0x13, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0xa7, 0x02, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x65, 0x72, 0x72, 0x53, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x70, 0x61, 0x74, 0x68, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x61, 0x74, - 0x68, 0x12, 0x4a, 0x0a, 0x0e, 0x75, 0x73, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, - 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x63, 0x2e, 0x61, + 0x78, 0x70, 0x6f, 0x72, 0x74, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x18, 0x0a, + 0x07, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x18, 0x18, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, + 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x1a, 0x41, 0x0a, 0x13, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa7, 0x02, 0x0a, 0x0f, 0x43, + 0x6f, 0x6d, 0x70, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, + 0x0a, 0x0a, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x09, 0x6f, 0x75, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, + 0x0a, 0x65, 0x72, 0x72, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x09, 0x65, 0x72, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x50, 0x61, 0x74, 0x68, 0x12, 0x4a, 0x0a, 0x0e, 0x75, + 0x73, 0x65, 0x64, 0x5f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x63, 0x63, 0x2e, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, + 0x2e, 0x63, 0x6c, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x2e, 0x76, 0x31, + 0x2e, 0x4c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x52, 0x0d, 0x75, 0x73, 0x65, 0x64, 0x4c, 0x69, + 0x62, 0x72, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x6b, 0x0a, 0x18, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x73, + 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x31, 0x2e, 0x63, 0x63, 0x2e, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2e, 0x63, 0x6c, 0x69, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, - 0x6e, 0x64, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x52, 0x0d, - 0x75, 0x73, 0x65, 0x64, 0x4c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x69, 0x65, 0x73, 0x12, 0x6b, 0x0a, - 0x18, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x73, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x31, 0x2e, 0x63, 0x63, 0x2e, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2e, 0x63, 0x6c, 0x69, - 0x2e, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x69, - 0x7a, 0x65, 0x52, 0x16, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x5a, 0x0a, 0x15, 0x45, 0x78, - 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6d, - 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, - 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, 0x42, 0x48, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2f, 0x61, 0x72, 0x64, - 0x75, 0x69, 0x6e, 0x6f, 0x2d, 0x63, 0x6c, 0x69, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x63, 0x2f, - 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, - 0x61, 0x6e, 0x64, 0x73, 0x2f, 0x76, 0x31, 0x3b, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x64, 0x73, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x52, 0x16, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x53, 0x69, 0x7a, 0x65, 0x22, 0x5a, 0x0a, 0x15, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x5f, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x6d, 0x61, 0x78, 0x53, 0x69, 0x7a, 0x65, + 0x42, 0x48, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, + 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2f, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2d, 0x63, + 0x6c, 0x69, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x63, 0x2f, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, + 0x6f, 0x2f, 0x63, 0x6c, 0x69, 0x2f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x2f, 0x76, + 0x31, 0x3b, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -542,7 +553,7 @@ var file_cc_arduino_cli_commands_v1_compile_proto_goTypes = []interface{}{ (*ExecutableSectionSize)(nil), // 2: cc.arduino.cli.commands.v1.ExecutableSectionSize nil, // 3: cc.arduino.cli.commands.v1.CompileRequest.SourceOverrideEntry (*Instance)(nil), // 4: cc.arduino.cli.commands.v1.Instance - (*wrappers.BoolValue)(nil), // 5: google.protobuf.BoolValue + (*wrapperspb.BoolValue)(nil), // 5: google.protobuf.BoolValue (*Library)(nil), // 6: cc.arduino.cli.commands.v1.Library } var file_cc_arduino_cli_commands_v1_compile_proto_depIdxs = []int32{ diff --git a/rpc/cc/arduino/cli/commands/v1/compile.proto b/rpc/cc/arduino/cli/commands/v1/compile.proto index e70c1f32965..ef8b7c75a31 100644 --- a/rpc/cc/arduino/cli/commands/v1/compile.proto +++ b/rpc/cc/arduino/cli/commands/v1/compile.proto @@ -57,7 +57,7 @@ message CompileRequest { // If jobs is set to 0, it will use the number of available CPUs as the // maximum. int32 jobs = 14; - // List of custom libraries paths separated by commas. + // List of custom libraries dir paths. repeated string libraries = 15; // Optimize compile output for debug, not for release. bool optimize_for_debug = 16; @@ -78,6 +78,8 @@ message CompileRequest { // When set to `true` the compiled binary will be copied to the export // directory. google.protobuf.BoolValue export_binaries = 23; + // List of paths to libraries root folders + repeated string library = 24; } message CompileResponse { diff --git a/rpc/cc/arduino/cli/commands/v1/core.pb.go b/rpc/cc/arduino/cli/commands/v1/core.pb.go index 0d65ba8c569..fa9b180a6fd 100644 --- a/rpc/cc/arduino/cli/commands/v1/core.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/core.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/commands/v1/core.proto package commands diff --git a/rpc/cc/arduino/cli/commands/v1/lib.pb.go b/rpc/cc/arduino/cli/commands/v1/lib.pb.go index 861303acee0..71bbfe32401 100644 --- a/rpc/cc/arduino/cli/commands/v1/lib.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/lib.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/commands/v1/lib.proto package commands diff --git a/rpc/cc/arduino/cli/commands/v1/upload.pb.go b/rpc/cc/arduino/cli/commands/v1/upload.pb.go index 0d78228fefb..a0d2b869197 100644 --- a/rpc/cc/arduino/cli/commands/v1/upload.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/upload.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/commands/v1/upload.proto package commands diff --git a/rpc/cc/arduino/cli/debug/v1/debug.pb.go b/rpc/cc/arduino/cli/debug/v1/debug.pb.go index b68bcf3040a..7a52ef95219 100644 --- a/rpc/cc/arduino/cli/debug/v1/debug.pb.go +++ b/rpc/cc/arduino/cli/debug/v1/debug.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/debug/v1/debug.proto package debug diff --git a/rpc/cc/arduino/cli/monitor/v1/monitor.pb.go b/rpc/cc/arduino/cli/monitor/v1/monitor.pb.go index ba393d293a3..3ea5652a546 100644 --- a/rpc/cc/arduino/cli/monitor/v1/monitor.pb.go +++ b/rpc/cc/arduino/cli/monitor/v1/monitor.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/monitor/v1/monitor.proto package monitor @@ -24,12 +24,12 @@ package monitor import ( context "context" proto "github.com/golang/protobuf/proto" - _struct "github.com/golang/protobuf/ptypes/struct" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + structpb "google.golang.org/protobuf/types/known/structpb" reflect "reflect" sync "sync" ) @@ -210,7 +210,7 @@ type MonitorConfig struct { Type MonitorConfig_TargetType `protobuf:"varint,2,opt,name=type,proto3,enum=cc.arduino.cli.monitor.v1.MonitorConfig_TargetType" json:"type,omitempty"` // Additional parameters that might be needed to configure the target or the // monitor itself. - AdditionalConfig *_struct.Struct `protobuf:"bytes,3,opt,name=additional_config,json=additionalConfig,proto3" json:"additional_config,omitempty"` + AdditionalConfig *structpb.Struct `protobuf:"bytes,3,opt,name=additional_config,json=additionalConfig,proto3" json:"additional_config,omitempty"` // This parameter indicates how many bytes should be buffered on the server // side before dropping. If >0 then the server will enable a rate limiter and // will send incoming data to the client only when the client allows it: see @@ -264,7 +264,7 @@ func (x *MonitorConfig) GetType() MonitorConfig_TargetType { return MonitorConfig_TARGET_TYPE_SERIAL } -func (x *MonitorConfig) GetAdditionalConfig() *_struct.Struct { +func (x *MonitorConfig) GetAdditionalConfig() *structpb.Struct { if x != nil { return x.AdditionalConfig } @@ -416,7 +416,7 @@ var file_cc_arduino_cli_monitor_v1_monitor_proto_goTypes = []interface{}{ (*StreamingOpenRequest)(nil), // 1: cc.arduino.cli.monitor.v1.StreamingOpenRequest (*MonitorConfig)(nil), // 2: cc.arduino.cli.monitor.v1.MonitorConfig (*StreamingOpenResponse)(nil), // 3: cc.arduino.cli.monitor.v1.StreamingOpenResponse - (*_struct.Struct)(nil), // 4: google.protobuf.Struct + (*structpb.Struct)(nil), // 4: google.protobuf.Struct } var file_cc_arduino_cli_monitor_v1_monitor_proto_depIdxs = []int32{ 2, // 0: cc.arduino.cli.monitor.v1.StreamingOpenRequest.config:type_name -> cc.arduino.cli.monitor.v1.MonitorConfig diff --git a/rpc/cc/arduino/cli/settings/v1/settings.pb.go b/rpc/cc/arduino/cli/settings/v1/settings.pb.go index 518b52aa44d..25dff905961 100644 --- a/rpc/cc/arduino/cli/settings/v1/settings.pb.go +++ b/rpc/cc/arduino/cli/settings/v1/settings.pb.go @@ -16,7 +16,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.25.0 -// protoc v3.12.4 +// protoc v3.15.7 // source: cc/arduino/cli/settings/v1/settings.proto package settings diff --git a/test/test_compile.py b/test/test_compile.py index 1def2d31606..5e860ad6c11 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -887,3 +887,65 @@ def test_compile_manually_installed_platform_using_boards_local_txt(run_command, shutil.copyfile(Path(__file__).parent / "testdata" / "boards.local.txt", boards_local_txt) assert run_command(f"compile --clean -b {fqbn} {sketch_path}") + + +def test_compile_with_library(run_command, data_dir): + assert run_command("update") + + assert run_command("core install arduino:avr@1.8.3") + + sketch_name = "CompileSketchWithWiFi101Dependency" + sketch_path = Path(data_dir, sketch_name) + fqbn = "arduino:avr:uno" + # Create new sketch and add library include + assert run_command(f"sketch new {sketch_path}") + sketch_file = sketch_path / f"{sketch_name}.ino" + lines = [] + with open(sketch_file, "r") as f: + lines = f.readlines() + lines = ["#include \n"] + lines + with open(sketch_file, "w") as f: + f.writelines(lines) + + # Manually installs a library + git_url = "https://github.com/arduino-libraries/WiFi101.git" + lib_path = Path(data_dir, "my-libraries", "WiFi101") + assert Repo.clone_from(git_url, lib_path, multi_options=["-b 0.16.1"]) + + res = run_command(f"compile -b {fqbn} {sketch_path} --library {lib_path} -v") + assert res.ok + assert "WiFi101" in res.stdout + + +def test_compile_with_library_priority(run_command, data_dir): + assert run_command("update") + + assert run_command("core install arduino:avr@1.8.3") + + sketch_name = "CompileSketchWithWiFi101Dependency" + sketch_path = Path(data_dir, sketch_name) + fqbn = "arduino:avr:uno" + + # Manually installs the same library and add a new custom header to it + git_url = "https://github.com/arduino-libraries/WiFi101.git" + lib_path = Path(data_dir, "my-libraries", "WiFi101") + assert Repo.clone_from(git_url, lib_path, multi_options=["-b 0.16.1"]) + with open(lib_path / "src" / "WiFiSomething.h", "x") as f: + f.writelines(["#ifndef WIFI_SOMETHING\n", "#define WIFI_SOMETHING\n", "#endif\n"]) + + # Install the same library we installed manually + assert run_command("lib install WiFi101") + + # Create new sketch and add custom header include + assert run_command(f"sketch new {sketch_path}") + sketch_file = sketch_path / f"{sketch_name}.ino" + lines = [] + with open(sketch_file, "r") as f: + lines = f.readlines() + lines = ["#include "] + lines + with open(sketch_file, "w") as f: + f.writelines(lines) + + res = run_command(f"compile -b {fqbn} {sketch_path} --library {lib_path} -v") + assert res.ok + assert "WiFi101" in res.stdout From 970b696d151d3051b4c34fe2d8fc9e9da0430573 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Thu, 8 Apr 2021 12:23:48 +0200 Subject: [PATCH 2/5] Fix library prioritization --- arduino/libraries/libraries_location.go | 13 +++++++++++++ arduino/libraries/librariesresolver/cpp.go | 2 ++ cli/compile/compile.go | 2 +- docs/sketch-build-process.md | 2 ++ legacy/builder/libraries_loader.go | 3 ++- rpc/cc/arduino/cli/commands/v1/lib.pb.go | 10 ++++++++-- rpc/cc/arduino/cli/commands/v1/lib.proto | 2 ++ test/test_compile.py | 22 ++++++++++++++-------- 8 files changed, 44 insertions(+), 12 deletions(-) diff --git a/arduino/libraries/libraries_location.go b/arduino/libraries/libraries_location.go index c40c6ad9dc0..9ac7aa7d8b6 100644 --- a/arduino/libraries/libraries_location.go +++ b/arduino/libraries/libraries_location.go @@ -35,6 +35,9 @@ const ( ReferencedPlatformBuiltIn // User are user installed libraries User + // Unmanaged is for libraries set manually by the user in the CLI command or from the gRPC function. + // Ideally it's used for `libraries` outside folders managed by the CLI. + Unmanaged ) func (d *LibraryLocation) String() string { @@ -47,6 +50,8 @@ func (d *LibraryLocation) String() string { return "ref-platform" case User: return "user" + case Unmanaged: + return "unmanaged" } panic(fmt.Sprintf("invalid LibraryLocation value %d", *d)) } @@ -62,6 +67,8 @@ func (d *LibraryLocation) MarshalJSON() ([]byte, error) { return json.Marshal("ref-platform") case User: return json.Marshal("user") + case Unmanaged: + return json.Marshal("unmanaged") } return nil, fmt.Errorf("invalid library location value: %d", *d) } @@ -81,6 +88,8 @@ func (d *LibraryLocation) UnmarshalJSON(b []byte) error { *d = ReferencedPlatformBuiltIn case "user": *d = User + case "unmanaged": + *d = Unmanaged } return fmt.Errorf("invalid library location: %s", s) } @@ -96,6 +105,8 @@ func (d *LibraryLocation) ToRPCLibraryLocation() rpc.LibraryLocation { return rpc.LibraryLocation_LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN case User: return rpc.LibraryLocation_LIBRARY_LOCATION_USER + case Unmanaged: + return rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED } panic(fmt.Sprintf("invalid LibraryLocation value %d", *d)) } @@ -111,6 +122,8 @@ func FromRPCLibraryLocation(l rpc.LibraryLocation) LibraryLocation { return ReferencedPlatformBuiltIn case rpc.LibraryLocation_LIBRARY_LOCATION_USER: return User + case rpc.LibraryLocation_LIBRARY_LOCATION_UNMANAGED: + return Unmanaged } panic(fmt.Sprintf("invalid rpc.LibraryLocation value %d", l)) } diff --git a/arduino/libraries/librariesresolver/cpp.go b/arduino/libraries/librariesresolver/cpp.go index 04e64dbc330..e21e5dd1061 100644 --- a/arduino/libraries/librariesresolver/cpp.go +++ b/arduino/libraries/librariesresolver/cpp.go @@ -158,6 +158,8 @@ func computePriority(lib *libraries.Library, header, arch string) int { priority += 2 case libraries.User: priority += 3 + case libraries.Unmanaged: + priority += 4 default: panic(fmt.Sprintf("Invalid library location: %d", lib.Location)) } diff --git a/cli/compile/compile.go b/cli/compile/compile.go index 4a14908b0c6..f6cd7627350 100644 --- a/cli/compile/compile.go +++ b/cli/compile/compile.go @@ -98,7 +98,7 @@ func NewCommand() *cobra.Command { command.Flags().BoolVarP(&verify, "verify", "t", false, "Verify uploaded binary after the upload.") command.Flags().StringVar(&vidPid, "vid-pid", "", "When specified, VID/PID specific build properties are used, if board supports them.") command.Flags().StringSliceVar(&library, "library", []string{}, - "List of paths to libraries root folders. Can be used multiple times for different libraries.") + "List of paths to libraries root folders. Libraries set this way have top priority in case of conflicts. Can be used multiple times for different libraries.") command.Flags().StringSliceVar(&libraries, "libraries", []string{}, "List of custom libraries dir paths separated by commas. Or can be used multiple times for multiple libraries dir paths.") command.Flags().BoolVar(&optimizeForDebug, "optimize-for-debug", false, "Optional, optimize compile output for debugging, rather than for release.") diff --git a/docs/sketch-build-process.md b/docs/sketch-build-process.md index 8dc42756704..7262a6d40c9 100644 --- a/docs/sketch-build-process.md +++ b/docs/sketch-build-process.md @@ -102,6 +102,8 @@ The "folder name priority" is determined as follows (in order of highest to lowe The "location priority" is determined as follows (in order of highest to lowest priority): +1. The library is specified using the [`--library` option](commands/arduino-cli_compile.md#options) of + `arduino-cli compile` 1. The library is under a custom libraries path specified via the [`--libraries` option](commands/arduino-cli_compile.md#options) of `arduino-cli compile` (in decreasing order of priority when multiple custom paths are defined) diff --git a/legacy/builder/libraries_loader.go b/legacy/builder/libraries_loader.go index abbbeb8c797..8bd11d08591 100644 --- a/legacy/builder/libraries_loader.go +++ b/legacy/builder/libraries_loader.go @@ -62,7 +62,8 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error { } for _, dir := range ctx.LibrariesDirs { - if err := lm.LoadLibraryFromDir(dir, libraries.User); err != nil { + // Libraries specified this way have top priority + if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { return err } } diff --git a/rpc/cc/arduino/cli/commands/v1/lib.pb.go b/rpc/cc/arduino/cli/commands/v1/lib.pb.go index 71bbfe32401..1da7ba93e00 100644 --- a/rpc/cc/arduino/cli/commands/v1/lib.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/lib.pb.go @@ -149,6 +149,8 @@ const ( // this indicates the library is in the `libraries` subdirectory of a // platform referenced by the board's platform. LibraryLocation_LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN LibraryLocation = 3 + // Outside the `libraries` folders managed by the CLI. + LibraryLocation_LIBRARY_LOCATION_UNMANAGED LibraryLocation = 4 ) // Enum value maps for LibraryLocation. @@ -158,12 +160,14 @@ var ( 1: "LIBRARY_LOCATION_USER", 2: "LIBRARY_LOCATION_PLATFORM_BUILTIN", 3: "LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN", + 4: "LIBRARY_LOCATION_UNMANAGED", } LibraryLocation_value = map[string]int32{ "LIBRARY_LOCATION_IDE_BUILTIN": 0, "LIBRARY_LOCATION_USER": 1, "LIBRARY_LOCATION_PLATFORM_BUILTIN": 2, "LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN": 3, + "LIBRARY_LOCATION_UNMANAGED": 4, } ) @@ -2355,7 +2359,7 @@ var file_cc_arduino_cli_commands_v1_lib_proto_rawDesc = []byte{ 0x4c, 0x49, 0x42, 0x52, 0x41, 0x52, 0x59, 0x5f, 0x4c, 0x41, 0x59, 0x4f, 0x55, 0x54, 0x5f, 0x46, 0x4c, 0x41, 0x54, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x4c, 0x49, 0x42, 0x52, 0x41, 0x52, 0x59, 0x5f, 0x4c, 0x41, 0x59, 0x4f, 0x55, 0x54, 0x5f, 0x52, 0x45, 0x43, 0x55, 0x52, 0x53, 0x49, 0x56, - 0x45, 0x10, 0x01, 0x2a, 0xa7, 0x01, 0x0a, 0x0f, 0x4c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x4c, + 0x45, 0x10, 0x01, 0x2a, 0xc7, 0x01, 0x0a, 0x0f, 0x4c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x1c, 0x4c, 0x49, 0x42, 0x52, 0x41, 0x52, 0x59, 0x5f, 0x4c, 0x4f, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x49, 0x44, 0x45, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x49, 0x4e, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x4c, 0x49, 0x42, @@ -2365,7 +2369,9 @@ var file_cc_arduino_cli_commands_v1_lib_proto_rawDesc = []byte{ 0x4d, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x49, 0x4e, 0x10, 0x02, 0x12, 0x30, 0x0a, 0x2c, 0x4c, 0x49, 0x42, 0x52, 0x41, 0x52, 0x59, 0x5f, 0x4c, 0x4f, 0x43, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x46, 0x45, 0x52, 0x45, 0x4e, 0x43, 0x45, 0x44, 0x5f, 0x50, 0x4c, 0x41, 0x54, 0x46, - 0x4f, 0x52, 0x4d, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x49, 0x4e, 0x10, 0x03, 0x42, 0x48, 0x5a, + 0x4f, 0x52, 0x4d, 0x5f, 0x42, 0x55, 0x49, 0x4c, 0x54, 0x49, 0x4e, 0x10, 0x03, 0x12, 0x1e, 0x0a, + 0x1a, 0x4c, 0x49, 0x42, 0x52, 0x41, 0x52, 0x59, 0x5f, 0x4c, 0x4f, 0x43, 0x41, 0x54, 0x49, 0x4f, + 0x4e, 0x5f, 0x55, 0x4e, 0x4d, 0x41, 0x4e, 0x41, 0x47, 0x45, 0x44, 0x10, 0x04, 0x42, 0x48, 0x5a, 0x46, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2f, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2d, 0x63, 0x6c, 0x69, 0x2f, 0x72, 0x70, 0x63, 0x2f, 0x63, 0x63, 0x2f, 0x61, 0x72, 0x64, 0x75, 0x69, 0x6e, 0x6f, 0x2f, 0x63, diff --git a/rpc/cc/arduino/cli/commands/v1/lib.proto b/rpc/cc/arduino/cli/commands/v1/lib.proto index 4dbb1684288..502e10a6831 100644 --- a/rpc/cc/arduino/cli/commands/v1/lib.proto +++ b/rpc/cc/arduino/cli/commands/v1/lib.proto @@ -297,6 +297,8 @@ enum LibraryLocation { // this indicates the library is in the `libraries` subdirectory of a // platform referenced by the board's platform. LIBRARY_LOCATION_REFERENCED_PLATFORM_BUILTIN = 3; + // Outside the `libraries` folders managed by the CLI. + LIBRARY_LOCATION_UNMANAGED = 4; } message ZipLibraryInstallRequest { diff --git a/test/test_compile.py b/test/test_compile.py index 5e860ad6c11..12b624e83f8 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -928,24 +928,30 @@ def test_compile_with_library_priority(run_command, data_dir): # Manually installs the same library and add a new custom header to it git_url = "https://github.com/arduino-libraries/WiFi101.git" - lib_path = Path(data_dir, "my-libraries", "WiFi101") - assert Repo.clone_from(git_url, lib_path, multi_options=["-b 0.16.1"]) - with open(lib_path / "src" / "WiFiSomething.h", "x") as f: - f.writelines(["#ifndef WIFI_SOMETHING\n", "#define WIFI_SOMETHING\n", "#endif\n"]) + manually_install_lib_path = Path(data_dir, "my-libraries", "WiFi101") + assert Repo.clone_from(git_url, manually_install_lib_path, multi_options=["-b 0.16.1"]) + # with open(manually_install_lib_path / "src" / "WiFiSomething.h", "x") as f: + # f.writelines(["#ifndef WIFI_SOMETHING\n", "#define WIFI_SOMETHING\n", "#endif\n"]) # Install the same library we installed manually assert run_command("lib install WiFi101") - # Create new sketch and add custom header include + # Create new sketch and add library include assert run_command(f"sketch new {sketch_path}") sketch_file = sketch_path / f"{sketch_name}.ino" lines = [] with open(sketch_file, "r") as f: lines = f.readlines() - lines = ["#include "] + lines + lines = ["#include "] + lines with open(sketch_file, "w") as f: f.writelines(lines) - res = run_command(f"compile -b {fqbn} {sketch_path} --library {lib_path} -v") + res = run_command(f"compile -b {fqbn} {sketch_path} --library {manually_install_lib_path} -v") assert res.ok - assert "WiFi101" in res.stdout + cli_installed_lib_path = Path(data_dir, "libraries", "WiFi101") + expected_output = [ + 'Multiple libraries were found for "WiFi101.h"', + f" Used: {manually_install_lib_path}", + f" Not used: {cli_installed_lib_path}", + ] + assert "\n".join(expected_output) in res.stdout From b0ad3e2e6f2dab207c1a03870f6c77aff0d6eaf6 Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Thu, 8 Apr 2021 12:26:55 +0200 Subject: [PATCH 3/5] [skip changelog] Fix receiver name in LibraryManager functions --- .../librariesmanager/librariesmanager.go | 46 +++++++++---------- test/test_compile.py | 6 +-- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/arduino/libraries/librariesmanager/librariesmanager.go b/arduino/libraries/librariesmanager/librariesmanager.go index e01c8d681ae..a33e69a8675 100644 --- a/arduino/libraries/librariesmanager/librariesmanager.go +++ b/arduino/libraries/librariesmanager/librariesmanager.go @@ -82,10 +82,10 @@ func (alts *LibraryAlternatives) FindVersion(version *semver.Version) *libraries } // Names returns an array with all the names of the installed libraries. -func (sc LibrariesManager) Names() []string { - res := make([]string, len(sc.Libraries)) +func (lm LibrariesManager) Names() []string { + res := make([]string, len(lm.Libraries)) i := 0 - for n := range sc.Libraries { + for n := range lm.Libraries { res[i] = n i++ } @@ -109,27 +109,27 @@ func NewLibraryManager(indexDir *paths.Path, downloadsDir *paths.Path) *Librarie // LoadIndex reads a library_index.json from a file and returns // the corresponding Index structure. -func (sc *LibrariesManager) LoadIndex() error { - index, err := librariesindex.LoadIndex(sc.IndexFile) +func (lm *LibrariesManager) LoadIndex() error { + index, err := librariesindex.LoadIndex(lm.IndexFile) if err != nil { - sc.Index = librariesindex.EmptyIndex + lm.Index = librariesindex.EmptyIndex return err } - sc.Index = index + lm.Index = index return nil } // AddLibrariesDir adds path to the list of directories // to scan when searching for libraries. If a path is already // in the list it is ignored. -func (sc *LibrariesManager) AddLibrariesDir(path *paths.Path, location libraries.LibraryLocation) { - for _, dir := range sc.LibrariesDir { +func (lm *LibrariesManager) AddLibrariesDir(path *paths.Path, location libraries.LibraryLocation) { + for _, dir := range lm.LibrariesDir { if dir.Path.EquivalentTo(path) { return } } logrus.WithField("dir", path).WithField("location", location.String()).Info("Adding libraries dir") - sc.LibrariesDir = append(sc.LibrariesDir, &LibrariesDir{ + lm.LibrariesDir = append(lm.LibrariesDir, &LibrariesDir{ Path: path, Location: location, }) @@ -138,18 +138,18 @@ func (sc *LibrariesManager) AddLibrariesDir(path *paths.Path, location libraries // AddPlatformReleaseLibrariesDir add the libraries directory in the // specified PlatformRelease to the list of directories to scan when // searching for libraries. -func (sc *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *cores.PlatformRelease, location libraries.LibraryLocation) { +func (lm *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *cores.PlatformRelease, location libraries.LibraryLocation) { path := plaftormRelease.GetLibrariesDir() if path == nil { return } - for _, dir := range sc.LibrariesDir { + for _, dir := range lm.LibrariesDir { if dir.Path.EquivalentTo(path) { return } } logrus.WithField("dir", path).WithField("location", location.String()).Info("Adding libraries dir") - sc.LibrariesDir = append(sc.LibrariesDir, &LibrariesDir{ + lm.LibrariesDir = append(lm.LibrariesDir, &LibrariesDir{ Path: path, Location: location, PlatformRelease: plaftormRelease, @@ -157,17 +157,17 @@ func (sc *LibrariesManager) AddPlatformReleaseLibrariesDir(plaftormRelease *core } // RescanLibraries reload all installed libraries in the system. -func (sc *LibrariesManager) RescanLibraries() error { - for _, dir := range sc.LibrariesDir { - if err := sc.LoadLibrariesFromDir(dir); err != nil { +func (lm *LibrariesManager) RescanLibraries() error { + for _, dir := range lm.LibrariesDir { + if err := lm.LoadLibrariesFromDir(dir); err != nil { return fmt.Errorf("loading libs from %s: %s", dir.Path, err) } } return nil } -func (sc *LibrariesManager) getUserLibrariesDir() *paths.Path { - for _, dir := range sc.LibrariesDir { +func (lm *LibrariesManager) getUserLibrariesDir() *paths.Path { + for _, dir := range lm.LibrariesDir { if dir.Location == libraries.User { return dir.Path } @@ -177,7 +177,7 @@ func (sc *LibrariesManager) getUserLibrariesDir() *paths.Path { // LoadLibrariesFromDir loads all libraries in the given directory. Returns // nil if the directory doesn't exists. -func (sc *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) error { +func (lm *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) error { subDirs, err := librariesDir.Path.ReadDir() if os.IsNotExist(err) { return nil @@ -194,10 +194,10 @@ func (sc *LibrariesManager) LoadLibrariesFromDir(librariesDir *LibrariesDir) err return fmt.Errorf("loading library from %s: %s", subDir, err) } library.ContainerPlatform = librariesDir.PlatformRelease - alternatives, ok := sc.Libraries[library.Name] + alternatives, ok := lm.Libraries[library.Name] if !ok { alternatives = &LibraryAlternatives{} - sc.Libraries[library.Name] = alternatives + lm.Libraries[library.Name] = alternatives } alternatives.Add(library) } @@ -230,9 +230,9 @@ func (lm *LibrariesManager) LoadLibraryFromDir(libRootDir *paths.Path, location // FindByReference return the installed library matching the Reference // name and version or, if the version is nil, the library installed // in the User folder. -func (sc *LibrariesManager) FindByReference(libRef *librariesindex.Reference) *libraries.Library { +func (lm *LibrariesManager) FindByReference(libRef *librariesindex.Reference) *libraries.Library { saneName := utils.SanitizeName(libRef.Name) - alternatives, have := sc.Libraries[saneName] + alternatives, have := lm.Libraries[saneName] if !have { return nil } diff --git a/test/test_compile.py b/test/test_compile.py index 12b624e83f8..75bab6b773b 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -922,16 +922,14 @@ def test_compile_with_library_priority(run_command, data_dir): assert run_command("core install arduino:avr@1.8.3") - sketch_name = "CompileSketchWithWiFi101Dependency" + sketch_name = "CompileSketchWithLibraryPriority" sketch_path = Path(data_dir, sketch_name) fqbn = "arduino:avr:uno" - # Manually installs the same library and add a new custom header to it + # Manually installs a library git_url = "https://github.com/arduino-libraries/WiFi101.git" manually_install_lib_path = Path(data_dir, "my-libraries", "WiFi101") assert Repo.clone_from(git_url, manually_install_lib_path, multi_options=["-b 0.16.1"]) - # with open(manually_install_lib_path / "src" / "WiFiSomething.h", "x") as f: - # f.writelines(["#ifndef WIFI_SOMETHING\n", "#define WIFI_SOMETHING\n", "#endif\n"]) # Install the same library we installed manually assert run_command("lib install WiFi101") From a5b8f161b8cd4150cfdbfe03824acae870e8e2ee Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Fri, 9 Apr 2021 10:17:50 +0200 Subject: [PATCH 4/5] [skip changelog] Fix variables names and some docstrings --- commands/compile/compile.go | 2 +- legacy/builder/libraries_loader.go | 2 +- legacy/builder/types/context.go | 2 +- rpc/cc/arduino/cli/commands/v1/compile.pb.go | 2 +- rpc/cc/arduino/cli/commands/v1/compile.proto | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index b3964d58fc8..79d8a6b6fc9 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -137,7 +137,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream builderCtx.OtherLibrariesDirs = paths.NewPathList(req.GetLibraries()...) builderCtx.OtherLibrariesDirs.Add(configuration.LibrariesDir(configuration.Settings)) - builderCtx.LibrariesDirs = paths.NewPathList(req.Library...) + builderCtx.LibraryDirs = paths.NewPathList(req.Library...) if req.GetBuildPath() == "" { builderCtx.BuildPath = bldr.GenBuildPath(sketch.FullPath) diff --git a/legacy/builder/libraries_loader.go b/legacy/builder/libraries_loader.go index 8bd11d08591..690aa4c13c7 100644 --- a/legacy/builder/libraries_loader.go +++ b/legacy/builder/libraries_loader.go @@ -61,7 +61,7 @@ func (s *LibrariesLoader) Run(ctx *types.Context) error { return errors.WithStack(err) } - for _, dir := range ctx.LibrariesDirs { + for _, dir := range ctx.LibraryDirs { // Libraries specified this way have top priority if err := lm.LoadLibraryFromDir(dir, libraries.Unmanaged); err != nil { return err diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index e32b5b44105..079733db662 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -67,7 +67,7 @@ type Context struct { BuiltInToolsDirs paths.PathList BuiltInLibrariesDirs paths.PathList OtherLibrariesDirs paths.PathList - LibrariesDirs paths.PathList // List of paths pointing to individual libraries root folders + LibraryDirs paths.PathList // List of paths pointing to individual library root folders SketchLocation *paths.Path WatchedLocations paths.PathList ArduinoAPIVersion string diff --git a/rpc/cc/arduino/cli/commands/v1/compile.pb.go b/rpc/cc/arduino/cli/commands/v1/compile.pb.go index 275223759f4..9d7c135af3c 100644 --- a/rpc/cc/arduino/cli/commands/v1/compile.pb.go +++ b/rpc/cc/arduino/cli/commands/v1/compile.pb.go @@ -100,7 +100,7 @@ type CompileRequest struct { // When set to `true` the compiled binary will be copied to the export // directory. ExportBinaries *wrapperspb.BoolValue `protobuf:"bytes,23,opt,name=export_binaries,json=exportBinaries,proto3" json:"export_binaries,omitempty"` - // List of paths to libraries root folders + // List of paths to library root folders Library []string `protobuf:"bytes,24,rep,name=library,proto3" json:"library,omitempty"` } diff --git a/rpc/cc/arduino/cli/commands/v1/compile.proto b/rpc/cc/arduino/cli/commands/v1/compile.proto index ef8b7c75a31..85f8dd5ce13 100644 --- a/rpc/cc/arduino/cli/commands/v1/compile.proto +++ b/rpc/cc/arduino/cli/commands/v1/compile.proto @@ -78,7 +78,7 @@ message CompileRequest { // When set to `true` the compiled binary will be copied to the export // directory. google.protobuf.BoolValue export_binaries = 23; - // List of paths to libraries root folders + // List of paths to library root folders repeated string library = 24; } From 3d1de92a480d2a225d26ba2335ef489d3abac78d Mon Sep 17 00:00:00 2001 From: Silvano Cerza Date: Fri, 9 Apr 2021 12:36:24 +0200 Subject: [PATCH 5/5] Fix libraries not being recompiled when path to source file changes --- legacy/builder/builder_utils/utils.go | 10 +++++++ test/test_compile.py | 43 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index fc2f50fa832..5d568b91b67 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -357,6 +357,16 @@ func ObjFileIsUpToDate(ctx *types.Context, sourceFile, objectFile, dependencyFil return false, nil } + // The first line of the depfile contains the path to the object file to generate. + // The second line of the depfile contains the path to the source file. + // All subsequent lines contain the header files necessary to compile the object file. + + // If we don't do this check it might happen that trying to compile a source file + // that has the same name but a different path wouldn't recreate the object file. + if sourceFile.String() != strings.Trim(rows[1], " ") { + return false, nil + } + rows = rows[1:] for _, row := range rows { depStat, err := os.Stat(row) diff --git a/test/test_compile.py b/test/test_compile.py index 75bab6b773b..fa7658b67f0 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -953,3 +953,46 @@ def test_compile_with_library_priority(run_command, data_dir): f" Not used: {cli_installed_lib_path}", ] assert "\n".join(expected_output) in res.stdout + + +def test_recompile_with_different_library(run_command, data_dir): + assert run_command("update") + + assert run_command("core install arduino:avr@1.8.3") + + sketch_name = "RecompileCompileSketchWithDifferentLibrary" + sketch_path = Path(data_dir, sketch_name) + fqbn = "arduino:avr:uno" + + # Install library + assert run_command("lib install WiFi101") + + # Manually installs the same library already installed + git_url = "https://github.com/arduino-libraries/WiFi101.git" + manually_install_lib_path = Path(data_dir, "my-libraries", "WiFi101") + assert Repo.clone_from(git_url, manually_install_lib_path, multi_options=["-b 0.16.1"]) + + # Create new sketch and add library include + assert run_command(f"sketch new {sketch_path}") + sketch_file = sketch_path / f"{sketch_name}.ino" + lines = [] + with open(sketch_file, "r") as f: + lines = f.readlines() + lines = ["#include "] + lines + with open(sketch_file, "w") as f: + f.writelines(lines) + + sketch_path_md5 = hashlib.md5(bytes(sketch_path)).hexdigest().upper() + build_dir = Path(tempfile.gettempdir(), f"arduino-sketch-{sketch_path_md5}") + + # Compile sketch using library not managed by CLI + res = run_command(f"compile -b {fqbn} --library {manually_install_lib_path} {sketch_path} -v") + assert res.ok + obj_path = build_dir / "libraries" / "WiFi101" / "WiFi.cpp.o" + assert f"Using previously compiled file: {obj_path}" not in res.stdout + + # Compile again using library installed from CLI + res = run_command(f"compile -b {fqbn} {sketch_path} -v") + assert res.ok + obj_path = build_dir / "libraries" / "WiFi101" / "WiFi.cpp.o" + assert f"Using previously compiled file: {obj_path}" not in res.stdout