diff --git a/arduino/cores/packagemanager/download.go b/arduino/cores/packagemanager/download.go index be3df8895c6..003258a3226 100644 --- a/arduino/cores/packagemanager/download.go +++ b/arduino/cores/packagemanager/download.go @@ -19,6 +19,7 @@ import ( "fmt" "github.com/arduino/arduino-cli/arduino/cores" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "go.bug.st/downloader/v2" semver "go.bug.st/relaxed-semver" ) @@ -117,16 +118,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader // is returned. -func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) { +func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error { resource := tool.GetCompatibleFlavour() if resource == nil { - return nil, fmt.Errorf(tr("tool not available for your OS")) + return fmt.Errorf(tr("tool not available for your OS")) } - return resource.Download(pm.DownloadDir, config) + return resource.Download(pm.DownloadDir, config, label, progressCB) } // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a // nil Downloader is returned. -func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) { - return platform.Resource.Download(pm.DownloadDir, config) +func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error { + return platform.Resource.Download(pm.DownloadDir, config, label, progressCB) } diff --git a/arduino/httpclient/httpclient.go b/arduino/httpclient/httpclient.go new file mode 100644 index 00000000000..e7ed9458d1a --- /dev/null +++ b/arduino/httpclient/httpclient.go @@ -0,0 +1,117 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package httpclient + +import ( + "net/http" + "net/url" + "time" + + "github.com/arduino/arduino-cli/arduino" + "github.com/arduino/arduino-cli/configuration" + "github.com/arduino/arduino-cli/i18n" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + "github.com/arduino/go-paths-helper" + "go.bug.st/downloader/v2" +) + +var tr = i18n.Tr + +// DownloadFile downloads a file from a URL into the specified path. An optional config and options may be passed (or nil to use the defaults). +// A DownloadProgressCB callback function must be passed to monitor download progress. +func DownloadFile(path *paths.Path, URL string, label string, downloadCB rpc.DownloadProgressCB, config *downloader.Config, options ...downloader.DownloadOptions) error { + if config == nil { + c, err := GetDownloaderConfig() + if err != nil { + return err + } + config = c + } + + d, err := downloader.DownloadWithConfig(path.String(), URL, *config, options...) + if err != nil { + return err + } + downloadCB(&rpc.DownloadProgress{ + File: label, + Url: d.URL, + TotalSize: d.Size(), + }) + + err = d.RunAndPoll(func(downloaded int64) { + downloadCB(&rpc.DownloadProgress{Downloaded: downloaded}) + }, 250*time.Millisecond) + if err != nil { + return err + } + + // The URL is not reachable for some reason + if d.Resp.StatusCode >= 400 && d.Resp.StatusCode <= 599 { + return &arduino.FailedDownloadError{Message: tr("Server responded with: %s", d.Resp.Status)} + } + + downloadCB(&rpc.DownloadProgress{Completed: true}) + return nil +} + +// Config is the configuration of the http client +type Config struct { + UserAgent string + Proxy *url.URL +} + +// New returns a default http client for use in the arduino-cli +func New() (*http.Client, error) { + userAgent := configuration.UserAgent(configuration.Settings) + proxy, err := configuration.NetworkProxy(configuration.Settings) + if err != nil { + return nil, err + } + return NewWithConfig(&Config{UserAgent: userAgent, Proxy: proxy}), nil +} + +// NewWithConfig creates a http client for use in the arduino-cli, with a given configuration +func NewWithConfig(config *Config) *http.Client { + return &http.Client{ + Transport: &httpClientRoundTripper{ + transport: &http.Transport{ + Proxy: http.ProxyURL(config.Proxy), + }, + userAgent: config.UserAgent, + }, + } +} + +// GetDownloaderConfig returns the downloader configuration based on current settings. +func GetDownloaderConfig() (*downloader.Config, error) { + httpClient, err := New() + if err != nil { + return nil, &arduino.InvalidArgumentError{Message: tr("Could not connect via HTTP"), Cause: err} + } + return &downloader.Config{ + HttpClient: *httpClient, + }, nil +} + +type httpClientRoundTripper struct { + transport http.RoundTripper + userAgent string +} + +func (h *httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("User-Agent", h.userAgent) + return h.transport.RoundTrip(req) +} diff --git a/httpclient/httpclient_test.go b/arduino/httpclient/httpclient_test.go similarity index 100% rename from httpclient/httpclient_test.go rename to arduino/httpclient/httpclient_test.go diff --git a/arduino/resources/download.go b/arduino/resources/download.go new file mode 100644 index 00000000000..ba46414a926 --- /dev/null +++ b/arduino/resources/download.go @@ -0,0 +1,59 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package resources + +import ( + "fmt" + "os" + + "github.com/arduino/arduino-cli/arduino/httpclient" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + paths "github.com/arduino/go-paths-helper" + "go.bug.st/downloader/v2" +) + +// Download performs a download loop using the provided downloader.Config. +// Messages are passed back to the DownloadProgressCB using label as text for the File field. +func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config, label string, downloadCB rpc.DownloadProgressCB) error { + path, err := r.ArchivePath(downloadDir) + if err != nil { + return fmt.Errorf(tr("getting archive path: %s"), err) + } + + if _, err := path.Stat(); os.IsNotExist(err) { + // normal download + } else if err == nil { + // check local file integrity + ok, err := r.TestLocalArchiveIntegrity(downloadDir) + if err != nil || !ok { + if err := path.Remove(); err != nil { + return fmt.Errorf(tr("removing corrupted archive file: %s"), err) + } + } else { + // File is cached, nothing to do here + + // This signal means that the file is already downloaded + downloadCB(&rpc.DownloadProgress{ + File: label, + Completed: true, + }) + return nil + } + } else { + return fmt.Errorf(tr("getting archive file info: %s"), err) + } + return httpclient.DownloadFile(path, r.URL, label, downloadCB, config) +} diff --git a/arduino/resources/helpers.go b/arduino/resources/helpers.go index 0f1ccbf1004..39349dd28a9 100644 --- a/arduino/resources/helpers.go +++ b/arduino/resources/helpers.go @@ -17,10 +17,8 @@ package resources import ( "fmt" - "os" "github.com/arduino/go-paths-helper" - "go.bug.st/downloader/v2" ) // ArchivePath returns the path of the Archive of the specified DownloadResource relative @@ -41,30 +39,3 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) { } return archivePath.Exist(), nil } - -// Download a DownloadResource. -func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) { - path, err := r.ArchivePath(downloadDir) - if err != nil { - return nil, fmt.Errorf(tr("getting archive path: %s"), err) - } - - if _, err := path.Stat(); os.IsNotExist(err) { - // normal download - } else if err == nil { - // check local file integrity - ok, err := r.TestLocalArchiveIntegrity(downloadDir) - if err != nil || !ok { - if err := path.Remove(); err != nil { - return nil, fmt.Errorf(tr("removing corrupted archive file: %s"), err) - } - } else { - // File is cached, nothing to do here - return nil, nil - } - } else { - return nil, fmt.Errorf(tr("getting archive file info: %s"), err) - } - - return downloader.DownloadWithConfig(path.String(), r.URL, *config) -} diff --git a/arduino/resources/helpers_test.go b/arduino/resources/helpers_test.go index a0b8efa161d..4d97ecfd6e3 100644 --- a/arduino/resources/helpers_test.go +++ b/arduino/resources/helpers_test.go @@ -22,7 +22,8 @@ import ( "strings" "testing" - "github.com/arduino/arduino-cli/httpclient" + "github.com/arduino/arduino-cli/arduino/httpclient" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" "go.bug.st/downloader/v2" @@ -55,9 +56,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) { httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue}) - d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient}) - require.NoError(t, err) - err = d.Run() + err = r.Download(tmp, &downloader.Config{HttpClient: *httpClient}, "", func(progress *rpc.DownloadProgress) {}) require.NoError(t, err) // leverage the download helper to download the echo for the request made by the downloader itself diff --git a/arduino/resources/index.go b/arduino/resources/index.go new file mode 100644 index 00000000000..c50b0ae4daa --- /dev/null +++ b/arduino/resources/index.go @@ -0,0 +1,120 @@ +// This file is part of arduino-cli. +// +// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) +// +// This software is released under the GNU General Public License version 3, +// which covers the main part of arduino-cli. +// The terms of this license can be found at: +// https://www.gnu.org/licenses/gpl-3.0.en.html +// +// You can be released from the requirements of the above licenses by purchasing +// a commercial license. Buying such a license is mandatory if you want to +// modify or otherwise use the software for commercial activities involving the +// Arduino software without disclosing the source code of your own applications. +// To purchase a commercial license, send an email to license@arduino.cc. + +package resources + +import ( + "net/url" + "path" + "strings" + + "github.com/arduino/arduino-cli/arduino" + "github.com/arduino/arduino-cli/arduino/httpclient" + "github.com/arduino/arduino-cli/arduino/security" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" + "github.com/arduino/go-paths-helper" + "go.bug.st/downloader/v2" +) + +// IndexResource is a reference to an index file URL with an optional signature. +type IndexResource struct { + URL *url.URL + SignatureURL *url.URL +} + +// Download will download the index and possibly check the signature using the Arduino's public key. +// If the file is in .gz format it will be unpacked first. +func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadProgressCB) error { + // Create destination directory + if err := destDir.MkdirAll(); err != nil { + return &arduino.PermissionDeniedError{Message: tr("Can't create data directory %s", destDir), Cause: err} + } + + // Create a temp dir to stage all downloads + tmp, err := paths.MkTempDir("", "library_index_download") + if err != nil { + return &arduino.TempDirCreationFailedError{Cause: err} + } + defer tmp.RemoveAll() + + // Download index file + indexFileName := path.Base(res.URL.Path) // == package_index.json[.gz] + tmpIndexPath := tmp.Join(indexFileName) + if err := httpclient.DownloadFile(tmpIndexPath, res.URL.String(), tr("Downloading index: %s", indexFileName), downloadCB, nil, downloader.NoResume); err != nil { + return &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", res.URL), Cause: err} + } + + // Expand the index if it is compressed + if strings.HasSuffix(indexFileName, ".gz") { + indexFileName = strings.TrimSuffix(indexFileName, ".gz") // == package_index.json + tmpUnzippedIndexPath := tmp.Join(indexFileName) + if err := paths.GUnzip(tmpIndexPath, tmpUnzippedIndexPath); err != nil { + return &arduino.PermissionDeniedError{Message: tr("Error extracting %s", indexFileName), Cause: err} + } + tmpIndexPath = tmpUnzippedIndexPath + } + + // Check the signature if needed + var signaturePath, tmpSignaturePath *paths.Path + if res.SignatureURL != nil { + // Compose signature URL + signatureFileName := path.Base(res.SignatureURL.Path) + + // Download signature + signaturePath = destDir.Join(signatureFileName) + tmpSignaturePath = tmp.Join(signatureFileName) + if err := httpclient.DownloadFile(tmpSignaturePath, res.SignatureURL.String(), tr("Downloading index signature: %s", signatureFileName), downloadCB, nil, downloader.NoResume); err != nil { + return &arduino.FailedDownloadError{Message: tr("Error downloading index signature '%s'", res.SignatureURL), Cause: err} + } + + // Check signature... + if valid, _, err := security.VerifyArduinoDetachedSignature(tmpIndexPath, tmpSignaturePath); err != nil { + return &arduino.PermissionDeniedError{Message: tr("Error verifying signature"), Cause: err} + } else if !valid { + return &arduino.SignatureVerificationFailedError{File: res.URL.String()} + } + } + + // TODO: Implement a ResourceValidator + // if !validate(tmpIndexPath) { return error } + + // Make a backup copy of old index and signature so the defer function can rollback in case of errors. + indexPath := destDir.Join(indexFileName) + oldIndex := tmp.Join("old_index") + if indexPath.Exist() { + if err := indexPath.CopyTo(oldIndex); err != nil { + return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err} + } + defer oldIndex.CopyTo(indexPath) // will silently fail in case of success + } + oldSignature := tmp.Join("old_signature") + if oldSignature.Exist() { + if err := signaturePath.CopyTo(oldSignature); err != nil { + return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err} + } + defer oldSignature.CopyTo(signaturePath) // will silently fail in case of success + } + if err := tmpIndexPath.CopyTo(indexPath); err != nil { + return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err} + } + if res.SignatureURL != nil { + if err := tmpSignaturePath.CopyTo(signaturePath); err != nil { + return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err} + } + } + oldIndex.Remove() + oldSignature.Remove() + return nil +} diff --git a/arduino/resources/resources_test.go b/arduino/resources/resources_test.go index fc22e0ecaa8..efc6061ef57 100644 --- a/arduino/resources/resources_test.go +++ b/arduino/resources/resources_test.go @@ -20,6 +20,7 @@ import ( "encoding/hex" "testing" + rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/arduino/go-paths-helper" "github.com/stretchr/testify/require" "go.bug.st/downloader/v2" @@ -44,9 +45,7 @@ func TestDownloadAndChecksums(t *testing.T) { require.NoError(t, err) downloadAndTestChecksum := func() { - d, err := r.Download(tmp, &downloader.Config{}) - require.NoError(t, err) - err = d.Run() + err := r.Download(tmp, &downloader.Config{}, "", func(*rpc.DownloadProgress) {}) require.NoError(t, err) data, err := testFile.ReadFile() @@ -60,9 +59,8 @@ func TestDownloadAndChecksums(t *testing.T) { downloadAndTestChecksum() // Download with cached file - d, err := r.Download(tmp, &downloader.Config{}) + err = r.Download(tmp, &downloader.Config{}, "", func(*rpc.DownloadProgress) {}) require.NoError(t, err) - require.Nil(t, d) // Download if cached file has data in excess (redownload) data, err := testFile.ReadFile() diff --git a/cli/output/rpc_progress.go b/cli/output/rpc_progress.go index 6bb0707f98c..b52383e8e0e 100644 --- a/cli/output/rpc_progress.go +++ b/cli/output/rpc_progress.go @@ -18,7 +18,6 @@ package output import ( "fmt" - "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/i18n" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/cmaglie/pb" @@ -32,7 +31,7 @@ var ( // ProgressBar returns a DownloadProgressCB that prints a progress bar. // If JSON output format has been selected, the callback outputs nothing. -func ProgressBar() commands.DownloadProgressCB { +func ProgressBar() rpc.DownloadProgressCB { if OutputFormat != "json" { return NewDownloadProgressBarCB() } @@ -43,7 +42,7 @@ func ProgressBar() commands.DownloadProgressCB { // TaskProgress returns a TaskProgressCB that prints the task progress. // If JSON output format has been selected, the callback outputs nothing. -func TaskProgress() commands.TaskProgressCB { +func TaskProgress() rpc.TaskProgressCB { if OutputFormat != "json" { return NewTaskProgressCB() } diff --git a/cli/updater/updater.go b/cli/updater/updater.go index 6c62890cc6b..b6f5372ea77 100644 --- a/cli/updater/updater.go +++ b/cli/updater/updater.go @@ -20,10 +20,10 @@ import ( "strings" "time" + "github.com/arduino/arduino-cli/arduino/httpclient" "github.com/arduino/arduino-cli/cli/feedback" "github.com/arduino/arduino-cli/cli/globals" "github.com/arduino/arduino-cli/configuration" - "github.com/arduino/arduino-cli/httpclient" "github.com/arduino/arduino-cli/i18n" "github.com/arduino/arduino-cli/inventory" "github.com/fatih/color" diff --git a/commands/board/attach.go b/commands/board/attach.go index 14df75b3aeb..aa3f4f5626c 100644 --- a/commands/board/attach.go +++ b/commands/board/attach.go @@ -36,7 +36,7 @@ import ( var tr = i18n.Tr // Attach FIXMEDOC -func Attach(ctx context.Context, req *rpc.BoardAttachRequest, taskCB commands.TaskProgressCB) (*rpc.BoardAttachResponse, error) { +func Attach(ctx context.Context, req *rpc.BoardAttachRequest, taskCB rpc.TaskProgressCB) (*rpc.BoardAttachResponse, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { return nil, &arduino.InvalidInstanceError{} diff --git a/commands/board/list.go b/commands/board/list.go index 86baa16493a..1a826ef28f6 100644 --- a/commands/board/list.go +++ b/commands/board/list.go @@ -28,8 +28,8 @@ import ( "github.com/arduino/arduino-cli/arduino" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" "github.com/arduino/arduino-cli/arduino/discovery" + "github.com/arduino/arduino-cli/arduino/httpclient" "github.com/arduino/arduino-cli/commands" - "github.com/arduino/arduino-cli/httpclient" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" "github.com/pkg/errors" "github.com/segmentio/stats/v4" diff --git a/commands/bundled_tools.go b/commands/bundled_tools.go index 3322dc9bcfe..e55af7ae21a 100644 --- a/commands/bundled_tools.go +++ b/commands/bundled_tools.go @@ -19,24 +19,21 @@ import ( "github.com/arduino/arduino-cli/arduino" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" + "github.com/arduino/arduino-cli/arduino/httpclient" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" ) // DownloadToolRelease downloads a ToolRelease -func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, downloadCB DownloadProgressCB) error { - config, err := GetDownloaderConfig() +func DownloadToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, downloadCB rpc.DownloadProgressCB) error { + config, err := httpclient.GetDownloaderConfig() if err != nil { return err } - resp, err := pm.DownloadToolRelease(toolRelease, config) - if err != nil { - return err - } - return Download(resp, toolRelease.String(), downloadCB) + return pm.DownloadToolRelease(toolRelease, config, toolRelease.String(), downloadCB) } // InstallToolRelease installs a ToolRelease -func InstallToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, taskCB TaskProgressCB) error { +func InstallToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, taskCB rpc.TaskProgressCB) error { log := pm.Log.WithField("Tool", toolRelease) if toolRelease.IsInstalled() { diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 84fddd78470..e9921ea2bdc 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -45,7 +45,7 @@ import ( var tr = i18n.Tr // Compile FIXMEDOC -func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream io.Writer, progressCB commands.TaskProgressCB, debug bool) (r *rpc.CompileResponse, e error) { +func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream io.Writer, progressCB rpc.TaskProgressCB, debug bool) (r *rpc.CompileResponse, e error) { // There is a binding between the export binaries setting and the CLI flag to explicitly set it, // since we want this binding to work also for the gRPC interface we must read it here in this diff --git a/commands/core/download.go b/commands/core/download.go index 2068945b6b4..f9a92f580b0 100644 --- a/commands/core/download.go +++ b/commands/core/download.go @@ -22,6 +22,7 @@ import ( "github.com/arduino/arduino-cli/arduino" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" + "github.com/arduino/arduino-cli/arduino/httpclient" "github.com/arduino/arduino-cli/commands" "github.com/arduino/arduino-cli/i18n" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" @@ -30,7 +31,7 @@ import ( var tr = i18n.Tr // PlatformDownload FIXMEDOC -func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadRequest, downloadCB commands.DownloadProgressCB) (*rpc.PlatformDownloadResponse, error) { +func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadRequest, downloadCB rpc.DownloadProgressCB) (*rpc.PlatformDownloadResponse, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { return nil, &arduino.InvalidInstanceError{} @@ -64,20 +65,16 @@ func PlatformDownload(ctx context.Context, req *rpc.PlatformDownloadRequest, dow return &rpc.PlatformDownloadResponse{}, nil } -func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, downloadCB commands.DownloadProgressCB) error { +func downloadPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, downloadCB rpc.DownloadProgressCB) error { // Download platform - config, err := commands.GetDownloaderConfig() + config, err := httpclient.GetDownloaderConfig() if err != nil { return &arduino.FailedDownloadError{Message: tr("Error downloading platform %s", platformRelease), Cause: err} } - resp, err := pm.DownloadPlatformRelease(platformRelease, config) - if err != nil { - return &arduino.FailedDownloadError{Message: tr("Error downloading platform %s", platformRelease), Cause: err} - } - return commands.Download(resp, platformRelease.String(), downloadCB) + return pm.DownloadPlatformRelease(platformRelease, config, platformRelease.String(), downloadCB) } -func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB commands.DownloadProgressCB) error { +func downloadTool(pm *packagemanager.PackageManager, tool *cores.ToolRelease, downloadCB rpc.DownloadProgressCB) error { // Check if tool has a flavor available for the current OS if tool.GetCompatibleFlavour() == nil { return &arduino.FailedDownloadError{ diff --git a/commands/core/install.go b/commands/core/install.go index 185e687bee4..a150e71be15 100644 --- a/commands/core/install.go +++ b/commands/core/install.go @@ -27,7 +27,7 @@ import ( // PlatformInstall FIXMEDOC func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallRequest, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformInstallResponse, error) { + downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) (*rpc.PlatformInstallResponse, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { @@ -63,7 +63,7 @@ func PlatformInstall(ctx context.Context, req *rpc.PlatformInstallRequest, func installPlatform(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, requiredTools []*cores.ToolRelease, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, + downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB, skipPostInstall bool) error { log := pm.Log.WithField("platform", platformRelease) diff --git a/commands/core/uninstall.go b/commands/core/uninstall.go index ceae27432dd..bbd9a0347d5 100644 --- a/commands/core/uninstall.go +++ b/commands/core/uninstall.go @@ -26,7 +26,7 @@ import ( ) // PlatformUninstall FIXMEDOC -func PlatformUninstall(ctx context.Context, req *rpc.PlatformUninstallRequest, taskCB commands.TaskProgressCB) (*rpc.PlatformUninstallResponse, error) { +func PlatformUninstall(ctx context.Context, req *rpc.PlatformUninstallRequest, taskCB rpc.TaskProgressCB) (*rpc.PlatformUninstallResponse, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { return nil, &arduino.InvalidInstanceError{} @@ -70,7 +70,7 @@ func PlatformUninstall(ctx context.Context, req *rpc.PlatformUninstallRequest, t return &rpc.PlatformUninstallResponse{}, nil } -func uninstallPlatformRelease(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, taskCB commands.TaskProgressCB) error { +func uninstallPlatformRelease(pm *packagemanager.PackageManager, platformRelease *cores.PlatformRelease, taskCB rpc.TaskProgressCB) error { log := pm.Log.WithField("platform", platformRelease) log.Info("Uninstalling platform") @@ -86,7 +86,7 @@ func uninstallPlatformRelease(pm *packagemanager.PackageManager, platformRelease return nil } -func uninstallToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, taskCB commands.TaskProgressCB) error { +func uninstallToolRelease(pm *packagemanager.PackageManager, toolRelease *cores.ToolRelease, taskCB rpc.TaskProgressCB) error { log := pm.Log.WithField("Tool", toolRelease) log.Info("Uninstalling tool") diff --git a/commands/core/upgrade.go b/commands/core/upgrade.go index d83fca8c10b..20c88e5707b 100644 --- a/commands/core/upgrade.go +++ b/commands/core/upgrade.go @@ -26,7 +26,7 @@ import ( // PlatformUpgrade FIXMEDOC func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeRequest, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) (*rpc.PlatformUpgradeResponse, error) { + downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) (*rpc.PlatformUpgradeResponse, error) { pm := commands.GetPackageManager(req.GetInstance().GetId()) if pm == nil { @@ -50,7 +50,7 @@ func PlatformUpgrade(ctx context.Context, req *rpc.PlatformUpgradeRequest, } func upgradePlatform(pm *packagemanager.PackageManager, platformRef *packagemanager.PlatformReference, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB, skipPostInstall bool) error { + downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB, skipPostInstall bool) error { if platformRef.PlatformVersion != nil { return &arduino.InvalidArgumentError{Message: tr("Upgrade doesn't accept parameters with version")} } diff --git a/commands/download.go b/commands/download.go deleted file mode 100644 index b6ff58134e6..00000000000 --- a/commands/download.go +++ /dev/null @@ -1,67 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package commands - -import ( - "time" - - "github.com/arduino/arduino-cli/arduino" - "github.com/arduino/arduino-cli/httpclient" - rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" - "go.bug.st/downloader/v2" -) - -// GetDownloaderConfig returns the downloader configuration based on -// current settings. -func GetDownloaderConfig() (*downloader.Config, error) { - httpClient, err := httpclient.New() - if err != nil { - return nil, &arduino.InvalidArgumentError{Message: tr("Could not connect via HTTP"), Cause: err} - } - return &downloader.Config{ - HttpClient: *httpClient, - }, nil -} - -// Download performs a download loop using the provided downloader.Downloader. -// Messages are passed back to the DownloadProgressCB using label as text for the File field. -func Download(d *downloader.Downloader, label string, downloadCB DownloadProgressCB) error { - if d == nil { - // This signal means that the file is already downloaded - downloadCB(&rpc.DownloadProgress{ - File: label, - Completed: true, - }) - return nil - } - downloadCB(&rpc.DownloadProgress{ - File: label, - Url: d.URL, - TotalSize: d.Size(), - }) - d.RunAndPoll(func(downloaded int64) { - downloadCB(&rpc.DownloadProgress{Downloaded: downloaded}) - }, 250*time.Millisecond) - if d.Error() != nil { - return d.Error() - } - // The URL is not reachable for some reason - if d.Resp.StatusCode >= 400 && d.Resp.StatusCode <= 599 { - return &arduino.FailedDownloadError{Message: tr("Server responded with: %s", d.Resp.Status)} - } - downloadCB(&rpc.DownloadProgress{Completed: true}) - return nil -} diff --git a/commands/instances.go b/commands/instances.go index cf336ceb48d..0f9722133cb 100644 --- a/commands/instances.go +++ b/commands/instances.go @@ -19,19 +19,19 @@ import ( "context" "errors" "fmt" - "io/ioutil" "net/url" "os" - "path" + "strings" "github.com/arduino/arduino-cli/arduino" "github.com/arduino/arduino-cli/arduino/cores" "github.com/arduino/arduino-cli/arduino/cores/packageindex" "github.com/arduino/arduino-cli/arduino/cores/packagemanager" + "github.com/arduino/arduino-cli/arduino/httpclient" "github.com/arduino/arduino-cli/arduino/libraries" "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" - "github.com/arduino/arduino-cli/arduino/security" + "github.com/arduino/arduino-cli/arduino/resources" sk "github.com/arduino/arduino-cli/arduino/sketch" "github.com/arduino/arduino-cli/arduino/utils" "github.com/arduino/arduino-cli/cli/globals" @@ -40,7 +40,6 @@ import ( rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" "github.com/sirupsen/logrus" - "go.bug.st/downloader/v2" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -90,7 +89,7 @@ func GetLibraryManager(instanceID int32) *librariesmanager.LibrariesManager { return i.lm } -func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB DownloadProgressCB, taskCB TaskProgressCB) (bool, error) { +func (instance *CoreInstance) installToolIfMissing(tool *cores.ToolRelease, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) (bool, error) { if tool.IsInstalled() { return false, nil } @@ -362,16 +361,12 @@ func Destroy(ctx context.Context, req *rpc.DestroyRequest) (*rpc.DestroyResponse } // UpdateLibrariesIndex updates the library_index.json -func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequest, downloadCB func(*rpc.DownloadProgress)) error { +func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB) error { logrus.Info("Updating libraries index") lm := GetLibraryManager(req.GetInstance().GetId()) if lm == nil { return &arduino.InvalidInstanceError{} } - config, err := GetDownloaderConfig() - if err != nil { - return err - } if err := lm.IndexFile.Parent().MkdirAll(); err != nil { return &arduino.PermissionDeniedError{Message: tr("Could not create index directory"), Cause: err} @@ -384,54 +379,19 @@ func UpdateLibrariesIndex(ctx context.Context, req *rpc.UpdateLibrariesIndexRequ } defer tmp.RemoveAll() - // Download gzipped library_index - tmpIndexGz := tmp.Join("library_index.json.gz") - if d, err := downloader.DownloadWithConfig(tmpIndexGz.String(), librariesmanager.LibraryIndexGZURL.String(), *config, downloader.NoResume); err == nil { - if err := Download(d, tr("Updating index: library_index.json.gz"), downloadCB); err != nil { - return &arduino.FailedDownloadError{Message: tr("Error downloading library_index.json.gz"), Cause: err} - } - } else { - return &arduino.FailedDownloadError{Message: tr("Error downloading library_index.json.gz"), Cause: err} - } - - // Download signature - tmpSignature := tmp.Join("library_index.json.sig") - if d, err := downloader.DownloadWithConfig(tmpSignature.String(), librariesmanager.LibraryIndexSignature.String(), *config, downloader.NoResume); err == nil { - if err := Download(d, tr("Updating index: library_index.json.sig"), downloadCB); err != nil { - return &arduino.FailedDownloadError{Message: tr("Error downloading library_index.json.sig"), Cause: err} - } - } else { - return &arduino.FailedDownloadError{Message: tr("Error downloading library_index.json.sig"), Cause: err} - } - - // Extract the real library_index - tmpIndex := tmp.Join("library_index.json") - if err := paths.GUnzip(tmpIndexGz, tmpIndex); err != nil { - return &arduino.PermissionDeniedError{Message: tr("Error extracting library_index.json.gz"), Cause: err} - } - - // Check signature - if ok, _, err := security.VerifyArduinoDetachedSignature(tmpIndex, tmpSignature); err != nil { - return &arduino.PermissionDeniedError{Message: tr("Error verifying signature"), Cause: err} - } else if !ok { - return &arduino.SignatureVerificationFailedError{File: "library_index.json"} + indexResource := resources.IndexResource{ + URL: librariesmanager.LibraryIndexGZURL, + SignatureURL: librariesmanager.LibraryIndexSignature, } - - // Copy extracted library_index and signature to final destination - lm.IndexFile.Remove() - lm.IndexFileSignature.Remove() - if err := tmpIndex.CopyTo(lm.IndexFile); err != nil { - return &arduino.PermissionDeniedError{Message: tr("Error writing library_index.json"), Cause: err} - } - if err := tmpSignature.CopyTo(lm.IndexFileSignature); err != nil { - return &arduino.PermissionDeniedError{Message: tr("Error writing library_index.json.sig"), Cause: err} + if err := indexResource.Download(lm.IndexFile.Parent(), downloadCB); err != nil { + return err } return nil } // UpdateIndex FIXMEDOC -func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB DownloadProgressCB) (*rpc.UpdateIndexResponse, error) { +func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB rpc.DownloadProgressCB) (*rpc.UpdateIndexResponse, error) { id := req.GetInstance().GetId() _, ok := instances[id] if !ok { @@ -460,89 +420,22 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB Do fi, _ := os.Stat(path.String()) downloadCB(&rpc.DownloadProgress{ - File: tr("Updating index: %s", path.Base()), + File: tr("Downloading index: %s", path.Base()), TotalSize: fi.Size(), }) downloadCB(&rpc.DownloadProgress{Completed: true}) continue } - var tmp *paths.Path - if tmpFile, err := ioutil.TempFile("", ""); err != nil { - return nil, &arduino.TempFileCreationFailedError{Cause: err} - } else if err := tmpFile.Close(); err != nil { - return nil, &arduino.TempFileCreationFailedError{Cause: err} - } else { - tmp = paths.New(tmpFile.Name()) - } - defer tmp.Remove() - - config, err := GetDownloaderConfig() - if err != nil { - return nil, &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", URL), Cause: err} + indexResource := resources.IndexResource{ + URL: URL, } - d, err := downloader.DownloadWithConfig(tmp.String(), URL.String(), *config) - if err != nil { - return nil, &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", URL), Cause: err} + if strings.HasSuffix(URL.Host, "arduino.cc") { + indexResource.SignatureURL, _ = url.Parse(u) // should not fail because we already parsed it + indexResource.SignatureURL.Path += ".sig" } - coreIndexPath := indexpath.Join(path.Base(URL.Path)) - err = Download(d, tr("Updating index: %s", coreIndexPath.Base()), downloadCB) - if err != nil { - return nil, &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", URL), Cause: err} - } - - // Check for signature - var tmpSig *paths.Path - var coreIndexSigPath *paths.Path - if URL.Hostname() == "downloads.arduino.cc" { - URLSig, err := url.Parse(URL.String()) - if err != nil { - return nil, &arduino.InvalidURLError{Cause: err} - } - URLSig.Path += ".sig" - - if t, err := ioutil.TempFile("", ""); err != nil { - return nil, &arduino.TempFileCreationFailedError{Cause: err} - } else if err := t.Close(); err != nil { - return nil, &arduino.TempFileCreationFailedError{Cause: err} - } else { - tmpSig = paths.New(t.Name()) - } - defer tmpSig.Remove() - - d, err := downloader.DownloadWithConfig(tmpSig.String(), URLSig.String(), *config) - if err != nil { - return nil, &arduino.FailedDownloadError{Message: tr("Error downloading index signature '%s'", URLSig), Cause: err} - } - - coreIndexSigPath = indexpath.Join(path.Base(URLSig.Path)) - Download(d, tr("Updating index: %s", coreIndexSigPath.Base()), downloadCB) - if d.Error() != nil { - return nil, &arduino.FailedDownloadError{Message: tr("Error downloading index signature '%s'", URLSig), Cause: err} - } - - if valid, _, err := security.VerifyArduinoDetachedSignature(tmp, tmpSig); err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Error verifying signature"), Cause: err} - } else if !valid { - return nil, &arduino.SignatureVerificationFailedError{File: URL.String()} - } - } - - if _, err := packageindex.LoadIndex(tmp); err != nil { - return nil, &arduino.InvalidArgumentError{Message: tr("Invalid package index in %s", URL), Cause: err} - } - - if err := indexpath.MkdirAll(); err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Can't create data directory %s", indexpath), Cause: err} - } - - if err := tmp.CopyTo(coreIndexPath); err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index %s", URL), Cause: err} - } - if tmpSig != nil { - if err := tmpSig.CopyTo(coreIndexSigPath); err != nil { - return nil, &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err} - } + if err := indexResource.Download(indexpath, downloadCB); err != nil { + return nil, err } } @@ -550,7 +443,7 @@ func UpdateIndex(ctx context.Context, req *rpc.UpdateIndexRequest, downloadCB Do } // UpdateCoreLibrariesIndex updates both Cores and Libraries indexes -func UpdateCoreLibrariesIndex(ctx context.Context, req *rpc.UpdateCoreLibrariesIndexRequest, downloadCB DownloadProgressCB) error { +func UpdateCoreLibrariesIndex(ctx context.Context, req *rpc.UpdateCoreLibrariesIndexRequest, downloadCB rpc.DownloadProgressCB) error { _, err := UpdateIndex(ctx, &rpc.UpdateIndexRequest{ Instance: req.Instance, }, downloadCB) @@ -687,8 +580,8 @@ func getOutputRelease(lib *librariesindex.Release) *rpc.LibraryRelease { } // Upgrade downloads and installs outdated Cores and Libraries -func Upgrade(ctx context.Context, req *rpc.UpgradeRequest, downloadCB DownloadProgressCB, taskCB TaskProgressCB) error { - downloaderConfig, err := GetDownloaderConfig() +func Upgrade(ctx context.Context, req *rpc.UpgradeRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { + downloaderConfig, err := httpclient.GetDownloaderConfig() if err != nil { return err } @@ -710,9 +603,7 @@ func Upgrade(ctx context.Context, req *rpc.UpgradeRequest, downloadCB DownloadPr // Downloads latest library release taskCB(&rpc.TaskProgress{Name: tr("Downloading %s", available)}) - if d, err := available.Resource.Download(lm.DownloadsDir, downloaderConfig); err != nil { - return &arduino.FailedDownloadError{Message: tr("Error downloading library"), Cause: err} - } else if err := Download(d, available.String(), downloadCB); err != nil { + if err := available.Resource.Download(lm.DownloadsDir, downloaderConfig, available.String(), downloadCB); err != nil { return &arduino.FailedDownloadError{Message: tr("Error downloading library"), Cause: err} } @@ -793,9 +684,7 @@ func Upgrade(ctx context.Context, req *rpc.UpgradeRequest, downloadCB DownloadPr } // Downloads platform - if d, err := pm.DownloadPlatformRelease(latest, downloaderConfig); err != nil { - return &arduino.FailedDownloadError{Message: tr("Error downloading platform %s", latest), Cause: err} - } else if err := Download(d, latest.String(), downloadCB); err != nil { + if err := pm.DownloadPlatformRelease(latest, downloaderConfig, latest.String(), downloadCB); err != nil { return &arduino.FailedDownloadError{Message: tr("Error downloading platform %s", latest), Cause: err} } diff --git a/commands/lib/download.go b/commands/lib/download.go index acbebe55adc..f845d4e73f0 100644 --- a/commands/lib/download.go +++ b/commands/lib/download.go @@ -19,6 +19,7 @@ import ( "context" "github.com/arduino/arduino-cli/arduino" + "github.com/arduino/arduino-cli/arduino/httpclient" "github.com/arduino/arduino-cli/arduino/libraries/librariesindex" "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/commands" @@ -30,7 +31,7 @@ import ( var tr = i18n.Tr // LibraryDownload FIXMEDOC -func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downloadCB commands.DownloadProgressCB) (*rpc.LibraryDownloadResponse, error) { +func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downloadCB rpc.DownloadProgressCB) (*rpc.LibraryDownloadResponse, error) { logrus.Info("Executing `arduino-cli lib download`") lm := commands.GetLibraryManager(req.GetInstance().GetId()) @@ -53,16 +54,14 @@ func LibraryDownload(ctx context.Context, req *rpc.LibraryDownloadRequest, downl } func downloadLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { + downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { taskCB(&rpc.TaskProgress{Name: tr("Downloading %s", libRelease)}) - config, err := commands.GetDownloaderConfig() + config, err := httpclient.GetDownloaderConfig() if err != nil { return &arduino.FailedDownloadError{Message: tr("Can't download library"), Cause: err} } - if d, err := libRelease.Resource.Download(lm.DownloadsDir, config); err != nil { - return &arduino.FailedDownloadError{Message: tr("Can't download library"), Cause: err} - } else if err := commands.Download(d, libRelease.String(), downloadCB); err != nil { + if err := libRelease.Resource.Download(lm.DownloadsDir, config, libRelease.String(), downloadCB); err != nil { return &arduino.FailedDownloadError{Message: tr("Can't download library"), Cause: err} } taskCB(&rpc.TaskProgress{Completed: true}) diff --git a/commands/lib/install.go b/commands/lib/install.go index 5be44ca8ec7..205738e6f29 100644 --- a/commands/lib/install.go +++ b/commands/lib/install.go @@ -28,8 +28,7 @@ import ( ) // LibraryInstall FIXMEDOC -func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, - downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { +func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { lm := commands.GetLibraryManager(req.GetInstance().GetId()) if lm == nil { @@ -89,7 +88,7 @@ func LibraryInstall(ctx context.Context, req *rpc.LibraryInstallRequest, return nil } -func installLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release, taskCB commands.TaskProgressCB) error { +func installLibrary(lm *librariesmanager.LibrariesManager, libRelease *librariesindex.Release, taskCB rpc.TaskProgressCB) error { taskCB(&rpc.TaskProgress{Name: tr("Installing %s", libRelease)}) logrus.WithField("library", libRelease).Info("Installing library") libPath, libReplaced, err := lm.InstallPrerequisiteCheck(libRelease) @@ -115,7 +114,7 @@ func installLibrary(lm *librariesmanager.LibrariesManager, libRelease *libraries } //ZipLibraryInstall FIXMEDOC -func ZipLibraryInstall(ctx context.Context, req *rpc.ZipLibraryInstallRequest, taskCB commands.TaskProgressCB) error { +func ZipLibraryInstall(ctx context.Context, req *rpc.ZipLibraryInstallRequest, taskCB rpc.TaskProgressCB) error { lm := commands.GetLibraryManager(req.GetInstance().GetId()) if err := lm.InstallZipLib(ctx, req.Path, req.Overwrite); err != nil { return &arduino.FailedLibraryInstallError{Cause: err} @@ -125,7 +124,7 @@ func ZipLibraryInstall(ctx context.Context, req *rpc.ZipLibraryInstallRequest, t } //GitLibraryInstall FIXMEDOC -func GitLibraryInstall(ctx context.Context, req *rpc.GitLibraryInstallRequest, taskCB commands.TaskProgressCB) error { +func GitLibraryInstall(ctx context.Context, req *rpc.GitLibraryInstallRequest, taskCB rpc.TaskProgressCB) error { lm := commands.GetLibraryManager(req.GetInstance().GetId()) if err := lm.InstallGitLib(req.Url, req.Overwrite); err != nil { return &arduino.FailedLibraryInstallError{Cause: err} diff --git a/commands/lib/uninstall.go b/commands/lib/uninstall.go index 4145586fd74..f3d80ccb361 100644 --- a/commands/lib/uninstall.go +++ b/commands/lib/uninstall.go @@ -24,7 +24,7 @@ import ( ) // LibraryUninstall FIXMEDOC -func LibraryUninstall(ctx context.Context, req *rpc.LibraryUninstallRequest, taskCB commands.TaskProgressCB) error { +func LibraryUninstall(ctx context.Context, req *rpc.LibraryUninstallRequest, taskCB rpc.TaskProgressCB) error { lm := commands.GetLibraryManager(req.GetInstance().GetId()) ref, err := createLibIndexReference(lm, req) if err != nil { diff --git a/commands/lib/upgrade.go b/commands/lib/upgrade.go index 3f6c83662e7..41aef204265 100644 --- a/commands/lib/upgrade.go +++ b/commands/lib/upgrade.go @@ -23,7 +23,7 @@ import ( ) // LibraryUpgradeAll upgrades all the available libraries -func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { +func LibraryUpgradeAll(instanceID int32, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { lm := commands.GetLibraryManager(instanceID) if lm == nil { return &arduino.InvalidInstanceError{} @@ -41,7 +41,7 @@ func LibraryUpgradeAll(instanceID int32, downloadCB commands.DownloadProgressCB, } // LibraryUpgrade upgrades only the given libraries -func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { +func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { lm := commands.GetLibraryManager(instanceID) if lm == nil { return &arduino.InvalidInstanceError{} @@ -54,7 +54,7 @@ func LibraryUpgrade(instanceID int32, libraryNames []string, downloadCB commands return upgrade(lm, libs, downloadCB, taskCB) } -func upgrade(lm *librariesmanager.LibrariesManager, libs []*installedLib, downloadCB commands.DownloadProgressCB, taskCB commands.TaskProgressCB) error { +func upgrade(lm *librariesmanager.LibrariesManager, libs []*installedLib, downloadCB rpc.DownloadProgressCB, taskCB rpc.TaskProgressCB) error { // Go through the list and download them for _, lib := range libs { if err := downloadLibrary(lm, lib.Available, downloadCB, taskCB); err != nil { diff --git a/httpclient/httpclient_config.go b/configuration/network.go similarity index 52% rename from httpclient/httpclient_config.go rename to configuration/network.go index a60eefec86a..17149391fc2 100644 --- a/httpclient/httpclient_config.go +++ b/configuration/network.go @@ -13,7 +13,7 @@ // Arduino software without disclosing the source code of your own applications. // To purchase a commercial license, send an email to license@arduino.cc. -package httpclient +package configuration import ( "fmt" @@ -21,39 +21,12 @@ import ( "runtime" "github.com/arduino/arduino-cli/cli/globals" - "github.com/arduino/arduino-cli/configuration" + "github.com/spf13/viper" ) -// Config is the configuration of the http client -type Config struct { - UserAgent string - Proxy *url.URL -} - -// DefaultConfig returns the default http client config -func DefaultConfig() (*Config, error) { - var proxy *url.URL - var err error - if configuration.Settings.IsSet("network.proxy") { - proxyConfig := configuration.Settings.GetString("network.proxy") - if proxyConfig == "" { - // empty configuration - // this workaround must be here until viper can UnSet properties: - // https://github.com/spf13/viper/pull/519 - } else if proxy, err = url.Parse(proxyConfig); err != nil { - return nil, fmt.Errorf(tr("Invalid network.proxy '%[1]s': %[2]s"), proxyConfig, err) - } - } - - return &Config{ - UserAgent: UserAgent(), - Proxy: proxy, - }, nil -} - -// UserAgent returns the user agent for the cli http client -func UserAgent() string { - subComponent := configuration.Settings.GetString("network.user_agent_ext") +// UserAgent returns the user agent (mainly used by HTTP clients) +func UserAgent(settings *viper.Viper) string { + subComponent := settings.GetString("network.user_agent_ext") if subComponent != "" { subComponent = " " + subComponent } @@ -65,3 +38,20 @@ func UserAgent() string { runtime.GOARCH, runtime.GOOS, runtime.Version(), globals.VersionInfo.Commit) } + +// NetworkProxy returns the proxy configuration (mainly used by HTTP clients) +func NetworkProxy(settings *viper.Viper) (*url.URL, error) { + if !settings.IsSet("network.proxy") { + return nil, nil + } + if proxyConfig := settings.GetString("network.proxy"); proxyConfig == "" { + // empty configuration + // this workaround must be here until viper can UnSet properties: + // https://github.com/spf13/viper/pull/519 + return nil, nil + } else if proxy, err := url.Parse(proxyConfig); err != nil { + return nil, fmt.Errorf(tr("Invalid network.proxy '%[1]s': %[2]s"), proxyConfig, err) + } else { + return proxy, nil + } +} diff --git a/docs/UPGRADING.md b/docs/UPGRADING.md index 54f5b40c5eb..bca31e0b706 100644 --- a/docs/UPGRADING.md +++ b/docs/UPGRADING.md @@ -4,6 +4,69 @@ Here you can find a list of migration guides to handle breaking changes between ## 0.22.0 +### The content of package `github.com/arduino/arduino-cli/httpclient` has been moved to a different path + +In particular: + +- `UserAgent` and `NetworkProxy` have been moved to `github.com/arduino/arduino-cli/configuration` +- the remainder of the package `github.com/arduino/arduino-cli/httpclient` has been moved to + `github.com/arduino/arduino-cli/arduino/httpclient` + +The old imports must be updated according to the list above. + +### `commands.DownloadProgressCB` and `commands.TaskProgressCB` have been moved to package `github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1` + +All references to these types must be updated with the new import. + +### `commands.GetDownloaderConfig` has been moved to package `github.com/arduino/arduino-cli/arduino/httpclient` + +All references to this function must be updated with the new import. + +### `commands.Download` has been removed and replaced by `github.com/arduino/arduino-cli/arduino/httpclient.DownloadFile` + +The old function must be replaced by the new one that is much more versatile. + +### `packagemanager.PackageManager.DownloadToolRelease`, `packagemanager.PackageManager.DownloadPlatformRelease`, and `resources.DownloadResource.Download` functions change signature and behaviour + +The following functions: + +```go +func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) +func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) +func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) +``` + +now requires a label and a progress callback parameter, do not return the `Downloader` object anymore, and they +automatically handles the download internally: + +```go +func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error +func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error +func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config, label string, downloadCB rpc.DownloadProgressCB) error +``` + +The new progress parameters must be added to legacy code, if progress reports are not needed an empty stub for `label` +and `progressCB` must be provided. There is no more need to execute the `downloader.Run()` or +`downloader.RunAndPoll(...)` method. + +For example, the old legacy code like: + +```go +downloader, err := pm.DownloadPlatformRelease(platformToDownload, config) +if err != nil { + ... +} +if err := downloader.Run(); err != nil { + ... +} +``` + +may be ported to the new version as: + +```go +err := pm.DownloadPlatformRelease(platformToDownload, config, "", func(progress *rpc.DownloadProgress) {}) +``` + ### `packagemanager.Load*` functions now returns `error` instead of `*status.Status` The following functions signature: diff --git a/httpclient/httpclient.go b/httpclient/httpclient.go deleted file mode 100644 index f849c75ca63..00000000000 --- a/httpclient/httpclient.go +++ /dev/null @@ -1,44 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package httpclient - -import ( - "net/http" - - "github.com/arduino/arduino-cli/i18n" -) - -var tr = i18n.Tr - -// New returns a default http client for use in the cli API calls -func New() (*http.Client, error) { - config, err := DefaultConfig() - - if err != nil { - return nil, err - } - - return NewWithConfig(config), nil -} - -// NewWithConfig creates a http client for use in the cli API calls with a given configuration -func NewWithConfig(config *Config) *http.Client { - transport := newHTTPClientTransport(config) - - return &http.Client{ - Transport: transport, - } -} diff --git a/httpclient/httpclient_transport.go b/httpclient/httpclient_transport.go deleted file mode 100644 index 45cba70f016..00000000000 --- a/httpclient/httpclient_transport.go +++ /dev/null @@ -1,41 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package httpclient - -import "net/http" - -type httpClientRoundTripper struct { - transport http.RoundTripper - config *Config -} - -func newHTTPClientTransport(config *Config) http.RoundTripper { - proxy := http.ProxyURL(config.Proxy) - - transport := &http.Transport{ - Proxy: proxy, - } - - return &httpClientRoundTripper{ - transport: transport, - config: config, - } -} - -func (h *httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { - req.Header.Add("User-Agent", h.config.UserAgent) - return h.transport.RoundTrip(req) -} diff --git a/legacy/builder/types/context.go b/legacy/builder/types/context.go index 84c59c7efa6..09a5504af15 100644 --- a/legacy/builder/types/context.go +++ b/legacy/builder/types/context.go @@ -30,7 +30,6 @@ import ( "github.com/arduino/arduino-cli/arduino/libraries/librariesmanager" "github.com/arduino/arduino-cli/arduino/libraries/librariesresolver" "github.com/arduino/arduino-cli/arduino/sketch" - "github.com/arduino/arduino-cli/commands" rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" paths "github.com/arduino/go-paths-helper" properties "github.com/arduino/go-properties-orderedmap" @@ -148,7 +147,7 @@ type Context struct { // Dry run, only create progress map Progress ProgressStruct // Send progress events to this callback - ProgressCB commands.TaskProgressCB + ProgressCB rpc.TaskProgressCB // Contents of a custom build properties file (line by line) CustomBuildProperties []string diff --git a/commands/progress.go b/rpc/cc/arduino/cli/commands/v1/common.go similarity index 82% rename from commands/progress.go rename to rpc/cc/arduino/cli/commands/v1/common.go index ec879a8fb8a..cc987d2f4ab 100644 --- a/commands/progress.go +++ b/rpc/cc/arduino/cli/commands/v1/common.go @@ -15,10 +15,8 @@ package commands -import rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1" - // DownloadProgressCB is a callback to get updates on download progress -type DownloadProgressCB func(curr *rpc.DownloadProgress) +type DownloadProgressCB func(curr *DownloadProgress) // TaskProgressCB is a callback to receive progress messages -type TaskProgressCB func(msg *rpc.TaskProgress) +type TaskProgressCB func(msg *TaskProgress) diff --git a/test/test_core.py b/test/test_core.py index a3f2b121840..0f32ad74f69 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -201,7 +201,7 @@ def test_core_install_without_updateindex(run_command): # Download samd core pinned to 1.8.6 result = run_command(["core", "install", "arduino:samd@1.8.6"]) assert result.ok - assert "Updating index: package_index.json downloaded" in result.stdout + assert "Downloading index: package_index.json downloaded" in result.stdout @pytest.mark.skipif( @@ -363,7 +363,7 @@ def test_core_update_with_local_url(run_command): res = run_command(["core", "update-index", f'--additional-urls="file://{test_index}"']) assert res.ok - assert "Updating index: test_index.json downloaded" in res.stdout + assert "Downloading index: test_index.json downloaded" in res.stdout def test_core_search_manually_installed_cores_not_printed(run_command, data_dir): @@ -523,7 +523,7 @@ def test_core_search_update_index_delay(run_command, data_dir): # Verifies index update is not run res = run_command(["core", "search"]) assert res.ok - assert "Updating index" not in res.stdout + assert "Downloading index" not in res.stdout # Change edit time of package index file index_file = Path(data_dir, "package_index.json") @@ -534,12 +534,12 @@ def test_core_search_update_index_delay(run_command, data_dir): # Verifies index update is run res = run_command(["core", "search"]) assert res.ok - assert "Updating index" in res.stdout + assert "Downloading index" in res.stdout # Verifies index update is not run again res = run_command(["core", "search"]) assert res.ok - assert "Updating index" not in res.stdout + assert "Downloading index" not in res.stdout def test_core_search_sorted_results(run_command, httpserver): diff --git a/test/test_lib.py b/test/test_lib.py index 8d89dffd8b7..1e33e7a18af 100644 --- a/test/test_lib.py +++ b/test/test_lib.py @@ -402,8 +402,8 @@ def test_update_index(run_command): result = run_command(["lib", "update-index"]) assert result.ok lines = [l.strip() for l in result.stdout.splitlines()] - assert "Updating index: library_index.json.gz downloaded" in lines - assert "Updating index: library_index.json.sig downloaded" in lines + assert "Downloading index: library_index.json.gz downloaded" in lines + assert "Downloading index signature: library_index.json.sig downloaded" in lines def test_uninstall(run_command): @@ -453,8 +453,8 @@ def test_search(run_command): result = run_command(["lib", "search", "--names"]) assert result.ok lines = [l.strip() for l in result.stdout.strip().splitlines()] - assert "Updating index: library_index.json.gz downloaded" in lines - assert "Updating index: library_index.json.sig downloaded" in lines + assert "Downloading index: library_index.json.gz downloaded" in lines + assert "Downloading index signature: library_index.json.sig downloaded" in lines libs = [l[6:].strip('"') for l in lines if "Name:" in l] expected = {"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101"} diff --git a/test/test_update.py b/test/test_update.py index 322685ec646..c241cedbe8a 100644 --- a/test/test_update.py +++ b/test/test_update.py @@ -21,10 +21,10 @@ def test_update(run_command): assert res.ok lines = [l.strip() for l in res.stdout.splitlines()] - assert "Updating index: package_index.json downloaded" in lines - assert "Updating index: package_index.json.sig downloaded" in lines - assert "Updating index: library_index.json.gz downloaded" in lines - assert "Updating index: library_index.json.sig downloaded" in lines + assert "Downloading index: package_index.json downloaded" in lines + assert "Downloading index signature: package_index.json.sig downloaded" in lines + assert "Downloading index: library_index.json.gz downloaded" in lines + assert "Downloading index signature: library_index.json.sig downloaded" in lines def test_update_showing_outdated(run_command): @@ -45,10 +45,10 @@ def test_update_showing_outdated(run_command): assert result.ok lines = [l.strip() for l in result.stdout.splitlines()] - assert "Updating index: package_index.json downloaded" in lines - assert "Updating index: package_index.json.sig downloaded" in lines - assert "Updating index: library_index.json.gz downloaded" in lines - assert "Updating index: library_index.json.sig downloaded" in lines + assert "Downloading index: package_index.json downloaded" in lines + assert "Downloading index signature: package_index.json.sig downloaded" in lines + assert "Downloading index: library_index.json.gz downloaded" in lines + assert "Downloading index signature: library_index.json.sig downloaded" in lines assert lines[-5].startswith("Arduino AVR Boards") assert lines[-2].startswith("USBHost")