[breaking] Refactoring of download subroutines

arduino/cores/packagemanager/download.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"go.bug.st/downloader/v2"
 	semver "go.bug.st/relaxed-semver"
 )
@@ -117,16 +118,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
 
 // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
 // is returned.
-func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) {
+func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error {
 	resource := tool.GetCompatibleFlavour()
 	if resource == nil {
-		return nil, fmt.Errorf(tr("tool not available for your OS"))
+		return fmt.Errorf(tr("tool not available for your OS"))
 	}
-	return resource.Download(pm.DownloadDir, config)
+	return resource.Download(pm.DownloadDir, config, label, progressCB)
 }
 
 // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
 // nil Downloader is returned.
-func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) {
-	return platform.Resource.Download(pm.DownloadDir, config)
+func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error {
+	return platform.Resource.Download(pm.DownloadDir, config, label, progressCB)
 }

arduino/httpclient/httpclient.go

@@ -0,0 +1,117 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package httpclient
+
+import (
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/arduino/arduino-cli/arduino"
+	"github.com/arduino/arduino-cli/configuration"
+	"github.com/arduino/arduino-cli/i18n"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+	"github.com/arduino/go-paths-helper"
+	"go.bug.st/downloader/v2"
+)
+
+var tr = i18n.Tr
+
+// DownloadFile downloads a file from a URL into the specified path. An optional config and options may be passed (or nil to use the defaults).
+// A DownloadProgressCB callback function must be passed to monitor download progress.
+func DownloadFile(path *paths.Path, URL string, label string, downloadCB rpc.DownloadProgressCB, config *downloader.Config, options ...downloader.DownloadOptions) error {
+	if config == nil {
+		c, err := GetDownloaderConfig()
+		if err != nil {
+			return err
+		}
+		config = c
+	}
+
+	d, err := downloader.DownloadWithConfig(path.String(), URL, *config, options...)
+	if err != nil {
+		return err
+	}
+	downloadCB(&rpc.DownloadProgress{
+		File:      label,
+		Url:       d.URL,
+		TotalSize: d.Size(),
+	})
+
+	err = d.RunAndPoll(func(downloaded int64) {
+		downloadCB(&rpc.DownloadProgress{Downloaded: downloaded})
+	}, 250*time.Millisecond)
+	if err != nil {
+		return err
+	}
+
+	// The URL is not reachable for some reason
+	if d.Resp.StatusCode >= 400 && d.Resp.StatusCode <= 599 {
+		return &arduino.FailedDownloadError{Message: tr("Server responded with: %s", d.Resp.Status)}
+	}
+
+	downloadCB(&rpc.DownloadProgress{Completed: true})
+	return nil
+}
+
+// Config is the configuration of the http client
+type Config struct {
+	UserAgent string
+	Proxy     *url.URL
+}
+
+// New returns a default http client for use in the arduino-cli
+func New() (*http.Client, error) {
+	userAgent := configuration.UserAgent(configuration.Settings)
+	proxy, err := configuration.NetworkProxy(configuration.Settings)
+	if err != nil {
+		return nil, err
+	}
+	return NewWithConfig(&Config{UserAgent: userAgent, Proxy: proxy}), nil
+}
+
+// NewWithConfig creates a http client for use in the arduino-cli, with a given configuration
+func NewWithConfig(config *Config) *http.Client {
+	return &http.Client{
+		Transport: &httpClientRoundTripper{
+			transport: &http.Transport{
+				Proxy: http.ProxyURL(config.Proxy),
+			},
+			userAgent: config.UserAgent,
+		},
+	}
+}
+
+// GetDownloaderConfig returns the downloader configuration based on current settings.
+func GetDownloaderConfig() (*downloader.Config, error) {
+	httpClient, err := New()
+	if err != nil {
+		return nil, &arduino.InvalidArgumentError{Message: tr("Could not connect via HTTP"), Cause: err}
+	}
+	return &downloader.Config{
+		HttpClient: *httpClient,
+	}, nil
+}
+
+type httpClientRoundTripper struct {
+	transport http.RoundTripper
+	userAgent string
+}
+
+func (h *httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.Header.Add("User-Agent", h.userAgent)
+	return h.transport.RoundTrip(req)
+}

arduino/resources/download.go

@@ -0,0 +1,59 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package resources
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/arduino/arduino-cli/arduino/httpclient"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+	paths "github.com/arduino/go-paths-helper"
+	"go.bug.st/downloader/v2"
+)
+
+// Download performs a download loop using the provided downloader.Config.
+// Messages are passed back to the DownloadProgressCB using label as text for the File field.
+func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config, label string, downloadCB rpc.DownloadProgressCB) error {
+	path, err := r.ArchivePath(downloadDir)
+	if err != nil {
+		return fmt.Errorf(tr("getting archive path: %s"), err)
+	}
+
+	if _, err := path.Stat(); os.IsNotExist(err) {
+		// normal download
+	} else if err == nil {
+		// check local file integrity
+		ok, err := r.TestLocalArchiveIntegrity(downloadDir)
+		if err != nil || !ok {
+			if err := path.Remove(); err != nil {
+				return fmt.Errorf(tr("removing corrupted archive file: %s"), err)
+			}
+		} else {
+			// File is cached, nothing to do here
+
+			// This signal means that the file is already downloaded
+			downloadCB(&rpc.DownloadProgress{
+				File:      label,
+				Completed: true,
+			})
+			return nil
+		}
+	} else {
+		return fmt.Errorf(tr("getting archive file info: %s"), err)
+	}
+	return httpclient.DownloadFile(path, r.URL, label, downloadCB, config)
+}

arduino/resources/helpers.go

@@ -17,10 +17,8 @@ package resources
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/arduino/go-paths-helper"
-	"go.bug.st/downloader/v2"
 )
 
 // ArchivePath returns the path of the Archive of the specified DownloadResource relative
@@ -41,30 +39,3 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) {
 	}
 	return archivePath.Exist(), nil
 }
-
-// Download a DownloadResource.
-func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) {
-	path, err := r.ArchivePath(downloadDir)
-	if err != nil {
-		return nil, fmt.Errorf(tr("getting archive path: %s"), err)
-	}
-
-	if _, err := path.Stat(); os.IsNotExist(err) {
-		// normal download
-	} else if err == nil {
-		// check local file integrity
-		ok, err := r.TestLocalArchiveIntegrity(downloadDir)
-		if err != nil || !ok {
-			if err := path.Remove(); err != nil {
-				return nil, fmt.Errorf(tr("removing corrupted archive file: %s"), err)
-			}
-		} else {
-			// File is cached, nothing to do here
-			return nil, nil
-		}
-	} else {
-		return nil, fmt.Errorf(tr("getting archive file info: %s"), err)
-	}
-
-	return downloader.DownloadWithConfig(path.String(), r.URL, *config)
-}

arduino/resources/helpers_test.go

@@ -22,7 +22,8 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/arduino/arduino-cli/httpclient"
+	"github.com/arduino/arduino-cli/arduino/httpclient"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
 	"go.bug.st/downloader/v2"
@@ -55,9 +56,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
 
 	httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue})
 
-	d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient})
-	require.NoError(t, err)
-	err = d.Run()
+	err = r.Download(tmp, &downloader.Config{HttpClient: *httpClient}, "", func(progress *rpc.DownloadProgress) {})
 	require.NoError(t, err)
 
 	// leverage the download helper to download the echo for the request made by the downloader itself

arduino/resources/index.go

@@ -0,0 +1,120 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package resources
+
+import (
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/arduino/arduino-cli/arduino"
+	"github.com/arduino/arduino-cli/arduino/httpclient"
+	"github.com/arduino/arduino-cli/arduino/security"
+	rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
+	"github.com/arduino/go-paths-helper"
+	"go.bug.st/downloader/v2"
+)
+
+// IndexResource is a reference to an index file URL with an optional signature.
+type IndexResource struct {
+	URL          *url.URL
+	SignatureURL *url.URL
+}
+
+// Download will download the index and possibly check the signature using the Arduino's public key.
+// If the file is in .gz format it will be unpacked first.
+func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadProgressCB) error {
+	// Create destination directory
+	if err := destDir.MkdirAll(); err != nil {
+		return &arduino.PermissionDeniedError{Message: tr("Can't create data directory %s", destDir), Cause: err}
+	}
+
+	// Create a temp dir to stage all downloads
+	tmp, err := paths.MkTempDir("", "library_index_download")
+	if err != nil {
+		return &arduino.TempDirCreationFailedError{Cause: err}
+	}
+	defer tmp.RemoveAll()
+
+	// Download index file
+	indexFileName := path.Base(res.URL.Path) // == package_index.json[.gz]
+	tmpIndexPath := tmp.Join(indexFileName)
+	if err := httpclient.DownloadFile(tmpIndexPath, res.URL.String(), tr("Downloading index: %s", indexFileName), downloadCB, nil, downloader.NoResume); err != nil {
+		return &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", res.URL), Cause: err}
+	}
+
+	// Expand the index if it is compressed
+	if strings.HasSuffix(indexFileName, ".gz") {
+		indexFileName = strings.TrimSuffix(indexFileName, ".gz") // == package_index.json
+		tmpUnzippedIndexPath := tmp.Join(indexFileName)
+		if err := paths.GUnzip(tmpIndexPath, tmpUnzippedIndexPath); err != nil {
+			return &arduino.PermissionDeniedError{Message: tr("Error extracting %s", indexFileName), Cause: err}
+		}
+		tmpIndexPath = tmpUnzippedIndexPath
+	}
+
+	// Check the signature if needed
+	var signaturePath, tmpSignaturePath *paths.Path
+	if res.SignatureURL != nil {
+		// Compose signature URL
+		signatureFileName := path.Base(res.SignatureURL.Path)
+
+		// Download signature
+		signaturePath = destDir.Join(signatureFileName)
+		tmpSignaturePath = tmp.Join(signatureFileName)
+		if err := httpclient.DownloadFile(tmpSignaturePath, res.SignatureURL.String(), tr("Downloading index signature: %s", signatureFileName), downloadCB, nil, downloader.NoResume); err != nil {
+			return &arduino.FailedDownloadError{Message: tr("Error downloading index signature '%s'", res.SignatureURL), Cause: err}
+		}
+
+		// Check signature...
+		if valid, _, err := security.VerifyArduinoDetachedSignature(tmpIndexPath, tmpSignaturePath); err != nil {
+			return &arduino.PermissionDeniedError{Message: tr("Error verifying signature"), Cause: err}
+		} else if !valid {
+			return &arduino.SignatureVerificationFailedError{File: res.URL.String()}
+		}
+	}
+
+	// TODO: Implement a ResourceValidator
+	// if !validate(tmpIndexPath) { return error }
+
+	// Make a backup copy of old index and signature so the defer function can rollback in case of errors.
+	indexPath := destDir.Join(indexFileName)
+	oldIndex := tmp.Join("old_index")
+	if indexPath.Exist() {
+		if err := indexPath.CopyTo(oldIndex); err != nil {
+			return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err}
+		}
+		defer oldIndex.CopyTo(indexPath) // will silently fail in case of success
+	}
+	oldSignature := tmp.Join("old_signature")
+	if oldSignature.Exist() {
+		if err := signaturePath.CopyTo(oldSignature); err != nil {
+			return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err}
+		}
+		defer oldSignature.CopyTo(signaturePath) // will silently fail in case of success
+	}
+	if err := tmpIndexPath.CopyTo(indexPath); err != nil {
+		return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err}
+	}
+	if res.SignatureURL != nil {
+		if err := tmpSignaturePath.CopyTo(signaturePath); err != nil {
+			return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err}
+		}
+	}
+	oldIndex.Remove()
+	oldSignature.Remove()
+	return nil
+}