From af08c7e7ba949c9fed027b047d8d988b79572cb8 Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Fri, 18 Feb 2022 16:23:48 +0100 Subject: [PATCH 1/3] IDE updater assorted bugfix - add linux AppImage target - fix hardcoded if condition that causes to always show the update dialog - fix redundant test build version - recalculate sha512 after notarization on macOS --- .github/workflows/build.yml | 4 +- .../src/node/ide-updater/ide-updater-impl.ts | 5 +- electron/build/scripts/hash.js | 31 ++++++++ electron/build/scripts/notarize.js | 79 ++++++++++++++----- electron/build/template-package.json | 17 ++-- electron/packager/config.js | 15 +--- electron/packager/package.json | 6 +- electron/packager/yarn.lock | 2 +- 8 files changed, 107 insertions(+), 52 deletions(-) create mode 100644 electron/build/scripts/hash.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 25dbc8b2e..270877953 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -96,7 +96,9 @@ jobs: matrix: artifact: - path: '*Linux_64bit.zip' - name: Linux_X86-64 + name: Linux_X86-64_zip + - path: '*Linux_64bit.AppImage' + name: Linux_X86-64_app_image - path: '*macOS_64bit.dmg' name: macOS_dmg - path: '*macOS_64bit.zip' diff --git a/arduino-ide-extension/src/node/ide-updater/ide-updater-impl.ts b/arduino-ide-extension/src/node/ide-updater/ide-updater-impl.ts index 1b871bba9..e40dbd997 100644 --- a/arduino-ide-extension/src/node/ide-updater/ide-updater-impl.ts +++ b/arduino-ide-extension/src/node/ide-updater/ide-updater-impl.ts @@ -58,10 +58,7 @@ export class IDEUpdaterImpl implements IDEUpdater { } = await this.updater.checkForUpdates(); this.cancellationToken = cancellationToken; - if ( - this.updater.currentVersion.compare(updateInfo.version) === -1 || - true - ) { + if (this.updater.currentVersion.compare(updateInfo.version) === -1) { /* 'latest.txt' points to the latest changelog that has been generated by the CI, so we need to make a first GET request to get the filename of the changelog diff --git a/electron/build/scripts/hash.js b/electron/build/scripts/hash.js new file mode 100644 index 000000000..e03ed2d8e --- /dev/null +++ b/electron/build/scripts/hash.js @@ -0,0 +1,31 @@ +const fs = require('fs'); +const crypto = require('crypto'); + +async function hashFile( + file, + algorithm = 'sha512', + encoding = 'base64', + options +) { + return await new Promise((resolve, reject) => { + const hash = crypto.createHash(algorithm); + hash.on('error', reject).setEncoding(encoding); + fs.createReadStream( + file, + Object.assign({}, options, { + highWaterMark: 1024 * 1024, + /* better to use more memory but hash faster */ + }) + ) + .on('error', reject) + .on('end', () => { + hash.end(); + resolve(hash.read()); + }) + .pipe(hash, { + end: false, + }); + }); +} + +module.exports = { hashFile }; diff --git a/electron/build/scripts/notarize.js b/electron/build/scripts/notarize.js index b297f867a..cdc5c5ccf 100644 --- a/electron/build/scripts/notarize.js +++ b/electron/build/scripts/notarize.js @@ -1,28 +1,65 @@ +const fs = require('fs'); const isCI = require('is-ci'); const { notarize } = require('electron-notarize'); +const utils = require('../../packager/utils'); +const { getChannelFile } = utils; +const join = require('path').join; +const { hashFile } = require('./hash'); exports.default = async function notarizing(context) { - if (!isCI) { - console.log('Skipping notarization: not on CI.'); - return; - } - if (process.env.IS_FORK === 'true') { - console.log('Skipping the app notarization: building from a fork.'); - return; - } - const { electronPlatformName, appOutDir } = context; - if (electronPlatformName !== 'darwin') { - return; - } + if (!isCI) { + console.log('Skipping notarization: not on CI.'); + return; + } + if (process.env.IS_FORK === 'true') { + console.log('Skipping the app notarization: building from a fork.'); + return; + } + const { electronPlatformName, appOutDir } = context; + if (electronPlatformName !== 'darwin') { + return; + } - const appName = context.packager.appInfo.productFilename; - const appBundleId = context.packager.config.appId; - console.log(`>>> Notarizing ${appBundleId} at ${appOutDir}/${appName}.app...`); + const appName = context.packager.appInfo.productFilename; + const appBundleId = context.packager.config.appId; + console.log( + `>>> Notarizing ${appBundleId} at ${appOutDir}/${appName}.app...` + ); - return await notarize({ - appBundleId, - appPath: `${appOutDir}/${appName}.app`, - appleId: process.env.AC_USERNAME, - appleIdPassword: process.env.AC_PASSWORD, - }); + await notarize({ + appBundleId, + appPath: `${appOutDir}/${appName}.app`, + appleId: process.env.AC_USERNAME, + appleIdPassword: process.env.AC_PASSWORD, + }); + return await recalculateHash(); }; + +async function recalculateHash() { + const { platform } = process; + const cwd = join(__dirname, '..', 'build', 'dist'); + const channelFilePath = join(cwd, getChannelFile(platform)); + const yaml = require('yaml'); + + try { + let fileContents = fs.readFileSync(channelFilePath, 'utf8'); + const newChannelFile = yaml.parse(fileContents); + const { files, path } = newChannelFile; + const newSha512 = await hashFile(join(cwd, path)); + newChannelFile.sha512 = newSha512; + if (!!files) { + const newFiles = []; + for (let file of files) { + const { url } = file; + const { size } = fs.statSync(join(cwd, url)); + const newSha512 = await hashFile(join(cwd, url)); + newFiles.push({ ...file, sha512: newSha512, size }); + } + newChannelFile.files = newFiles; + } + console.log(channelFilePath); + fs.writeFileSync(channelFilePath, yaml.stringify(newChannelFile)); + } catch (e) { + console.log(e); + } +} diff --git a/electron/build/template-package.json b/electron/build/template-package.json index 4f21f78ff..059e4efe8 100644 --- a/electron/build/template-package.json +++ b/electron/build/template-package.json @@ -17,7 +17,9 @@ "electron-notarize": "^0.3.0", "is-ci": "^2.0.0", "ncp": "^2.0.0", - "shelljs": "^0.8.3" + "shelljs": "^0.8.3", + "crypto": "^1.0.1", + "yaml": "^1.10.2" }, "scripts": { "build": "yarn download:plugins && theia build --mode development && yarn patch", @@ -92,19 +94,12 @@ "gatekeeperAssess": false, "entitlements": "resources/entitlements.mac.plist", "entitlementsInherit": "resources/entitlements.mac.plist", - "target": [ - "dmg", - "zip" - ] + "target": "default" }, "linux": { "target": [ - { - "target": "zip" - }, - { - "target": "AppImage" - } + "zip", + "AppImage" ], "category": "Development", "icon": "resources/icons" diff --git a/electron/packager/config.js b/electron/packager/config.js index 5b2aeeaa1..44af9f323 100644 --- a/electron/packager/config.js +++ b/electron/packager/config.js @@ -80,19 +80,12 @@ function getVersion() { } if (!isRelease) { if (isNightly) { - version = `${version}-nightly.${timestamp()}`; + version = `${version}-nightly-${timestamp()}`; } else { - version = `${version}-snapshot.${currentCommitish()}`; + version = `${version}-snapshot-${currentCommitish()}`; } - if (!isRelease) { - if (isNightly) { - version = `${version}-nightly-${timestamp()}`; - } else { - version = `${version}-snapshot-${currentCommitish()}`; - } - if (!semver.valid(version)) { - throw new Error(`Invalid patched version: '${version}'.`); - } + if (!semver.valid(version)) { + throw new Error(`Invalid patched version: '${version}'.`); } } return version; diff --git a/electron/packager/package.json b/electron/packager/package.json index 6ffbeb1ef..91414cac7 100644 --- a/electron/packager/package.json +++ b/electron/packager/package.json @@ -13,9 +13,9 @@ "author": "Arduino SA", "license": "AGPL-3.0-or-later", "dependencies": { + "7zip-min": "^1.1.1", "@types/file-type": "^10.9.1", "@types/temp": "^0.8.32", - "7zip-min": "^1.1.1", "chai": "^4.2.0", "dateformat": "^3.0.3", "deepmerge": "2.01", @@ -25,8 +25,8 @@ "is-ci": "^2.0.0", "mocha": "^7.1.1", "semver": "^7.3.2", - "sinon": "^9.0.1", "shelljs": "^0.8.3", + "sinon": "^9.0.1", "temp": "^0.9.1", "yargs": "^12.0.5" }, @@ -39,4 +39,4 @@ "watch-extensions": "js", "timeout": 10000 } -} +} \ No newline at end of file diff --git a/electron/packager/yarn.lock b/electron/packager/yarn.lock index 396dd6c1d..473faa674 100644 --- a/electron/packager/yarn.lock +++ b/electron/packager/yarn.lock @@ -1679,4 +1679,4 @@ yargs@^15.0.2: string-width "^4.2.0" which-module "^2.0.0" y18n "^4.0.0" - yargs-parser "^18.1.1" \ No newline at end of file + yargs-parser "^18.1.1" From 1436d3b4d46fed5c5b484bc7460862ce79bea82b Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Fri, 18 Feb 2022 16:27:46 +0100 Subject: [PATCH 2/3] boost notarization speed --- .github/workflows/build.yml | 1 + electron/build/scripts/notarize.js | 2 ++ electron/build/template-package.json | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 270877953..f6915756e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,6 +50,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} + AC_TEAM_ID: ${{ secrets.AC_TEAM_ID }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} IS_NIGHTLY: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') }} diff --git a/electron/build/scripts/notarize.js b/electron/build/scripts/notarize.js index cdc5c5ccf..c9bb1cb47 100644 --- a/electron/build/scripts/notarize.js +++ b/electron/build/scripts/notarize.js @@ -31,6 +31,8 @@ exports.default = async function notarizing(context) { appPath: `${appOutDir}/${appName}.app`, appleId: process.env.AC_USERNAME, appleIdPassword: process.env.AC_PASSWORD, + teamId: process.env.AC_TEAM_ID, + tool: 'notarytool', }); return await recalculateHash(); }; diff --git a/electron/build/template-package.json b/electron/build/template-package.json index 059e4efe8..52a1c2905 100644 --- a/electron/build/template-package.json +++ b/electron/build/template-package.json @@ -14,7 +14,7 @@ "@theia/cli": "1.22.1", "cross-env": "^7.0.2", "electron-builder": "22.10.5", - "electron-notarize": "^0.3.0", + "electron-notarize": "^1.1.1", "is-ci": "^2.0.0", "ncp": "^2.0.0", "shelljs": "^0.8.3", From 41edad75007331e838f489f4df4c098e1ddb649c Mon Sep 17 00:00:00 2001 From: Alberto Iannaccone Date: Mon, 21 Feb 2022 15:47:45 +0100 Subject: [PATCH 3/3] recalculate artifacts hash --- .github/workflows/build.yml | 2 - electron/build/scripts/hash.js | 31 ------------- electron/build/scripts/notarize.js | 35 --------------- electron/build/template-package.json | 8 ++-- electron/packager/index.js | 65 +++++++++++++++++++++++++++- electron/packager/package.json | 2 + electron/packager/utils.js | 10 +---- electron/packager/yarn.lock | 10 +++++ 8 files changed, 82 insertions(+), 81 deletions(-) delete mode 100644 electron/build/scripts/hash.js diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f6915756e..4465c1cd8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,8 +102,6 @@ jobs: name: Linux_X86-64_app_image - path: '*macOS_64bit.dmg' name: macOS_dmg - - path: '*macOS_64bit.zip' - name: macOS_zip - path: '*Windows_64bit.exe' name: Windows_X86-64_interactive_installer - path: '*Windows_64bit.msi' diff --git a/electron/build/scripts/hash.js b/electron/build/scripts/hash.js deleted file mode 100644 index e03ed2d8e..000000000 --- a/electron/build/scripts/hash.js +++ /dev/null @@ -1,31 +0,0 @@ -const fs = require('fs'); -const crypto = require('crypto'); - -async function hashFile( - file, - algorithm = 'sha512', - encoding = 'base64', - options -) { - return await new Promise((resolve, reject) => { - const hash = crypto.createHash(algorithm); - hash.on('error', reject).setEncoding(encoding); - fs.createReadStream( - file, - Object.assign({}, options, { - highWaterMark: 1024 * 1024, - /* better to use more memory but hash faster */ - }) - ) - .on('error', reject) - .on('end', () => { - hash.end(); - resolve(hash.read()); - }) - .pipe(hash, { - end: false, - }); - }); -} - -module.exports = { hashFile }; diff --git a/electron/build/scripts/notarize.js b/electron/build/scripts/notarize.js index c9bb1cb47..c13111854 100644 --- a/electron/build/scripts/notarize.js +++ b/electron/build/scripts/notarize.js @@ -1,10 +1,5 @@ -const fs = require('fs'); const isCI = require('is-ci'); const { notarize } = require('electron-notarize'); -const utils = require('../../packager/utils'); -const { getChannelFile } = utils; -const join = require('path').join; -const { hashFile } = require('./hash'); exports.default = async function notarizing(context) { if (!isCI) { @@ -34,34 +29,4 @@ exports.default = async function notarizing(context) { teamId: process.env.AC_TEAM_ID, tool: 'notarytool', }); - return await recalculateHash(); }; - -async function recalculateHash() { - const { platform } = process; - const cwd = join(__dirname, '..', 'build', 'dist'); - const channelFilePath = join(cwd, getChannelFile(platform)); - const yaml = require('yaml'); - - try { - let fileContents = fs.readFileSync(channelFilePath, 'utf8'); - const newChannelFile = yaml.parse(fileContents); - const { files, path } = newChannelFile; - const newSha512 = await hashFile(join(cwd, path)); - newChannelFile.sha512 = newSha512; - if (!!files) { - const newFiles = []; - for (let file of files) { - const { url } = file; - const { size } = fs.statSync(join(cwd, url)); - const newSha512 = await hashFile(join(cwd, url)); - newFiles.push({ ...file, sha512: newSha512, size }); - } - newChannelFile.files = newFiles; - } - console.log(channelFilePath); - fs.writeFileSync(channelFilePath, yaml.stringify(newChannelFile)); - } catch (e) { - console.log(e); - } -} diff --git a/electron/build/template-package.json b/electron/build/template-package.json index 52a1c2905..6d15db0a4 100644 --- a/electron/build/template-package.json +++ b/electron/build/template-package.json @@ -17,9 +17,7 @@ "electron-notarize": "^1.1.1", "is-ci": "^2.0.0", "ncp": "^2.0.0", - "shelljs": "^0.8.3", - "crypto": "^1.0.1", - "yaml": "^1.10.2" + "shelljs": "^0.8.3" }, "scripts": { "build": "yarn download:plugins && theia build --mode development && yarn patch", @@ -94,7 +92,9 @@ "gatekeeperAssess": false, "entitlements": "resources/entitlements.mac.plist", "entitlementsInherit": "resources/entitlements.mac.plist", - "target": "default" + "target": [ + "dmg" + ] }, "linux": { "target": [ diff --git a/electron/packager/index.js b/electron/packager/index.js index 5fc213164..784a5a38d 100644 --- a/electron/packager/index.js +++ b/electron/packager/index.js @@ -257,10 +257,12 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()} ); //-----------------------------------------------------------------------------------------------------+ - // Copy to another folder. Azure does not support wildcard for `PublishBuildArtifacts@1.pathToPublish` | + // Recalculate artifacts hash and copy to another folder (because they can change after signing them). + // Azure does not support wildcard for `PublishBuildArtifacts@1.pathToPublish` | //-----------------------------------------------------------------------------------------------------+ if (isCI) { try { + await recalculateArtifactsHash(); await copyFilesToBuildArtifacts(); } catch (e) { echo(JSON.stringify(e)); @@ -400,6 +402,67 @@ ${fs.readFileSync(path('..', 'build', 'package.json')).toString()} } } + async function recalculateArtifactsHash() { + echo(`🚢 Detected CI, recalculating artifacts hash...`); + const { platform } = process; + const cwd = path('..', 'build', 'dist'); + const channelFilePath = join(cwd, getChannelFile(platform)); + const yaml = require('yaml'); + + try { + let fileContents = fs.readFileSync(channelFilePath, 'utf8'); + const newChannelFile = yaml.parse(fileContents); + const { files, path } = newChannelFile; + const newSha512 = await hashFile(join(cwd, path)); + newChannelFile.sha512 = newSha512; + if (!!files) { + const newFiles = []; + for (let file of files) { + const { url } = file; + const { size } = fs.statSync(join(cwd, url)); + const newSha512 = await hashFile(join(cwd, url)); + newFiles.push({ ...file, sha512: newSha512, size }); + } + newChannelFile.files = newFiles; + } + + const newChannelFileRaw = yaml.stringify(newChannelFile); + fs.writeFileSync(channelFilePath, newChannelFileRaw); + echo(`👌 >>> Channel file updated successfully. New channel file:`); + echo(newChannelFileRaw); + } catch (e) { + console.log(e); + } + } + + async function hashFile( + file, + algorithm = 'sha512', + encoding = 'base64', + options + ) { + const crypto = require('crypto'); + return await new Promise((resolve, reject) => { + const hash = crypto.createHash(algorithm); + hash.on('error', reject).setEncoding(encoding); + fs.createReadStream( + file, + Object.assign({}, options, { + highWaterMark: 1024 * 1024, + /* better to use more memory but hash faster */ + }) + ) + .on('error', reject) + .on('end', () => { + hash.end(); + resolve(hash.read()); + }) + .pipe(hash, { + end: false, + }); + }); + } + /** * Joins tha path from `__dirname`. */ diff --git a/electron/packager/package.json b/electron/packager/package.json index 91414cac7..e1f6c72df 100644 --- a/electron/packager/package.json +++ b/electron/packager/package.json @@ -17,6 +17,7 @@ "@types/file-type": "^10.9.1", "@types/temp": "^0.8.32", "chai": "^4.2.0", + "crypto": "^1.0.1", "dateformat": "^3.0.3", "deepmerge": "2.01", "depcheck": "^0.9.2", @@ -28,6 +29,7 @@ "shelljs": "^0.8.3", "sinon": "^9.0.1", "temp": "^0.9.1", + "yaml": "^1.10.2", "yargs": "^12.0.5" }, "engines": { diff --git a/electron/packager/utils.js b/electron/packager/utils.js index d746a98f3..01be7f92e 100644 --- a/electron/packager/utils.js +++ b/electron/packager/utils.js @@ -197,15 +197,9 @@ function git(command) { // to work correctly. // For more information: https://www.electron.build/auto-update function getChannelFile(platform) { - let currentChannel = ''; - if (isNightly) { - currentChannel = 'beta'; - } else if (isRelease) { + let currentChannel = 'beta'; + if (isRelease) { currentChannel = 'latest'; - } else { - // We're not creating a nightly build nor releasing - // a new version, no need for a channel file. - return ''; } return ( currentChannel + diff --git a/electron/packager/yarn.lock b/electron/packager/yarn.lock index 473faa674..460fd9699 100644 --- a/electron/packager/yarn.lock +++ b/electron/packager/yarn.lock @@ -437,6 +437,11 @@ cross-spawn@^6.0.0: shebang-command "^1.2.0" which "^1.2.9" +crypto@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/crypto/-/crypto-1.0.1.tgz#2af1b7cad8175d24c8a1b0778255794a21803037" + integrity sha512-VxBKmeNcqQdiUQUW2Tzq0t377b54N2bMtXO/qiLa+6eRRmmC4qT3D4OnTGoT/U6O9aklQ/jTwbOtRMTTY8G0Ig== + dateformat@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" @@ -1597,6 +1602,11 @@ wrappy@1: resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== +yaml@^1.10.2: + version "1.10.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b" + integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== + yargs-parser@13.1.2, yargs-parser@^13.1.2: version "13.1.2" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-13.1.2.tgz#130f09702ebaeef2650d54ce6e3e5706f7a4fb38"