diff --git a/tools/releaseBuild/yaml/compliance.yml b/tools/releaseBuild/yaml/compliance.yml new file mode 100644 index 0000000..354eeb9 --- /dev/null +++ b/tools/releaseBuild/yaml/compliance.yml @@ -0,0 +1,57 @@ +steps: + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 + displayName: 'Run BinSkim ' + inputs: + InputType: Basic + AnalyzeTarget: '$(System.ArtifactsDirectory)\uncompressed\*.dll;$(System.ArtifactsDirectory)\uncompressed\*.exe' + AnalyzeSymPath: 'SRV*http://symweb' + AnalyzeVerbose: true + AnalyzeHashes: true + AnalyzeStatistics: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 + displayName: 'Run MpCmdRun.exe' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-apiscan.APIScan@1 + displayName: 'Run APIScan' + inputs: + softwareFolder: '$(System.ArtifactsDirectory)\uncompressed' + softwareName: PowerShell + softwareVersionNum: '$(NugetPackageVersion)' + isLargeApp: false + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 + displayName: 'Run CredScan' + inputs: + debugMode: false + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 + displayName: 'Run PoliCheck' + inputs: + targetType: F + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 + displayName: 'Publish Security Analysis Logs' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1 + displayName: 'TSA upload to Codebase: PowerShellNative Stamp: Azure' + inputs: + codebase: NewOrUpdate + tsaStamp: Azure + codeBaseName: PowerShellNative + notificationAlias: '4e50053a.microsoft.com@amer.teams.ms' + codeBaseAdmins: 'adityap;tplunk;dongbow;slee' + instanceUrlAzure: MSAZURE + projectNameMSAZURE: One + areaPath: 'One\MGMT\Azure Automation\PowerShell\PowerShell Core' + iterationPath: 'One\_Default' + uploadFortifySCA: false + uploadFxCop: false + uploadModernCop: false + uploadPREfast: false + uploadRoslyn: false + uploadTSLint: false + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' diff --git a/tools/releaseBuild/yaml/linux.yml b/tools/releaseBuild/yaml/linux.yml new file mode 100644 index 0000000..39151db --- /dev/null +++ b/tools/releaseBuild/yaml/linux.yml @@ -0,0 +1,7 @@ +steps: +- task: PowerShell@2 + displayName: 'Build native assemblies for Linux' + inputs: + targetType: filePath + filePath: ./tools/releaseBuild/vstsBuild.ps1 + arguments: '-Name $(LINUX_BUILDNAME) -Verbose' diff --git a/tools/releaseBuild/yaml/mac.yml b/tools/releaseBuild/yaml/mac.yml new file mode 100644 index 0000000..7ec28fd --- /dev/null +++ b/tools/releaseBuild/yaml/mac.yml @@ -0,0 +1,18 @@ +steps: +- task: PowerShell@2 + displayName: 'Build native assemblies' + inputs: + targetType: filePath + filePath: ./tools/releaseBuild/PowershellNative.ps1 + arguments: '-Arch osx -Configuration Release -RepoRoot $(Build.SourcesDirectory) -TargetLocation $(PackageRoot)' + +- powershell: | + $filePath = "$(PackageRoot)/osx-symbols.zip" + Write-Verbose "Artifact to upload: $filePath" -Verbose + Write-Host "##vso[artifact.upload containerfolder=release;artifactname=release]$filePath" + displayName: 'Upload artifact' + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)' diff --git a/tools/releaseBuild/yaml/nuget.yml b/tools/releaseBuild/yaml/nuget.yml new file mode 100644 index 0000000..bf106f3 --- /dev/null +++ b/tools/releaseBuild/yaml/nuget.yml @@ -0,0 +1,164 @@ +steps: +- task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: release + targetPath: "$(System.ArtifactsDirectory)/release" + displayName: Download native packages + +- task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: signed + targetPath: "$(System.ArtifactsDirectory)/signed" + displayName: Download signed windows native packages + +- powershell: | + $null = New-Item $(PackageRoot) -ItemType Directory -Force -Verbose + if(-not (Test-Path '$(System.ArtifactsDirectory)/release' )) + { + New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)/release' -Force + } + Invoke-WebRequest -Uri '$(PSRPBlobUrl)' -OutFile $(System.ArtifactsDirectory)/release/psrp.zip -Verbose + displayName: 'Download PSRP package' + +- powershell: 'Get-ChildItem $(System.ArtifactsDirectory)/release' + displayName: 'Capture downloaded zips' + +- powershell: | + $extractedRoot = New-Item $(System.ArtifactsDirectory)/uncompressed -ItemType Directory -Force -Verbose + + Get-ChildItem $(System.ArtifactsDirectory)/release/*.zip | ForEach-Object { + $baseName = $_.BaseName + if($baseName -match 'x64_arm') { + Write-Verbose "Skipping expanding file $_.Name" -Verbose + } + else { + $folderPath = Join-Path $extractedRoot $baseName + Expand-Archive $_.FullName -DestinationPath $folderPath -Force + } + } + + Write-Host "Extracted files:" + Get-ChildItem -Recurse $extractedRoot -File + displayName: 'Extract All Zips' + +- template: compliance.yml + +- template: upload.yml + parameters: + fileName: 'x64_arm64-symbols.zip' + sourcePath: '$(System.ArtifactsDirectory)\release\x64_arm64-symbols.zip' +- template: upload.yml + parameters: + fileName: 'x64-symbols.zip' + sourcePath: '$(System.ArtifactsDirectory)\release\x64-symbols.zip' +- template: upload.yml + parameters: + fileName: 'x64_arm-symbols.zip' + sourcePath: '$(System.ArtifactsDirectory)\release\x64_arm-symbols.zip' +- template: upload.yml + parameters: + fileName: 'x86-symbols.zip' + sourcePath: '$(System.ArtifactsDirectory)\release\x86-symbols.zip' + +- task: NuGetToolInstaller@0 + displayName: 'Install NuGet 5.3.1' + inputs: + versionSpec: 5.3.1 + +- powershell: | + $Branch = $env:BUILD_SOURCEBRANCH + $branchOnly = $Branch -replace '^refs/heads/'; + $branchOnly = $branchOnly -replace '[_\-]' + + $packageVersion = if($env:PACKAGE_VERSION -eq 'fromBranch' -or !$env:PACKAGE_VERSION) + { + if($Branch -match '^.*(release[-/])') + { + Write-verbose "release branch:" -verbose + $Branch -replace '^.*(release[-/]v)' + } + else + { + "0.0.0" + } + } + else { + "0.0.0" + } + + $vstsCommandString = "vso[task.setvariable variable=NugetPackageVersion]$packageVersion" + Write-Verbose -Message "setting $Variable to $packageVersion" -Verbose + Write-Host -Object "##$vstsCommandString" + + displayName: Set NuGetPackageVersion + +- powershell: | + Import-Module $(Build.SourcesDirectory)\build.psm1 + $PackageRoot = New-Item -ItemType Directory -Path $(System.ArtifactsDirectory)\NugetPackage + + $WindowsX64ZipPath = Join-Path "$(System.ArtifactsDirectory)/signed" 'win-x64.zip' + $WindowsX86ZipPath = Join-Path "$(System.ArtifactsDirectory)/signed" 'win-x86.zip' + $WindowsARMZipPath = Join-Path "$(System.ArtifactsDirectory)/signed" 'win-x64_arm.zip' + $WindowsARM64ZipPath = Join-Path "$(System.ArtifactsDirectory)/signed" 'win-x64_arm64.zip' + $LinuxZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'linux-x64-symbols.zip' + $LinuxARMZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'linux-arm-symbols.zip' + $LinuxARM64ZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'linux-arm64-symbols.zip' + $LinuxAlpineZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'linux-musl-x64-symbols.zip' + $macOSZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'osx-symbols.zip' + $psrpZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'psrp.zip' + + Start-BuildPowerShellNativePackage -PackageRoot $PackageRoot -Version $(NugetPackageVersion) -WindowsX64ZipPath $WindowsX64ZipPath -WindowsX86ZipPath $WindowsX86ZipPath -WindowsARMZipPath $WindowsARMZipPath -WindowsARM64ZipPath $WindowsARM64ZipPath -LinuxZipPath $LinuxZipPath -LinuxARMZipPath $LinuxARMZipPath -LinuxARM64ZipPath $LinuxARM64ZipPath -LinuxAlpineZipPath $LinuxAlpineZipPath -macOSZipPath $macOSZipPath -psrpZipPath $psrpZipPath -NuGetOutputPath $(NuGetPackagePath) + + displayName: 'Build NuGet package' + +- powershell: 'Get-Childitem $(NuGetPackagePath)' + displayName: 'Capture nuget package' + +- powershell: | + ## Get the nuget file paths + $files = (Get-ChildItem $(NuGetPackagePath)\*.nupkg).FullName + + if($files.Count -lt 1) { throw "No packages available to publish" } + + & $(Build.SourcesDirectory)\tools\releaseBuild\generatePackgeSigning.ps1 -Path "$(NuGetPackagePath)\NugetSigning.xml" -NuPkgFiles $files + + displayName: 'Generate signing file' + +- powershell: 'Get-Content $(NuGetPackagePath)\NugetSigning.xml' + displayName: 'Capture package signing file' + +- task: PkgESCodeSign@10 + displayName: 'Package ES - CodeSign $(NuGetPackagePath)\NugetSigning.xml' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + signConfigXml: '$(NuGetPackagePath)\NugetSigning.xml' + inPathRoot: ' $(NuGetPackagePath)' + outPathRoot: '$(System.ArtifactsDirectory)\signed' + condition: ne(variables['SKIP_SIGNING'], 'True') + +- powershell: | + Copy-Item $(NuGetPackagePath)\*.nupkg -DestinationPath '$(System.ArtifactsDirectory)\signed' -Force -Verbose + displayName: Copy unsigned nuget packge as signing is disabled + condition: eq(variables['SKIP_SIGNING'], 'True') + +- powershell: | + Import-Module $env:BUILD_SOURCESDIRECTORY\build.psm1 -Force + + $packages = Get-ChildItem -Recurse $(System.ArtifactsDirectory)\signed\*.nupkg -Verbose + + if ($packages.Count -eq 0) + { + throw "No signed package found at: $(System.ArtifactsDirectory)\signed" + } + + $packages | ForEach-Object { Start-NativeExecution -sb { nuget.exe verify -All $_.FullName } } + displayName: 'Validate NuGet package is signed' + +- template: uploadArtifact.yml + parameters: + artifactPath: '$(System.ArtifactsDirectory)\signed' + artifactFilter: 'Microsoft.PowerShell.Native.$(NugetPackageVersion).nupkg' + artifactName: 'finalResults' diff --git a/tools/releaseBuild/yaml/publish.yml b/tools/releaseBuild/yaml/publish.yml new file mode 100644 index 0000000..94a2625 --- /dev/null +++ b/tools/releaseBuild/yaml/publish.yml @@ -0,0 +1,40 @@ +parameters: + stageName: '' + environmentName: '' + feedCredential: '' + +stages: +- stage: ${{ parameters.stageName }} + displayName: Release PowerShell-Native to '${{ parameters.stageName }}' + condition: and(succeeded('Build'), eq(variables['Build.Reason'], 'Manual'), startsWith(variables['Build.SourceBranch'], 'refs/heads/release/')) + + jobs: + - deployment: Publish_${{ parameters.stageName }} + displayName: Release to Feed + pool: + vmImage: windows-latest + + environment: ${{ parameters.environmentName }} + strategy: + runOnce: + deploy: + steps: + - download: current + artifact: release + + - powershell: | + Get-ChildItem -Recurse "$(Pipeline.Workspace)/release" + displayName: Capture downloaded artifact + + - powershell: | + Copy-Item "$(Pipeline.Workspace)/release/Microsoft.PowerShell.Native.*.nupkg" -Destination "$(Build.ArtifactStagingDirectory)" -Force + Get-ChildItem "$(Build.ArtifactStagingDirectory)/Microsoft.PowerShell.Native.*.nupkg" + displayName: 'Copy and Capture NuGet package' + + - task: NuGetCommand@2 + displayName: 'NuGet push' + inputs: + command: push + packagesToPush: '$(Build.ReleaseDirectory)/Microsoft.PowerShell.Native.*.nupkg' + nuGetFeedType: external + publishFeedCredentials: ${{ parameters.feedCredential }} diff --git a/tools/releaseBuild/yaml/releaseBuild.yml b/tools/releaseBuild/yaml/releaseBuild.yml new file mode 100644 index 0000000..2330cf2 --- /dev/null +++ b/tools/releaseBuild/yaml/releaseBuild.yml @@ -0,0 +1,80 @@ +variables: + AuthenticodeSignType: '400' + BuildConfiguration: 'Release' + NuGetPackagePath: '$(System.ArtifactsDirectory)/NuGetRelease' + PackageRoot: '$(System.ArtifactsDirectory)/Packages' + +stages: +- stage: Build + displayName: Build Native Binaries + dependsOn: [] + jobs: + - job: BuildWin + pool: + name: PowerShell + demands: + - DotNetFramework + - Agent.Image + displayName: Windows + strategy: + matrix: + x64: + ARCHITECTURE: x64 + x86: + ARCHITECTURE: x86 + x64ARM: + ARCHITECTURE: x64_arm + x64ARM64: + ARCHITECTURE: x64_arm64 + steps: + - template: windows.yml + + - job: BuildLinux + displayName: Linux + pool: + vmImage: ubuntu-16.04 + strategy: + matrix: + UbuntuArm: + LINUX_BUILDNAME: 'ubuntu.16.04-arm' + CentOS: + LINUX_BUILDNAME: 'centos.7' + Alpine: + LINUX_BUILDNAME: 'alpine' + UbuntuArm64: + LINUX_BUILDNAME: 'ubuntu.16.04-arm64' + steps: + - template: linux.yml + + - job: BuildMac + displayName: Build Native Binaries on macOS + pool: Hosted Mac Internal + steps: + - template: mac.yml + + - job: BuildNuGetPkg + displayName: Build NuGet Package + dependsOn: + - BuildWin + - BuildLinux + - BuildMac + pool: + name: PowerShell + demands: + - DotNetFramework + - Agent.Image + + steps: + - template: nuget.yml + +- template: publish.yml + parameters: + stageName: AzArtifactsFeed + environmentName: PSNativeAzArtifactApproval + feedCredential: AzArtifactsFeed + +- template: publish.yml + parameters: + stageName: NuGet + environmentName: PSNativeNuGetApproval + feedCredential: PowerShellNuGetOrgPush diff --git a/tools/releaseBuild/yaml/upload.yml b/tools/releaseBuild/yaml/upload.yml new file mode 100644 index 0000000..256bde8 --- /dev/null +++ b/tools/releaseBuild/yaml/upload.yml @@ -0,0 +1,14 @@ +parameters: + fileName: '' + sourcePath: '' + +steps: +- task: AzureFileCopy@2 + displayName: 'Upload ${{ parameters.fileName }} to azure blob' + inputs: + SourcePath: '${{ parameters.sourcePath }}' + azureSubscription: 'MGMT-Powershell-CICDInfra (94cf12ad-4fe9-490b-b281-0a260198a4e0)' + Destination: AzureBlob + storage: pscoretestdata + ContainerName: 'PowerShell-Native-Symbols' + BlobPrefix: '$(Build.SourceBranchName)' diff --git a/tools/releaseBuild/yaml/uploadArtifact.yml b/tools/releaseBuild/yaml/uploadArtifact.yml new file mode 100644 index 0000000..001b86b --- /dev/null +++ b/tools/releaseBuild/yaml/uploadArtifact.yml @@ -0,0 +1,16 @@ +parameters: + artifactPath: + artifactFilter: '*' + artifactName: 'finalResults' + condition: succeeded() + +steps: + - powershell: | + Get-ChildItem -Path '${{ parameters.artifactPath }}' -File -filter '${{ parameters.artifactFilter }}' -ErrorAction SilentlyContinue | + Select-Object -ExpandProperty FullName | + ForEach-Object { + Write-Verbose 'Uploading $_' -Verbose + Write-Host "##vso[artifact.upload containerfolder=${{ parameters.artifactName }};artifactname=${{ parameters.artifactName }}]$_" + } + displayName: Upload Final Artifacts ${{ parameters.artifactFilter }} from ${{ parameters.artifactPath }} + condition: ${{ parameters.condition }} diff --git a/tools/releaseBuild/yaml/windows.yml b/tools/releaseBuild/yaml/windows.yml new file mode 100644 index 0000000..da55144 --- /dev/null +++ b/tools/releaseBuild/yaml/windows.yml @@ -0,0 +1,94 @@ +steps: +- task: PkgESSetupBuild@10 + displayName: 'Initialize build' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + useDfs: false + productName: PowerShellCore + branchVersion: true + disableWorkspace: true + disableBuildTools: true + disableNugetPack: true + condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) + +- powershell: | + docker container prune --force + docker container ls --all --format '{{ json .ID }}' | ConvertFrom-Json | %{docker container rm --force --volumes $_} + docker image ls --format '{{ json .}}' | ConvertFrom-Json |% { + if($_.tag -eq '') + { + $formatString = 'yyyy-MM-dd HH:mm:ss zz00' + $createdAtString = $_.CreatedAt.substring(0,$_.CreatedAt.Length -4) + $createdAt = [DateTime]::ParseExact($createdAtString, $formatString,[System.Globalization.CultureInfo]::InvariantCulture) + if($createdAt -lt (Get-Date).adddays(-1)) + { + docker image rm $_.ID + } + } + } + + Write-verbose "--docker info---" -verbose + docker info + Write-verbose "--docker image ls---" -verbose + docker image ls + Write-verbose "--docker container ls --all---" -verbose + docker container ls --all + Write-verbose "--git branch ---" -verbose + git branch + exit 0 + displayName: 'Docker cleanup and get environment' + continueOnError: true + +- task: PowerShell@2 + displayName: 'Build Windows Universal' + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)/tools/releaseBuild/vstsBuild.ps1 + arguments: '-Name win-$(ARCHITECTURE)-symbols -Verbose' + +- task: PowerShell@2 + displayName: 'Update Signing Xml' + inputs: + targetType: filePath + filePath: $(Build.SourcesDirectory)/tools/releaseBuild/updateSigning.ps1 + +- powershell: | + $vstsCommandString = "vso[task.setvariable variable=Symbols]${env:Symbols_$(ARCHITECTURE)}" + Write-Host "sending " + $vstsCommandString + Write-Host "##$vstsCommandString" + displayName: 'Get Symbols Path' + +- task: PkgESCodeSign@10 + displayName: 'CodeSign $(ARCHITECTURE)' + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + signConfigXml: '$(Build.SourcesDirectory)\tools\releaseBuild\signing.xml' + inPathRoot: '$(Symbols)' + outPathRoot: '$(Symbols)\Signed' + condition: ne(variables['SKIP_SIGNING'], 'True') + +- powershell: | + Compress-Archive -Path '$(Symbols)\Signed\*' -DestinationPath '$(Symbols)\Signed\win-$(ARCHITECTURE).zip' + displayName: Compress signed binaries + condition: ne(variables['SKIP_SIGNING'], 'True') + +- powershell: | + Get-ChildItem -Path '$(Symbols)\*' -Recurse | Copy-Item -Destination '$(Symbols)\Signed' -Force -Verbose + displayName: Copy unsigned binaries as signing is skipped + condition: eq(variables['SKIP_SIGNING'], 'True') + +- template: uploadArtifact.yml + parameters: + artifactPath: '$(Symbols)\Signed' + artifactFilter: 'win-*.zip' + artifactName: 'signed' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-antimalware.AntiMalware@3 + displayName: 'Run MpCmdRun.exe' + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection' + inputs: + sourceScanPath: '$(Build.SourcesDirectory)'