Skip to content

Commit 23eddd5

Browse files
mattsb42-awsmattsb42-aws
committed
add ci/cd pipeline build/deploy tool
1 parent 21bed1c commit 23eddd5

File tree

2 files changed

+362
-0
lines changed

2 files changed

+362
-0
lines changed

‎decrypt_oracle/.chalice/pipeline.py

Lines changed: 351 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,351 @@
1+
"""
2+
Generate the CloudFormation template for the deployment pipeline.
3+
"""
4+
import argparse
5+
import getpass
6+
import logging
7+
from typing import Iterable
8+
9+
import boto3
10+
import troposphere
11+
from awacs import (
12+
aws as AWS,
13+
awslambda as LAMBDA,
14+
cloudformation as CLOUDFORMATION,
15+
cloudwatch as CLOUDWATCH,
16+
codebuild as CODEBUILD,
17+
codepipeline as CODEPIPELINE,
18+
iam as IAM,
19+
logs as LOGS,
20+
s3 as S3,
21+
sts as STS,
22+
)
23+
from botocore.exceptions import ClientError
24+
from troposphere import GetAtt, Ref, Sub, Template, codebuild, codepipeline, iam, s3
25+
26+
APPLICATION_NAME = "AwsEncryptionSdkDecryptOraclePython"
27+
PIPELINE_STACK_NAME = "{}DeployPipeline".format(APPLICATION_NAME)
28+
CODEBUILD_IMAGE = "aws/codebuild/python:3.6.5"
29+
BUILDSPEC = "decrypt_oracle/.chalice/buildspec.yaml"
30+
GITHUB_REPO = "aws-encryption-sdk-python"
31+
WAITER_CONFIG = dict(Delay=10)
32+
_LOGGER = logging.getLogger("Decrypt Oracle Build Pipeline Deployer")
33+
34+
35+
class AllowEverywhere(AWS.Statement):
36+
def __init__(self, *args, **kwargs):
37+
my_kwargs = dict(Effect=AWS.Allow, Resource=["*"])
38+
my_kwargs.update(kwargs)
39+
super(AllowEverywhere, self).__init__(*args, **my_kwargs)
40+
41+
42+
def _service_assume_role(service: str) -> AWS.Policy:
43+
""""""
44+
return AWS.Policy(
45+
Statement=[
46+
AWS.Statement(
47+
Effect=AWS.Allow,
48+
Action=[STS.AssumeRole],
49+
Principal=AWS.Principal("Service", ["{}.amazonaws.com".format(service)]),
50+
)
51+
]
52+
)
53+
54+
55+
def _codebuild_role() -> iam.Role:
56+
""""""
57+
policy = iam.Policy(
58+
"CodeBuildPolicy",
59+
PolicyName="CodeBuildPolicy",
60+
PolicyDocument=AWS.PolicyDocument(
61+
Statement=[
62+
AllowEverywhere(Action=[LOGS.CreateLogGroup, LOGS.CreateLogStream, LOGS.PutLogEvents]),
63+
AllowEverywhere(Action=[S3.GetObject, S3.GetObjectVersion, S3.PutObject]),
64+
]
65+
),
66+
)
67+
return iam.Role("CodeBuildRole", AssumeRolePolicyDocument=_service_assume_role(CODEBUILD.prefix), Policies=[policy])
68+
69+
70+
def _codebuild_builder(role: iam.Role, application_bucket: s3.Bucket) -> codebuild.Project:
71+
""""""
72+
artifacts = codebuild.Artifacts(Type="CODEPIPELINE")
73+
environment = codebuild.Environment(
74+
ComputeType="BUILD_GENERAL1_SMALL",
75+
Image=CODEBUILD_IMAGE,
76+
Type="LINUX_CONTAINER",
77+
EnvironmentVariables=[codebuild.EnvironmentVariable(Name="APP_S3_BUCKET", Value=Ref(application_bucket))],
78+
)
79+
source = codebuild.Source(Type="CODEPIPELINE", BuildSpec=BUILDSPEC)
80+
return codebuild.Project(
81+
"AppPackageBuild",
82+
Artifacts=artifacts,
83+
Environment=environment,
84+
Name="{}Build".format(APPLICATION_NAME),
85+
ServiceRole=Ref(role),
86+
Source=source,
87+
)
88+
89+
90+
def _pipeline_role(buckets: Iterable[s3.Bucket]) -> iam.Role:
91+
""""""
92+
bucket_statements = [
93+
AWS.Statement(
94+
Effect=AWS.Allow,
95+
Action=[S3.GetBucketVersioning, S3.PutBucketVersioning],
96+
Resource=[GetAtt(bucket, "Arn") for bucket in buckets],
97+
),
98+
AWS.Statement(
99+
Effect=AWS.Allow,
100+
Action=[S3.GetObject, S3.PutObject],
101+
Resource=[Sub("${{{bucket}.Arn}}/*".format(bucket=bucket.title)) for bucket in buckets],
102+
),
103+
]
104+
policy = iam.Policy(
105+
"PipelinePolicy",
106+
PolicyName="PipelinePolicy",
107+
PolicyDocument=AWS.PolicyDocument(
108+
Statement=bucket_statements
109+
+ [
110+
AllowEverywhere(Action=[CLOUDWATCH.Action("*"), IAM.PassRole]),
111+
AllowEverywhere(Action=[LAMBDA.InvokeFunction, LAMBDA.ListFunctions]),
112+
AllowEverywhere(
113+
Action=[
114+
CLOUDFORMATION.CreateStack,
115+
CLOUDFORMATION.DeleteStack,
116+
CLOUDFORMATION.DescribeStacks,
117+
CLOUDFORMATION.UpdateStack,
118+
CLOUDFORMATION.CreateChangeSet,
119+
CLOUDFORMATION.DeleteChangeSet,
120+
CLOUDFORMATION.DescribeChangeSet,
121+
CLOUDFORMATION.ExecuteChangeSet,
122+
CLOUDFORMATION.SetStackPolicy,
123+
CLOUDFORMATION.ValidateTemplate,
124+
]
125+
),
126+
AllowEverywhere(Action=[CODEBUILD.BatchGetBuilds, CODEBUILD.StartBuild]),
127+
]
128+
),
129+
)
130+
return iam.Role(
131+
"CodePipelinesRole", AssumeRolePolicyDocument=_service_assume_role(CODEPIPELINE.prefix), Policies=[policy]
132+
)
133+
134+
135+
def _cloudformation_role() -> iam.Role:
136+
""""""
137+
policy = iam.Policy(
138+
"CloudFormationPolicy",
139+
PolicyName="CloudFormationPolicy",
140+
PolicyDocument=AWS.PolicyDocument(Statement=[AllowEverywhere(Action=[AWS.Action("*")])]),
141+
)
142+
return iam.Role(
143+
"CloudFormationRole", AssumeRolePolicyDocument=_service_assume_role(CLOUDFORMATION.prefix), Policies=[policy]
144+
)
145+
146+
147+
def _pipeline(
148+
pipeline_role: iam.Role,
149+
cfn_role: iam.Role,
150+
codebuild_builder: codebuild.Project,
151+
artifact_bucket: s3.Bucket,
152+
github_owner: str,
153+
github_branch: str,
154+
github_access_token: troposphere.AWSProperty,
155+
) -> codepipeline.Pipeline:
156+
""""""
157+
_source_output = "SourceOutput"
158+
get_source = codepipeline.Stages(
159+
Name="Source",
160+
Actions=[
161+
codepipeline.Actions(
162+
Name="PullSource",
163+
RunOrder="1",
164+
OutputArtifacts=[codepipeline.OutputArtifacts(Name=_source_output)],
165+
ActionTypeId=codepipeline.ActionTypeId(
166+
Category="Source", Owner="ThirdParty", Version="1", Provider="GitHub"
167+
),
168+
Configuration=dict(
169+
Owner=github_owner,
170+
Repo=GITHUB_REPO,
171+
OAuthToken=Ref(github_access_token),
172+
Branch=github_branch,
173+
PollForSourceChanges=True,
174+
),
175+
)
176+
],
177+
)
178+
_compiled_cfn_template = "CompiledCfnTemplate"
179+
_changeset_name = "{}ChangeSet".format(APPLICATION_NAME)
180+
_stack_name = "{}Stack".format(APPLICATION_NAME)
181+
do_build = codepipeline.Stages(
182+
Name="Build",
183+
Actions=[
184+
codepipeline.Actions(
185+
Name="BuildChanges",
186+
RunOrder="1",
187+
InputArtifacts=[codepipeline.InputArtifacts(Name=_source_output)],
188+
OutputArtifacts=[codepipeline.OutputArtifacts(Name=_compiled_cfn_template)],
189+
ActionTypeId=codepipeline.ActionTypeId(
190+
Category="Build", Owner="AWS", Version="1", Provider="CodeBuild"
191+
),
192+
Configuration=dict(ProjectName=Ref(codebuild_builder)),
193+
)
194+
],
195+
)
196+
stage_changeset = codepipeline.Actions(
197+
Name="StageChanges",
198+
RunOrder="1",
199+
ActionTypeId=codepipeline.ActionTypeId(Category="Deploy", Owner="AWS", Version="1", Provider="CloudFormation"),
200+
InputArtifacts=[codepipeline.InputArtifacts(Name=_compiled_cfn_template)],
201+
Configuration=dict(
202+
ActionMode="CHANGE_SET_REPLACE",
203+
ChangeSetName=_changeset_name,
204+
RoleArn=GetAtt(cfn_role, "Arn"),
205+
Capabilities="CAPABILITY_IAM",
206+
StackName=_stack_name,
207+
TemplatePath="{}::transformed.yaml".format(_compiled_cfn_template),
208+
),
209+
)
210+
deploy_changeset = codepipeline.Actions(
211+
Name="Deploy",
212+
RunOrder="2",
213+
ActionTypeId=codepipeline.ActionTypeId(Category="Deploy", Owner="AWS", Version="1", Provider="CloudFormation"),
214+
Configuration=dict(
215+
ActionMode="CHANGE_SET_EXECUTE",
216+
ChangeSetName=_changeset_name,
217+
StackName=_stack_name,
218+
OutputFileName="StackOutputs.json",
219+
),
220+
OutputArtifacts=[codepipeline.OutputArtifacts(Name="AppDeploymentValues")],
221+
)
222+
deploy = codepipeline.Stages(Name="Deploy", Actions=[stage_changeset, deploy_changeset])
223+
artifact_store = codepipeline.ArtifactStore(Type="S3", Location=Ref(artifact_bucket))
224+
return codepipeline.Pipeline(
225+
"{}Pipeline".format(APPLICATION_NAME),
226+
RoleArn=GetAtt(pipeline_role, "Arn"),
227+
ArtifactStore=artifact_store,
228+
Stages=[get_source, do_build, deploy],
229+
)
230+
231+
232+
def _build_template(github_owner: str, github_branch: str) -> Template:
233+
""""""
234+
template = Template(Description="CI/CD pipeline for Decrypt Oracle powered by the AWS Encryption SDK for Python")
235+
github_access_token = template.add_parameter(
236+
troposphere.Parameter(
237+
"GithubPersonalToken", Type="String", Description="Personal access token for the github repo.", NoEcho=True
238+
)
239+
)
240+
application_bucket = template.add_resource(s3.Bucket("ApplicationBucket"))
241+
artifact_bucket = template.add_resource(s3.Bucket("ArtifactBucketStore"))
242+
builder_role = template.add_resource(_codebuild_role())
243+
builder = template.add_resource(_codebuild_builder(builder_role, application_bucket))
244+
# add codepipeline role
245+
pipeline_role = template.add_resource(_pipeline_role(buckets=[application_bucket, artifact_bucket]))
246+
# add cloudformation deploy role
247+
cfn_role = template.add_resource(_cloudformation_role())
248+
# add codepipeline
249+
template.add_resource(
250+
_pipeline(
251+
pipeline_role=pipeline_role,
252+
cfn_role=cfn_role,
253+
codebuild_builder=builder,
254+
artifact_bucket=artifact_bucket,
255+
github_owner=github_owner,
256+
github_branch=github_branch,
257+
github_access_token=github_access_token,
258+
)
259+
)
260+
return template
261+
262+
263+
def _stack_exists(cloudformation) -> bool:
264+
"""Determine if the stack has already been deployed."""
265+
try:
266+
cloudformation.describe_stacks(StackName=PIPELINE_STACK_NAME)
267+
268+
except ClientError as error:
269+
if error.response["Error"]["Message"] == "Stack with id {name} does not exist".format(name=PIPELINE_STACK_NAME):
270+
return False
271+
raise
272+
273+
else:
274+
return True
275+
276+
277+
def _update_existing_stack(cloudformation, template: Template, github_token: str) -> None:
278+
"""Update a stack."""
279+
_LOGGER.info("Updating existing stack")
280+
281+
# 3. update stack
282+
cloudformation.update_stack(
283+
StackName=PIPELINE_STACK_NAME,
284+
TemplateBody=template.to_json(),
285+
Parameters=[dict(ParameterKey="GithubPersonalToken", ParameterValue=github_token)],
286+
Capabilities=["CAPABILITY_IAM"],
287+
)
288+
_LOGGER.info("Waiting for stack update to complete...")
289+
waiter = cloudformation.get_waiter("stack_update_complete")
290+
waiter.wait(StackName=PIPELINE_STACK_NAME, WaiterConfig=WAITER_CONFIG)
291+
_LOGGER.info("Stack update complete!")
292+
293+
294+
def _deploy_new_stack(cloudformation, template: Template, github_token: str) -> None:
295+
"""Deploy a new stack."""
296+
_LOGGER.info("Bootstrapping new stack")
297+
298+
# 2. deploy template
299+
cloudformation.create_stack(
300+
StackName=PIPELINE_STACK_NAME,
301+
TemplateBody=template.to_json(),
302+
Parameters=[dict(ParameterKey="GithubPersonalToken", ParameterValue=github_token)],
303+
Capabilities=["CAPABILITY_IAM"],
304+
)
305+
_LOGGER.info("Waiting for stack to deploy...")
306+
waiter = cloudformation.get_waiter("stack_create_complete")
307+
waiter.wait(StackName=PIPELINE_STACK_NAME, WaiterConfig=WAITER_CONFIG)
308+
_LOGGER.info("Stack deployment complete!")
309+
310+
311+
def _deploy_or_update_template(template: Template, github_token: str) -> None:
312+
"""Update a stack, deploying a new stack if nothing exists yet."""
313+
cloudformation = boto3.client("cloudformation")
314+
315+
if _stack_exists(cloudformation):
316+
return _update_existing_stack(
317+
cloudformation=cloudformation,
318+
template=template,
319+
github_token=github_token,
320+
)
321+
322+
return _deploy_new_stack(
323+
cloudformation=cloudformation,
324+
template=template,
325+
github_token=github_token,
326+
)
327+
328+
329+
def _setup_logging() -> None:
330+
"""Set up logging."""
331+
logging.basicConfig(level=logging.INFO)
332+
333+
334+
def main(args=None):
335+
"""Entry point for CLI."""
336+
_setup_logging()
337+
338+
parser = argparse.ArgumentParser(description="Pipeline deployer")
339+
parser.add_argument("--github-user", required=True, help="What Github user should be used?")
340+
parser.add_argument("--github-branch", required=False, default="master", help="What Github branch should be used?")
341+
342+
parsed = parser.parse_args(args)
343+
344+
access_token = getpass.getpass("Github personal token:")
345+
346+
template = _build_template(github_owner=parsed.github_user, github_branch=parsed.github_branch)
347+
_deploy_or_update_template(template=template, github_token=access_token)
348+
349+
350+
if __name__ == "__main__":
351+
main()

‎decrypt_oracle/tox.ini

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,15 @@ envlist =
3434
# release :: Builds dist files and uploads to pypi pypirc profile.
3535

3636

37+
[testenv:generate-pipeline]
38+
basepython = python3
39+
skip_install = true
40+
deps =
41+
troposphere[policy]
42+
boto3
43+
commands = python .chalice/pipeline.py {posargs}
44+
45+
3746
[testenv:chalice-prep]
3847
basepython = python3.6
3948
skip_install = true
@@ -201,6 +210,7 @@ commands =
201210
setup.py \
202211
#doc/conf.py \
203212
test/ \
213+
.chalice/pipeline.py \
204214
{posargs}
205215

206216

@@ -231,6 +241,7 @@ commands = isort -rc \
231241
test \
232242
#doc \
233243
setup.py \
244+
.chalice/pipeline.py \
234245
{posargs}
235246

236247
[testenv:isort-check]

0 commit comments

Comments
 (0)