diff --git a/.gitignore b/.gitignore index 953c625..ad4f921 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +*.ini +*.pyc *.retry *.tmp *~ @@ -7,4 +9,7 @@ /terraform/.terraform* /terraform/terraform.tfstate* /terraform/tf.plan +__pycache__ build/ +jmeter.log +venv/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b60f136 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ + +FROM centos:latest AS infra-demo + +# setup rpm repos, install base packages and create virtual env in a single step +RUN yum install -y https://centos7.iuscommunity.org/ius-release.rpm \ + && yum update -y \ + && yum install -y \ + python36u python36u-libs python36u-devel \ + python36u-pip uwsgi-plugin-python36u uwsgi \ + gcc make glibc-devel kernel-headers \ + pcre pcre-devel pcre2 pcre2-devel \ + postgresql-devel \ + && yum clean all \ + && mkdir /app \ + && python3.6 -m venv --copies --clear /app/venv + +# Copy in your requirements file +ADD src/requirements.txt /app/requirements.txt + +# setup python packages +RUN /app/venv/bin/pip install -U pip \ + && /bin/sh -c "/app/venv/bin/pip install --no-cache-dir -r /app/requirements.txt" diff --git a/Jenkinsfile b/Jenkinsfile index 0b0219a..f1533ad 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ * Use the Scripted style of Jenkinsfile in order to * write more Groovy functions and use variables to * control the workflow. - */ + */ import java.util.Random @@ -18,7 +18,7 @@ def get_captcha(Long hash_const) { Random rand = new Random() def op1 = rand.nextInt(MAX+1) def op2 = rand.nextInt(MAX+1) + MAX - def op3 = rand.nextInt(MAX+1) + def op3 = rand.nextInt(MAX+1) def captcha_problem = "CAPTCHA problem: What is the answer to this problem: ${op1} + ${op2} - ${op3}" Long captcha_answer = op1 + op2 - op3 Long captcha_hash = captcha_answer ^ hash_const @@ -46,30 +46,67 @@ final Long XOR_CONST = 3735928559 // 0xdeadbeef properties([ parameters([ booleanParam( - name: 'Run_Packer', - defaultValue: false, + name: 'Run_Packer', + defaultValue: false, description: 'Run Packer for this build?' ), booleanParam( - name: 'Apply_Terraform', - defaultValue: false, + name: 'Apply_Terraform', + defaultValue: false, description: 'Apply Terraform plan on this build?' ), booleanParam( - name: 'Destroy_Terraform', - defaultValue: false, + name: 'Destroy_Terraform', + defaultValue: false, description: 'Destroy Terraform resources?' ), + string( + name: 'Terraform_Targets', + defaultValue: '', + description: '''Specific Terraform resource or resource names to target + (Use this to modify or delete less than the full set of resources''' + ), + text( + name: 'Extra_Variables', + defaultValue: '', + description: '''Terraform Variables to define for this run. + Allows you to override declared variables. + Put one variable per line, in JSON or HCL like this: + associate_public_ip_address = "true"''' + ), booleanParam( - name: 'Rotate_Servers', - defaultValue: false, + name: 'Rotate_Servers', + defaultValue: false, description: """Rotate server instances in Auto Scaling Group? You should do this if you changed ASG size or baked a new AMI. - """ + """ + ), + booleanParam( + name: 'Run_JMeter', + defaultValue: false, + description: "Execute a JMeter load test against the stack" + ), + string( + name: 'JMETER_threads', + defaultValue: '2', + description: """number of jmeter threads. Resulting ASG stable sizes for t2.large instances are: + - 2 threads, 3 instances; + - 4 threads, 7 instances; + """ + ), + string( + name: 'JMETER_ramp_duration', + defaultValue: '900', + description: 'period in seconds of ramp-up time.' + ), + string( + name: 'JMETER_duration', + defaultValue: '1800', + description: 'time in seconds to the whole Jmeter test' ), string( - name: 'CAPTCHA_Guess', - defaultValue: '', + name: 'CAPTCHA_Guess', + defaultValue: '', description: captcha_problem ), string( @@ -77,27 +114,13 @@ properties([ defaultValue: captcha_hash, description: 'Hash for CAPTCHA answer (DO NOT modify)' ), - string( - name: 'Terraform_Targets', - defaultValue: '', - description: '''Specific Terraform resource or resource names to target - (Use this to modify or delete less than the full set of resources''' - ), - text( - name: 'Extra_Variables', - defaultValue: '', - description: '''Terraform Variables to define for this run. - Allows you to override declared variables. - Put one variable per line, in JSON or HCL like this: - associate_public_ip_address = "true"''' - ), ]) ]) stage('Preflight') { - + // Check CAPTCHA - def should_validate_captcha = params.Run_Packer || params.Apply_Terraform || params.Destroy_Terraform + def should_validate_captcha = params.Run_Packer || params.Apply_Terraform || params.Destroy_Terraform || params.Run_JMeter if (should_validate_captcha) { if (params.CAPTCHA_Guess == null || params.CAPTCHA_Guess == "") { @@ -155,6 +178,15 @@ if (params.Run_Packer) { } } +stage('Build CodeDeploy Archive') { + node { + unstash 'src' + wrap.call({ + sh ("./codedeploy/bin/build.sh") + }) + } +} + def terraform_prompt = 'Should we apply the Terraform plan?' @@ -200,11 +232,25 @@ if (params.Rotate_Servers) { stage('Rotate Servers') { node { unstash 'src' - ansiColor('xterm') { - prepEnv() + wrap.call({ sh ("./bin/rotate-asg.sh infra-demo-asg") - } + }) } } } +if (params.Run_JMeter) { + stage('Run JMeter') { + node { + unstash 'src' + wrap.call({ + sh (""" + HOST=\$(./bin/terraform.sh output route53-dns) + ./bin/jmeter.sh -Jthreads=${params.JMETER_threads} -Jramp_duration=${params.JMETER_ramp_duration} -Jduration=${params.JMETER_duration} -Jhost=\$HOST + ls -l build + """) + archiveArtifacts artifacts: 'build/*.jtl, build/*.xml, build/*.csv, build/*.html', fingerprint: true + }) + } + } +} diff --git a/README.md b/README.md index e46bf72..862f40d 100644 --- a/README.md +++ b/README.md @@ -63,17 +63,9 @@ In order to make developing the Ansible playbooks faster, a Vagrantfile is provi Install [Vagrant](https://www.vagrantup.com/). Change directory into the root of the repository at the command line and issue the command `vagrant up`. You can add or edit Ansible playbooks and support scripts then re-run the provisioning with `vagrant provision` to refine the remediations. This is more efficient that re-running packer and baking new AMIs for every change. -### Jenkins - -A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account. Packer needs this in order to create AMIs, key pairs, etc, and Terraform needs this to create a VPC and EC2 resources. This could be pared down further through some careful logging and role work. - -The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into. The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using. - -This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials. - ### Terraform -This Terraform setup stores its state in Amazon S3 and uses DynamoDB for locking. There is a bit of setup required to bootstrap that configuration. Yu can use [this repository](https://github.com/monterail/terraform-bootstrap-example) to use Terraform to do that bootstrap process. The `backend.tfvars` file in that repo should be modified as follows to work with this project: +This Terraform setup stores its state in Amazon S3 and uses DynamoDB for locking. There is a bit of setup required to bootstrap that configuration. You can use [this repository](https://github.com/monterail/terraform-bootstrap-example) to use Terraform to do that bootstrap process. The `backend.tfvars` file in that repo should be modified as follows to work with this project: (Replace us-east-1 and XXXXXXXXXXXX with the AWS region and your account ID) ``` @@ -97,10 +89,34 @@ These commands will then set up cloud resources using terraform: # check to see if everything worked - use the same variables here as above terraform destroy -var 'domain=example.net' +Alternatively, use the wrapper script in `bin/terraform.sh` which will work interactively or from CI: + + bin/terraform.sh plan + bin/terraform.sh apply + bin/terraform.sh plan-destroy + bin/terraform.sh destroy + This assumes that you already have a Route 53 domain in your AWS account created. You need to either edit variables.tf to match your domain and AWS zone or specify these values as command line `var` parameters. -The application loads an image from Google storage. To get it loading correctly, look in the X file and replace `example-media-website-storage.storage.googleapis.com` with a DNS reference for your Google storage location. +The application loads an image from Google storage. To get it loading correctly, edit the `application/assets/css/main.css` file and replace `example-media-website-storage.storage.googleapis.com` with a DNS reference for your Google storage location. + +### CodeDeploy + +The application enclosed in this demo is packaged and deployed using [AWS CodeDeploy](https://aws.amazon.com/codedeploy/). The script `codedeploy/bin/build.sh` will package the application so that it can be deployed on the AMI built with Ansible and Packer. + +The application contains both a simple HTML web site, and a Python app that has an API endpoint of `/api/spin` that spins the CPU of the server, in order to more easily test CPU-sensing auto scaling scale-out operations. + +### JMeter + +A JMeter test harness that will allow testing of a the application +### Jenkins + +A `Jenkinsfile` is provided that will allow Jenkins to execute Packer and Terraform, package a CodeDeploy application, and even run JMeter performance tests. In order for Jenkins to do this, it needs to have AWS credentials set up, preferably through an IAM role, granting full control of EC2 and VPC resources in that account, and write access to the S3 bucket used for storing CodeDeploy applications. Packer needs this in order to create AMIs, key pairs, etc, Terraform needs this to create a VPC and EC2 resources, and CodeDeploy needs this to store the artifact it creates. This could be pared down further through some careful logging and role work. + +The scripts here assume that Jenkins is running on EC2 and uses instance data from the Jenkins executor to infer what VPC and subnet to launch the new EC2 instance into. The AWS profile IAM user associated with your Jenkins instance or the Jenkins user's AWS credentials should have full control of EC2 in the account you are using. + +This script relies on Jenkins having a secret file containing the Google application credentials in JSON with the id "terraform-demo.json". You will need to add that to your Jenkins server's credentials. # Modus Create diff --git a/Vagrantfile b/Vagrantfile index 31b3246..d5eaffa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,6 +2,6 @@ Vagrant.configure("2") do |config| config.vm.box = "bento/centos-7.5" config.vm.synced_folder ".", "/app" config.vm.provision "shell", inline: "/app/bin/install-ansible.sh", upload_path: "/home/vagrant/install-ansible.sh" - config.vm.provision "shell", inline: "ansible-playbook -l localhost /app/ansible/local.yml", upload_path: "/home/vagrant/apl.sh" + config.vm.provision "shell", inline: "cd /app/ansible && ansible-playbook -l localhost bakery.yml app-AfterInstall.yml app-StartServer.yml", upload_path: "/home/vagrant/apl.sh" config.vm.network "forwarded_port", guest: 80, host: 6080, auto_correct: true end diff --git a/ansible/app-AfterInstall.yml b/ansible/app-AfterInstall.yml new file mode 100644 index 0000000..ed11aa2 --- /dev/null +++ b/ansible/app-AfterInstall.yml @@ -0,0 +1,15 @@ +--- +# Use ansible to install the codedeploy agent at boot time through cloudinit + +# Because AWS updates the CodeDeploy agent somewhat frequently, baking it into +# the image is an antipattern. It can cause instances to fail to register with +# a CodeDeploy deployment group if the version of CodeDeploy is too old. + +# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for +# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Perform CodeDeploy AfterInstall hook + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - app-AfterInstall diff --git a/ansible/app-StartServer.yml b/ansible/app-StartServer.yml new file mode 100644 index 0000000..d6c0620 --- /dev/null +++ b/ansible/app-StartServer.yml @@ -0,0 +1,10 @@ +--- + +# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for +# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Perform CodeDeploy StartServer hook + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - app-StartServer diff --git a/ansible/local.yml b/ansible/bakery.yml similarity index 89% rename from ansible/local.yml rename to ansible/bakery.yml index e7153be..93f8b58 100644 --- a/ansible/local.yml +++ b/ansible/bakery.yml @@ -10,6 +10,7 @@ roles: - nginxinc.nginx - prepare-web-content + - prepare-codedeploy - name: Harden Server hosts: 127.0.0.1 @@ -17,5 +18,5 @@ become: yes roles: - extra-cis-remediation - - MindPointGroup.RHEL7-CIS + #- MindPointGroup.RHEL7-CIS - scan-openscap diff --git a/ansible/cloudinit.yml b/ansible/cloudinit.yml new file mode 100644 index 0000000..9c41327 --- /dev/null +++ b/ansible/cloudinit.yml @@ -0,0 +1,15 @@ +--- +# Use ansible to install the codedeploy agent at boot time through cloudinit + +# Because AWS updates the CodeDeploy agent somewhat frequently, baking it into +# the image is an antipattern. It can cause instances to fail to register with +# a CodeDeploy deployment group if the version of CodeDeploy is too old. + +# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for +# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Install CodeDeploy agent + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - ansible-aws-codedeploy-agent diff --git a/ansible/codedeploy.yml b/ansible/codedeploy.yml new file mode 100644 index 0000000..9c41327 --- /dev/null +++ b/ansible/codedeploy.yml @@ -0,0 +1,15 @@ +--- +# Use ansible to install the codedeploy agent at boot time through cloudinit + +# Because AWS updates the CodeDeploy agent somewhat frequently, baking it into +# the image is an antipattern. It can cause instances to fail to register with +# a CodeDeploy deployment group if the version of CodeDeploy is too old. + +# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for +# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" +- name: Install CodeDeploy agent + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - ansible-aws-codedeploy-agent diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 477bea7..05be059 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -6,3 +6,12 @@ # NGINX web server - src: nginxinc.nginx +# AWS CodeDeploy agent +# The original version of this role in Ansible Galaxy +# is telus/ansible-aws-codedeploy-agent +# +# It has not been updated for Ansible 2.7 :( +# However CareerBuilder's fork of it works with 2.7 :) +# And the ModusCreateOrg fork has been fixed to avoid running stuff from /tmp +- src: https://github.com/ModusCreateOrg/ansible-aws-codedeploy-agent + diff --git a/ansible/roles/app-AfterInstall/README.md b/ansible/roles/app-AfterInstall/README.md new file mode 100644 index 0000000..5a787ec --- /dev/null +++ b/ansible/roles/app-AfterInstall/README.md @@ -0,0 +1,42 @@ +App AfterInstall Role +===================== + +This is intended to prepare an application that requires some setup after installation for running. It will modify SELinux labels. This is intended for use with AWS CodeDeploy as an AfterInstall hook. + +Requirements +------------ + +None + +Role Variables +-------------- + +None + +Dependencies +------------ + +SELinux and a Red Hat Linux family OS + +Example Playbook +---------------- + +Please see the example below: + + - name: Perform AfterInstall hook + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - app-AfterInstall + + +License +------- + +MIT + +Author Information +------------------ + +Richard Bullington-McGuire diff --git a/ansible/roles/app-AfterInstall/defaults/main.yml b/ansible/roles/app-AfterInstall/defaults/main.yml new file mode 100644 index 0000000..d2f6bac --- /dev/null +++ b/ansible/roles/app-AfterInstall/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# defaults file for app-AfterInstall +ec2: false +virtualbox: false +app_dir: /app/application +server_name: localhost diff --git a/ansible/roles/prepare-web-content/files/my_httpd_t.te b/ansible/roles/app-AfterInstall/files/my_httpd_t.te similarity index 100% rename from ansible/roles/prepare-web-content/files/my_httpd_t.te rename to ansible/roles/app-AfterInstall/files/my_httpd_t.te diff --git a/ansible/roles/app-AfterInstall/handlers/main.yml b/ansible/roles/app-AfterInstall/handlers/main.yml new file mode 100644 index 0000000..a060aba --- /dev/null +++ b/ansible/roles/app-AfterInstall/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for app-selinux \ No newline at end of file diff --git a/ansible/roles/app-AfterInstall/meta/main.yml b/ansible/roles/app-AfterInstall/meta/main.yml new file mode 100644 index 0000000..7223799 --- /dev/null +++ b/ansible/roles/app-AfterInstall/meta/main.yml @@ -0,0 +1,57 @@ +galaxy_info: + author: your name + description: your description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + + min_ansible_version: 1.2 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + #github_branch: + + # + # platforms is a list of platforms, and each platform has a name and a list of versions. + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. \ No newline at end of file diff --git a/ansible/roles/app-AfterInstall/tasks/main.yml b/ansible/roles/app-AfterInstall/tasks/main.yml new file mode 100644 index 0000000..526f627 --- /dev/null +++ b/ansible/roles/app-AfterInstall/tasks/main.yml @@ -0,0 +1,104 @@ +--- +# tasks file for prepare-web-content + +- name: Ensure python-virtualenv is present + package: name={{item}} state=present + with_items: + - python-virtualenv + +- name: Define VirtualBox variable + set_fact: + virtualbox: true + when: "'VirtualBox' in ansible_bios_version" + +- name: Debug ansible_bios_version + debug: + msg: "{{ ansible_bios_version }}" + +- name: Define ec2 variable + set_fact: + ec2: true + when: "'amazon' in ansible_bios_version or 'Amazon EC2' in ansible_system_vendor" + +- name: Ensure selinux modules are present + package: name={{item}} state=present + with_items: + - checkpolicy + - policycoreutils-python + - policycoreutils + +- name: Create temp directory + tempfile: + state: directory + suffix: selinux + register: tmpdir + when: virtualbox + +- name: Copy SELinux policy to temp directory + copy: + src: my_httpd_t.te + dest: "{{ tmpdir.path }}" + when: virtualbox + +- name: Set up SELinux rules for Virtualbox + shell: | + cd "{{ tmpdir.path }}" + checkmodule -M -m -o my_httpd_t.mod my_httpd_t.te + semodule_package -o my_httpd_t.pp -m my_httpd_t.mod + semodule -i my_httpd_t.pp + when: virtualbox + +- name: Set up SELinux rules for Amazon EC2 + shell: | + setsebool -P httpd_can_network_connect 1 + setsebool -P httpd_can_network_relay 1 + semanage fcontext -a -t httpd_sys_content_t "{{ app_dir }}(/.*)?" + restorecon -R "{{ app_dir }}" + when: ec2 + +- name: Ensure default web server config is removed + file: + path: /etc/nginx/conf.d/default.conf + state: absent + +- name: Copy web server config into place + template: + src: app.conf.j2 + dest: /etc/nginx/conf.d/app.conf + mode: 0640 + +- name: Install python dependencies for app + pip: + requirements: /app/src/requirements.txt + virtualenv: /app/venv + +- name: nginx owns /app/socket + file: + path: /app/socket + state: directory + owner: nginx + group: nginx + +- name: Emperor systemd config + template: + src: emperor.service.j2 + dest: /etc/systemd/system/emperor.service + owner: root + group: root + mode: 0640 + +- name: emperor.ini + template: + src: emperor.ini.j2 + dest: /app/emperor.ini + owner: root + group: root + mode: 0644 + +- name: uwsgi app config infra-demo.ini + template: + src: infra-demo.ini.j2 + dest: /app/src/infra-demo.ini + owner: root + group: root + mode: 0644 \ No newline at end of file diff --git a/ansible/roles/app-AfterInstall/templates/app.conf.j2 b/ansible/roles/app-AfterInstall/templates/app.conf.j2 new file mode 100644 index 0000000..01fcdc1 --- /dev/null +++ b/ansible/roles/app-AfterInstall/templates/app.conf.j2 @@ -0,0 +1,26 @@ +server { + listen 80; + server_name {{ server_name }}; + + location / { + root {{ app_dir }}; + index index.html index.htm; + } + + location /api/spin { + uwsgi_pass 127.0.0.1:8008; + include /etc/nginx/uwsgi_params; + uwsgi_param UWSGI_SCRIPT /app/src/wsgi.py; + # following config allow us to map /api/spin to /spin on uwsgi: + uwsgi_param SCRIPT_NAME /api; # set SCRIPT_NAME to match subpath + uwsgi_modifier1 30; # strips SCRIPT_NAME from PATH_INFO + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} + diff --git a/ansible/roles/app-AfterInstall/templates/emperor.ini.j2 b/ansible/roles/app-AfterInstall/templates/emperor.ini.j2 new file mode 100644 index 0000000..3021775 --- /dev/null +++ b/ansible/roles/app-AfterInstall/templates/emperor.ini.j2 @@ -0,0 +1,6 @@ +[uwsgi] +venv = /app/venv +emperor = /app/src +uid = nginx +gid = nginx + diff --git a/ansible/roles/app-AfterInstall/templates/emperor.service.j2 b/ansible/roles/app-AfterInstall/templates/emperor.service.j2 new file mode 100644 index 0000000..b87e4ac --- /dev/null +++ b/ansible/roles/app-AfterInstall/templates/emperor.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=uWSGI Emperor +After=syslog.target + +[Service] +ExecStart=/app/venv/bin/uwsgi --ini /app/emperor.ini +# Requires systemd version 211 or newer +RuntimeDirectory=uwsgi +Restart=always +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 new file mode 100644 index 0000000..e3e76e1 --- /dev/null +++ b/ansible/roles/app-AfterInstall/templates/infra-demo.ini.j2 @@ -0,0 +1,14 @@ +[uwsgi] +venv = /app/venv +wsgi-file = /app/src/wsgi.py +chdir = /app/src +master = 1 +workers = 2 +threads = 8 +lazy-apps = 1 +wsgi-env-behaviour = holy +enable-threads = 1 +http-auto-chunked = 1 +http-keepalive = 1 +uwsgi-socket = 127.0.0.1:8008 + diff --git a/ansible/roles/app-AfterInstall/tests/inventory b/ansible/roles/app-AfterInstall/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/ansible/roles/app-AfterInstall/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/ansible/roles/app-AfterInstall/tests/test.yml b/ansible/roles/app-AfterInstall/tests/test.yml new file mode 100644 index 0000000..f3119bc --- /dev/null +++ b/ansible/roles/app-AfterInstall/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - app-selinux \ No newline at end of file diff --git a/ansible/roles/app-AfterInstall/vars/main.yml b/ansible/roles/app-AfterInstall/vars/main.yml new file mode 100644 index 0000000..759e344 --- /dev/null +++ b/ansible/roles/app-AfterInstall/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for app-selinux \ No newline at end of file diff --git a/ansible/roles/app-StartServer/README.md b/ansible/roles/app-StartServer/README.md new file mode 100644 index 0000000..546d1f3 --- /dev/null +++ b/ansible/roles/app-StartServer/README.md @@ -0,0 +1,42 @@ +App StartServer Role +===================== + +This is intended to prepare an application that requires some setup after installation for running. This is intended for use with AWS CodeDeploy as a StartServer hook. + +Requirements +------------ + +None + +Role Variables +-------------- + +None + +Dependencies +------------ + +nginx + +Example Playbook +---------------- + +Please see the example below: + + - name: Perform StartServer hook + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - app-StartServer + + +License +------- + +MIT + +Author Information +------------------ + +Richard Bullington-McGuire diff --git a/ansible/roles/app-StartServer/defaults/main.yml b/ansible/roles/app-StartServer/defaults/main.yml new file mode 100644 index 0000000..d1a32c0 --- /dev/null +++ b/ansible/roles/app-StartServer/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for app-StartServer diff --git a/ansible/roles/app-StartServer/handlers/main.yml b/ansible/roles/app-StartServer/handlers/main.yml new file mode 100644 index 0000000..a060aba --- /dev/null +++ b/ansible/roles/app-StartServer/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for app-selinux \ No newline at end of file diff --git a/ansible/roles/app-StartServer/meta/main.yml b/ansible/roles/app-StartServer/meta/main.yml new file mode 100644 index 0000000..7223799 --- /dev/null +++ b/ansible/roles/app-StartServer/meta/main.yml @@ -0,0 +1,57 @@ +galaxy_info: + author: your name + description: your description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + + min_ansible_version: 1.2 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + #github_branch: + + # + # platforms is a list of platforms, and each platform has a name and a list of versions. + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. \ No newline at end of file diff --git a/ansible/roles/app-StartServer/tasks/main.yml b/ansible/roles/app-StartServer/tasks/main.yml new file mode 100644 index 0000000..126224b --- /dev/null +++ b/ansible/roles/app-StartServer/tasks/main.yml @@ -0,0 +1,15 @@ +--- +# tasks file for prepare-web-content + +- name: Start and enable emperor, as it should start off disabled + service: + name: emperor + enabled: yes + state: restarted + +- name: Start and enable nginx, as it should start off disabled + service: + name: nginx + enabled: yes + state: restarted + diff --git a/ansible/roles/app-StartServer/tests/inventory b/ansible/roles/app-StartServer/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/ansible/roles/app-StartServer/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/ansible/roles/app-StartServer/tests/test.yml b/ansible/roles/app-StartServer/tests/test.yml new file mode 100644 index 0000000..f3119bc --- /dev/null +++ b/ansible/roles/app-StartServer/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - app-selinux \ No newline at end of file diff --git a/ansible/roles/app-StartServer/vars/main.yml b/ansible/roles/app-StartServer/vars/main.yml new file mode 100644 index 0000000..759e344 --- /dev/null +++ b/ansible/roles/app-StartServer/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for app-selinux \ No newline at end of file diff --git a/ansible/roles/prepare-codedeploy/README.md b/ansible/roles/prepare-codedeploy/README.md new file mode 100644 index 0000000..0f39903 --- /dev/null +++ b/ansible/roles/prepare-codedeploy/README.md @@ -0,0 +1,38 @@ +Prepare CodeDeploy +================== + +Prepare an image for having AWS CodeDeploy installed on it with a minimum of runtime downloads. + +Requirements +------------ + +This is tested only on CentOS 7 currently. It might work on other distributions. + +Role Variables +-------------- + +None + +Dependencies +------------ + +None + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: prepare-codedeploy } + +License +------- + +MIT + +Author Information +------------------ + +Richard Bullington-McGuire diff --git a/ansible/roles/prepare-codedeploy/defaults/main.yml b/ansible/roles/prepare-codedeploy/defaults/main.yml new file mode 100644 index 0000000..dc55145 --- /dev/null +++ b/ansible/roles/prepare-codedeploy/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for prepare-codedeploy \ No newline at end of file diff --git a/ansible/roles/prepare-codedeploy/handlers/main.yml b/ansible/roles/prepare-codedeploy/handlers/main.yml new file mode 100644 index 0000000..3a81132 --- /dev/null +++ b/ansible/roles/prepare-codedeploy/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for prepare-codedeploy \ No newline at end of file diff --git a/ansible/roles/prepare-codedeploy/meta/main.yml b/ansible/roles/prepare-codedeploy/meta/main.yml new file mode 100644 index 0000000..5d50bf4 --- /dev/null +++ b/ansible/roles/prepare-codedeploy/meta/main.yml @@ -0,0 +1,60 @@ +galaxy_info: + author: your name + description: your description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + + min_ansible_version: 2.4 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + #github_branch: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. \ No newline at end of file diff --git a/ansible/roles/prepare-codedeploy/tasks/main.yml b/ansible/roles/prepare-codedeploy/tasks/main.yml new file mode 100644 index 0000000..ad2b609 --- /dev/null +++ b/ansible/roles/prepare-codedeploy/tasks/main.yml @@ -0,0 +1,13 @@ +--- +# tasks file for prepare-codedeploy + + +- name: Ensure packages used by CodeDeploy installer are present + package: name={{item}} state=present + with_items: + - python2-pip + - ruby + - git + +- name: Ensure awscli is installed + pip: name=awscli state=present diff --git a/ansible/roles/prepare-codedeploy/tests/inventory b/ansible/roles/prepare-codedeploy/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/ansible/roles/prepare-codedeploy/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/ansible/roles/prepare-codedeploy/tests/test.yml b/ansible/roles/prepare-codedeploy/tests/test.yml new file mode 100644 index 0000000..1490bea --- /dev/null +++ b/ansible/roles/prepare-codedeploy/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - prepare-codedeploy \ No newline at end of file diff --git a/ansible/roles/prepare-codedeploy/vars/main.yml b/ansible/roles/prepare-codedeploy/vars/main.yml new file mode 100644 index 0000000..3568e27 --- /dev/null +++ b/ansible/roles/prepare-codedeploy/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for prepare-codedeploy \ No newline at end of file diff --git a/ansible/roles/prepare-web-content/defaults/main.yml b/ansible/roles/prepare-web-content/defaults/main.yml index 8652201..7886f2e 100644 --- a/ansible/roles/prepare-web-content/defaults/main.yml +++ b/ansible/roles/prepare-web-content/defaults/main.yml @@ -1,6 +1,2 @@ --- # defaults file for prepare-web-content -ec2: false -virtualbox: false -app_dir: /app/application -server_name: localhost diff --git a/ansible/roles/prepare-web-content/tasks/main.yml b/ansible/roles/prepare-web-content/tasks/main.yml index d7277b5..1e05a37 100644 --- a/ansible/roles/prepare-web-content/tasks/main.yml +++ b/ansible/roles/prepare-web-content/tasks/main.yml @@ -1,16 +1,13 @@ --- # tasks file for prepare-web-content -- name: Define VirtualBox variable - set_fact: - virtualbox: true - when: "'VirtualBox' in ansible_bios_version" - -- name: Define ec2 variable - set_fact: - ec2: true - when: "'amazon' in ansible_bios_version" +- name: install the 'Development tools' package group + yum: + name: "@Development tools" + state: present +# Strictly speaking, we don't need the selinux modules at this early stage, +# but having them will help the app-AfterInstall playbook avoid delays. - name: Ensure firewall and selinux modules are present package: name={{item}} state=present with_items: @@ -20,11 +17,6 @@ - policycoreutils - python-firewall -- name: Ensure firewalld python lib is present - package: - name: python-firewall - state: present - - name: Open firewalld port for http firewalld: service: http @@ -44,45 +36,9 @@ name: firewalld state: reloaded -- name: Create temp directory - tempfile: - state: directory - suffix: selinux - register: tmpdir - when: virtualbox - -- name: Copy SELinux policy to temp directory - copy: - src: my_httpd_t.te - dest: "{{ tmpdir.path }}" - when: virtualbox - -- name: Set up SELinux rules for Virtualbox - shell: | - cd "{{ tmpdir.path }}" - checkmodule -M -m -o my_httpd_t.mod my_httpd_t.te - semodule_package -o my_httpd_t.pp -m my_httpd_t.mod - semodule -i my_httpd_t.pp - when: virtualbox - -- name: Set up SELinux rules for Amazon EC2 - shell: | - semanage fcontext -a -t httpd_sys_content_t "{{ app_dir }}(/.*)?" - restorecon -R "{{ app_dir }}" - when: ec2 - -- name: Copy web server config into place - template: - src: app.conf.j2 - dest: /etc/nginx/conf.d/app.conf - mode: 0640 - -- name: Ensure default web server config is removed - file: - path: /etc/nginx/conf.d/default.conf - state: absent - -- name: Reload service nginx +- name: Stop and disable nginx, as it should start off disabled service: name: nginx - state: reloaded + enabled: no + state: stopped + diff --git a/ansible/roles/prepare-web-content/templates/app.conf.j2 b/ansible/roles/prepare-web-content/templates/app.conf.j2 deleted file mode 100644 index 59b86bd..0000000 --- a/ansible/roles/prepare-web-content/templates/app.conf.j2 +++ /dev/null @@ -1,17 +0,0 @@ -server { - listen 80; - server_name {{ server_name }}; - - location / { - root {{ app_dir }}; - index index.html index.htm; - } - - # redirect server error pages to the static page /50x.html - # - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } -} - diff --git a/ansible/roles/scan-openscap/tasks/main.yml b/ansible/roles/scan-openscap/tasks/main.yml index a7778fb..3cc27e1 100644 --- a/ansible/roles/scan-openscap/tasks/main.yml +++ b/ansible/roles/scan-openscap/tasks/main.yml @@ -14,11 +14,11 @@ - name: Scan with OpenSCAP shell: | - set -euo pipefail - cd {{ build_dir }} + eet -euo pipefail + ed {{ build_dir }} # This will have a non-zero exit if any of the scans fail, so do not fail immediately on that set +e - oscap xccdf eval --profile {{ profile }} --results {{ output_file_xml }} {{ xccdf_file }} + oscap xccdf eval --fetch-remote-resources --profile {{ profile }} --results {{ output_file_xml }} {{ xccdf_file }} set -e oscap xccdf generate report {{ output_file_xml }} > {{ output_file_html }} args: diff --git a/ansible/scan.yml b/ansible/scan.yml new file mode 100644 index 0000000..79c1165 --- /dev/null +++ b/ansible/scan.yml @@ -0,0 +1,10 @@ +--- +# Thanks https://www.tricksofthetrades.net/2017/10/02/ansible-local-playbooks/ for +# the trick on installing locally using "hosts: 127.0.0.1" and "connection:local" + +- name: Scan Server + hosts: 127.0.0.1 + connection: local + become: yes + roles: + - scan-openscap diff --git a/bin/common.sh b/bin/common.sh index ccbf8c2..5baddf6 100755 --- a/bin/common.sh +++ b/bin/common.sh @@ -47,7 +47,7 @@ function clean_root_owned_docker_files { BASE_DIR="$(pwd)" if is_ec2; then docker run -i \ - --mount type=bind,source="${BASE_DIR}"/terraform,target="${TF_DIR}" \ + --mount type=bind,source="${BASE_DIR}",target="${TF_DIR}" \ -w "${TF_DIR}" \ --entrypoint /bin/sh \ busybox \ @@ -66,12 +66,12 @@ function get_docker_packer { PACKER_AWS_VPC_ID="$(curl --silent http://169.254.169.254/latest/meta-data/network/interfaces/macs/"$INTERFACE"/vpc-id)" fi - echo "docker run -i - ${USE_TTY} + echo "docker run -i + ${USE_TTY} --env-file $TMPFILE - -e PACKER_AWS_SUBNET_ID=$PACKER_AWS_SUBNET_ID - -e PACKER_AWS_VPC_ID=$PACKER_AWS_VPC_ID - --mount type=bind,source=${BASE_DIR},target=/app + -e PACKER_AWS_SUBNET_ID=$PACKER_AWS_SUBNET_ID + -e PACKER_AWS_VPC_ID=$PACKER_AWS_VPC_ID + --mount type=bind,source=${BASE_DIR},target=/app hashicorp/packer:light" } diff --git a/bin/install-ansible.sh b/bin/install-ansible.sh index 2c3f325..585ae0c 100755 --- a/bin/install-ansible.sh +++ b/bin/install-ansible.sh @@ -20,4 +20,5 @@ function quick_yum_install() { } quick_yum_install epel-release quick_yum_install ansible +quick_yum_install git ansible-galaxy install -r /app/ansible/requirements.yml diff --git a/bin/jmeter.sh b/bin/jmeter.sh new file mode 100755 index 0000000..1efe482 --- /dev/null +++ b/bin/jmeter.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." +BUILD_DIR="$BASE_DIR/build" +export BUILD_DIR + +mkdir -p "$BUILD_DIR" +docker run -t -v "$BASE_DIR:/repo" justb4/jmeter -n -t /repo/jmeter/api-spin.jmx -l /repo/build/jmeter.jtl "$@" diff --git a/bin/rotate-asg.sh b/bin/rotate-asg.sh index 7dea2c1..30d7bc4 100755 --- a/bin/rotate-asg.sh +++ b/bin/rotate-asg.sh @@ -4,8 +4,8 @@ set -euo pipefail IFS=$'\n\t' -# Enable for enhanced debugging -#set -vx +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx # Credit to https://stackoverflow.com/a/17805088 # and http://wiki.bash-hackers.org/scripting/debuggingtips export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' diff --git a/bin/spin-3.sh b/bin/spin-3.sh new file mode 100755 index 0000000..585c54f --- /dev/null +++ b/bin/spin-3.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +ab -c 2 -t 900 http://devops-infra-demo.modus.app/api/spin diff --git a/bin/spin-7.sh b/bin/spin-7.sh new file mode 100755 index 0000000..443a4bd --- /dev/null +++ b/bin/spin-7.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +ab -c 4 -t 900 http://devops-infra-demo.modus.app/api/spin diff --git a/bin/spin.sh b/bin/spin.sh new file mode 100755 index 0000000..2aff87a --- /dev/null +++ b/bin/spin.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +ab -c 8 -t 900 http://devops-infra-demo.modus.app/api/spin diff --git a/bin/terraform.sh b/bin/terraform.sh index 0f615bb..81d00bd 100755 --- a/bin/terraform.sh +++ b/bin/terraform.sh @@ -31,12 +31,12 @@ BUILD_DIR="$BASE_DIR/build" DOCKER_TERRAFORM=$(get_docker_terraform) DOCKER_LANDSCAPE=$(get_docker_landscape) -verb=${1:?You must specify a verb: plan, plan-destroy, apply} +verb=${1:?You must specify a verb: plan, plan-destroy, apply, show, output} # Inject Google application credentials into env file for docker GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE:-} if [[ -n "$GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE" ]]; then - echo "Overriding Google Application Credentials" + echo "Overriding Google Application Credentials" 1>&2 GOOGLE_APPLICATION_CREDENTIALS="$GOOGLE_APPLICATION_CREDENTIALS_OVERRIDE" fi @@ -60,6 +60,18 @@ EOF # http://redsymbol.net/articles/bash-exit-traps/ trap clean_root_owned_docker_files EXIT +function show() { + local -i retcode + #shellcheck disable=SC2086 + $DOCKER_TERRAFORM show +} + +function output () { + local -i retcode + local extra + #shellcheck disable=SC2086 + $DOCKER_TERRAFORM output "$@" +} function plan() { local extra @@ -94,7 +106,7 @@ function plan() { } function plan-destroy() { - cat <&2 <&2 exit 1 ;; esac - -echo "$Message" -init_terraform -"$verb" +shift +echo "$Message" 1>&2 +init_terraform 1>&2 +"$verb" "$@" diff --git a/bin/validate.sh b/bin/validate.sh index 0cce1fe..b26b761 100755 --- a/bin/validate.sh +++ b/bin/validate.sh @@ -5,6 +5,9 @@ set -euo pipefail # Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" ${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' # Credit to http://stackoverflow.com/a/246128/424301 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" diff --git a/codedeploy/appspec.yml b/codedeploy/appspec.yml new file mode 100644 index 0000000..e0f94aa --- /dev/null +++ b/codedeploy/appspec.yml @@ -0,0 +1,24 @@ +--- +version: 0.0 +os: linux +files: + - source: application + destination: /app/application + - source: src + destination: /app/src +hooks: + ApplicationStop: + - location: bin/ApplicationStop.sh + timeout: 600 + BeforeInstall: + - location: bin/BeforeInstall.sh + timeout: 30 + AfterInstall: + - location: bin/AfterInstall.sh + timeout: 300 + ApplicationStart: + - location: bin/ApplicationStart.sh + timeout: 120 + ValidateService: + - location: bin/ValidateService.sh + timeout: 60 diff --git a/codedeploy/bin/AfterInstall.sh b/codedeploy/bin/AfterInstall.sh new file mode 100755 index 0000000..91f49d4 --- /dev/null +++ b/codedeploy/bin/AfterInstall.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# AfterInstall.sh +# +# AWS CodeDeploy After Install hook script + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." +ANSIBLE_DIR="$BASE_DIR/ansible" + +# Invoke Ansible for final set up +ansible-playbook -l localhost "$ANSIBLE_DIR/app-AfterInstall.yml" diff --git a/codedeploy/bin/ApplicationStart.sh b/codedeploy/bin/ApplicationStart.sh new file mode 100755 index 0000000..5d08139 --- /dev/null +++ b/codedeploy/bin/ApplicationStart.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# ApplicationStart.sh +# +# AWS CodeDeploy Application Start hook script + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." +ANSIBLE_DIR="$BASE_DIR/ansible" + +# Invoke Ansible for final set up +ansible-playbook -l localhost "$ANSIBLE_DIR/app-StartServer.yml" diff --git a/codedeploy/bin/ApplicationStop.sh b/codedeploy/bin/ApplicationStop.sh new file mode 100755 index 0000000..bb44b7c --- /dev/null +++ b/codedeploy/bin/ApplicationStop.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# ApplicationStop.sh +# +# AWS CodeDeploy Application Stop hook script +# +# Useful when running in Jenkins CI or other contexts where you have Docker +# available. + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +systemctl stop emperor +systemctl disable emperor +systemctl stop nginx +systemctl disable nginx diff --git a/codedeploy/bin/BeforeInstall.sh b/codedeploy/bin/BeforeInstall.sh new file mode 100755 index 0000000..3e39ecc --- /dev/null +++ b/codedeploy/bin/BeforeInstall.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# BeforeInstall.sh +# +# AWS CodeDeploy Before Install hook script + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +rm -rf /app diff --git a/codedeploy/bin/ValidateService.sh b/codedeploy/bin/ValidateService.sh new file mode 100755 index 0000000..61a8c63 --- /dev/null +++ b/codedeploy/bin/ValidateService.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# ValidateService.sh +# +# AWS CodeDeploy Validate Service hook script + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +check_every() { + local delay=${1:-} + local host="http://localhost/" + # shellcheck disable=SC2048 + while ! curl -s -o /dev/null $host + do + sleep "$delay" + echo "Sleeping $delay, $host was not reachable" + done +} + +check_every 2 diff --git a/codedeploy/bin/build.sh b/codedeploy/bin/build.sh new file mode 100755 index 0000000..e61b0de --- /dev/null +++ b/codedeploy/bin/build.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +# Set bash unofficial strict mode http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail + +# Set DEBUG to true for enhanced debugging: run prefixed with "DEBUG=true" +${DEBUG:-false} && set -vx +# Credit to https://stackoverflow.com/a/17805088 +# and http://wiki.bash-hackers.org/scripting/debuggingtips +export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }' + +# Credit to http://stackoverflow.com/a/246128/424301 +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASE_DIR="$DIR/.." +BUILD_DIR="$BASE_DIR/build" +ANSIBLE_DIR="$BASE_DIR/../ansible" +APPLICTION_DIR="$BASE_DIR/../application" +SRC_DIR="$BASE_DIR/../src" +VENV_DIR="$BASE_DIR/../venv" +DOCKER_DIR="$BASE_DIR/.." + +GIT_REV="$(git rev-parse --short HEAD)" +BUILD_NUMBER=${BUILD_NUMBER:-0} +ARCHIVE="codedeploy-$BUILD_NUMBER-$GIT_REV.zip" +CONTAINERNAME=infra-demo + +echo "GIT_REV=$GIT_REV" +echo "BUILD_NUMBER=$BUILD_NUMBER" +echo "ARCHIVE=$ARCHIVE" + +# Thanks https://stackoverflow.com/questions/33791069/quick-way-to-get-aws-account-number-from-the-cli-tools +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --output text --query 'Account') +BUCKET="codedeploy-$AWS_ACCOUNT_ID" +S3_URL="s3://$BUCKET/$ARCHIVE" + +if [[ -d "$BUILD_DIR" ]]; then + rm -rf "$BUILD_DIR" +fi +mkdir -p "$BUILD_DIR/socket" + +echo Build docker container $CONTAINERNAME +docker build -f=Dockerfile -t "$CONTAINERNAME" "$DOCKER_DIR" + +echo Create python virtual environment +docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ + "mkdir -p /src/venv ; \ + cp -fa /app/venv/* /src/venv" + +SOURCES="$BASE_DIR/bin +$ANSIBLE_DIR +$APPLICTION_DIR +$SRC_DIR +$BASE_DIR/appspec.yml +$BASE_DIR/bin +$VENV_DIR" +for src in $SOURCES; do + cp -a "$src" "$BUILD_DIR" +done + +( + cd "$BUILD_DIR" + zip -q -r "$ARCHIVE" \ + appspec.yml \ + bin \ + ansible \ + application \ + src \ + venv \ + socket +) + +echo Remove docker generated files +docker run --rm -v "$DOCKER_DIR:/src" "$CONTAINERNAME" /bin/bash -c \ + "rm -rf /src/venv" + +cd "$BUILD_DIR" +aws s3 cp "$ARCHIVE" "$S3_URL" --quiet +echo "CodeDeploy archive uploaded OK: $S3_URL" +aws s3 ls "$S3_URL" diff --git a/jmeter/api-spin.jmx b/jmeter/api-spin.jmx new file mode 100644 index 0000000..cc44fcb --- /dev/null +++ b/jmeter/api-spin.jmx @@ -0,0 +1,184 @@ + + + + + + false + true + false + + + + + + + + parametrized to accept command line parameters for time, ramp up, and number of threads. + continue + + false + -1 + + ${__P(num_threads,2)} + ${__P(ramp_time,90)} + true + ${__P(duration,180)} + + 1548631349000 + 1548631349000 + + + + + + + ${__P(domain,devops-infra-demo.modus.app)} + 80 + + + http + + ${__P(path,/api/spin)} + GET + true + false + true + false + false + + + + + + spun + + Assertion.response_data + false + 2 + + + + + 200 + + Assertion.response_code + false + 1 + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + + + + + + + true + + saveConfig + + + true + true + true + + true + true + true + true + true + true + true + true + true + true + false + false + true + false + true + 0 + true + true + true + true + true + true + + + /repo/build/jmeter-results.xml + + + + + + diff --git a/packer/machines/web-server.json b/packer/machines/web-server.json index 4f04197..461d40d 100644 --- a/packer/machines/web-server.json +++ b/packer/machines/web-server.json @@ -47,7 +47,7 @@ "type": "shell", "inline": [ "bash /app/bin/install-ansible.sh", - "ansible-playbook -l localhost /app/ansible/local.yml", + "ansible-playbook -l localhost /app/ansible/bakery.yml", "bash /app/bin/scan.sh" ] }, diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..e250f8e --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,46 @@ +# Adapted from https://www.caktusgroup.com/blog/2017/03/14/production-ready-dockerfile-your-python-django-app/ +# which states: +# +# Without further ado, here’s a production-ready Dockerfile you can use as a starting point for your project +# +FROM python:3.6-alpine + +# Copy in your requirements file +ADD requirements.txt /requirements.txt + +# Install build deps, then run `pip install`, then remove unneeded build deps all in a single step. Correct the path to your production requirements file, if needed. +RUN set -ex \ + && apk add --no-cache --virtual .build-deps \ + gcc \ + make \ + libc-dev \ + musl-dev \ + linux-headers \ + pcre-dev \ + postgresql-dev \ + && pyvenv /venv \ + && /venv/bin/pip install -U pip \ + && LIBRARY_PATH=/lib:/usr/lib /bin/sh -c "/venv/bin/pip install --no-cache-dir -r /requirements.txt" \ + && runDeps="$( \ + scanelf --needed --nobanner --recursive /venv \ + | awk '{ gsub(/,/, "\nso:", $2); print "so:" $2 }' \ + | sort -u \ + | xargs -r apk info --installed \ + | sort -u \ + )" \ + && apk add --virtual .python-rundeps $runDeps \ + && apk del .build-deps + +# Copy your application code to the container (make sure you create a .dockerignore file if any large files or directories should be excluded) +RUN mkdir /code/ +WORKDIR /code/ +ADD . /code/ + +# uWSGI will listen on this port +EXPOSE 8000 + +# uWSGI configuration (customize as needed): +ENV UWSGI_VIRTUALENV=/venv UWSGI_WSGI_FILE=wsgi.py UWSGI_HTTP=:8000 UWSGI_MASTER=1 UWSGI_WORKERS=2 UWSGI_THREADS=8 UWSGI_UID=1000 UWSGI_GID=2000 UWSGI_LAZY_APPS=1 UWSGI_WSGI_ENV_BEHAVIOR=holy + +# Start uWSGI +CMD ["/venv/bin/uwsgi", "--http-auto-chunked", "--http-keepalive"] diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..d7d87aa --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1,2 @@ +bottle==0.12.13 +uwsgi diff --git a/src/run.py b/src/run.py new file mode 100755 index 0000000..81ba46c --- /dev/null +++ b/src/run.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +"""x""" + +if __name__ == "__main__": + from spin import spin + from bottle import run, default_app + + run(host='localhost', port=8080, debug=True) diff --git a/src/spin.py b/src/spin.py new file mode 100755 index 0000000..4070b85 --- /dev/null +++ b/src/spin.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +"""This module spins the CPU.""" +import os +import time +from bottle import route, default_app, response + +@route('/spin') +def spin(delay=5.0): + """Spin the CPU, return the process id at the end""" + child_pid = os.getpid() + upper_max = 100000000000000000000000000000000 + start_time = time.time() + current_time = start_time + scratch = 42 + int(current_time) + end_time = start_time + delay + calcs = 0 + while current_time < end_time: + calcs += 1 + scratch = (scratch * scratch) % upper_max + current_time = time.time() + final_time = time.time() + interval = final_time - start_time + rate = calcs / interval + response.set_header('Content-Type', 'text/plain') + return ('pid {0} spun {1} times over {2}s (rate {3}/s)\n' + .format(child_pid, calcs, interval, rate)) + +application = default_app() diff --git a/src/startit.sh b/src/startit.sh new file mode 100755 index 0000000..2737fc6 --- /dev/null +++ b/src/startit.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# current directory needs where this script is located +cd "$(dirname "$0")" || exit + +UWSGI_VIRTUALENV=/app/venv \ +UWSGI_WSGI_FILE=/app/src/wsgi.py \ +UWSGI_MASTER=1 \ +UWSGI_WORKERS=2 \ +UWSGI_THREADS=8 \ +UWSGI_UID=nobody \ +UWSGI_GID=nobody \ +UWSGI_LAZY_APPS=1 \ +UWSGI_WSGI_ENV_BEHAVIOR=holy \ + /app/venv/bin/uwsgi \ + --enable-threads \ + --http-auto-chunked \ + --http-keepalive \ + --socket=/app/socket/uwsgi.sock & + diff --git a/src/stopit.sh b/src/stopit.sh new file mode 100755 index 0000000..95a289e --- /dev/null +++ b/src/stopit.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +pkill --signal INT -f uwsgi || true + diff --git a/src/wsgi.py b/src/wsgi.py new file mode 100755 index 0000000..e9ce017 --- /dev/null +++ b/src/wsgi.py @@ -0,0 +1,6 @@ +#!/usr/bin/env python +"""This exports a WSGI application""" +from spin import spin +from bottle import route, default_app + +application = default_app() diff --git a/terraform/assume-role-policy-codedeploy.json b/terraform/assume-role-policy-codedeploy.json new file mode 100644 index 0000000..618b3c8 --- /dev/null +++ b/terraform/assume-role-policy-codedeploy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": [ + "codedeploy.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/terraform/assume-role-policy-ec2.json b/terraform/assume-role-policy-ec2.json new file mode 100644 index 0000000..ccc2b93 --- /dev/null +++ b/terraform/assume-role-policy-ec2.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": [ + "ec2.amazonaws.com" + ] + }, + "Action": "sts:AssumeRole" + } + ] +} diff --git a/terraform/aws.tf b/terraform/aws.tf index e26c33f..3f11d9a 100644 --- a/terraform/aws.tf +++ b/terraform/aws.tf @@ -2,7 +2,8 @@ # This makes it super-clear which AWS account, arn, and user_id are in use # in a way that can be conveniently tracked in the output of CI tools provider "aws" { - region = "${var.aws_region}" + region = "${var.aws_region}" + version = "~> 1.57" } data "aws_caller_identity" "current" {} diff --git a/terraform/cloud-config.yml b/terraform/cloud-config.yml new file mode 100644 index 0000000..3872c61 --- /dev/null +++ b/terraform/cloud-config.yml @@ -0,0 +1,3 @@ +#cloud-config +runcmd: + - sudo -u centos ansible-playbook -l localhost /app/ansible/codedeploy.yml diff --git a/terraform/codedeploy.tf b/terraform/codedeploy.tf new file mode 100644 index 0000000..184a825 --- /dev/null +++ b/terraform/codedeploy.tf @@ -0,0 +1,50 @@ +resource "aws_iam_role" "infra-demo" { + name = "tf-infra-demo-role" + + assume_role_policy = <