diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml new file mode 100644 index 0000000..ded55c9 --- /dev/null +++ b/.github/workflows/pg_dump.yml @@ -0,0 +1,56 @@ +name: 'Postgres: Dump Database' + +on: + workflow_call: + inputs: + pg_dump_name: + description: 'Dump Name' + required: true + type: string + database_name: + description: 'Database Name' + required: true + default: refinery + type: string + +jobs: + pg-dump: + name: 'Postgres: Dump ${{ inputs.database_name }}' + runs-on: [self-hosted, "${{ github.ref_name }}"] + environment: ${{ github.ref_name }} + env: + ENVIRONMENT_NAME: ${{ github.ref_name }} + FILE_SHARE_RESOURCE_GROUP: "${{ vars.FILE_SHARE_RESOURCE_GROUP }}" + STORAGE_ACCOUNT_NAME: "${{ vars.STORAGE_ACCOUNT_NAME }}" + STORAGE_ACCOUNT_KEY: "${{ secrets.STORAGE_ACCOUNT_KEY }}" + FILE_SHARE_NAME: "${{ vars.FILE_SHARE_NAME }}" + PG_HOST: "${{ secrets.PG_HOST }}" + PG_USER: "${{ secrets.PG_USER }}" + PG_PASSWORD: "${{ secrets.PG_PASSWORD }}" + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + with: + repository: '${{ github.repository_owner }}/cicd-deployment-scripts' + ref: 'pg-dump-restore' + + - name: Generate Dump + id: generate-dump + shell: bash + run: | + bash ./pg/dump.sh \ + -h "${{ env.PG_HOST }}" \ + -u "${{ env.PG_USER }}" \ + -d "${{ inputs.database_name }}" \ + -n "${{ inputs.pg_dump_name }}" \ + -p "${{ env.PG_PASSWORD }}" + + - name: Upload Dump + shell: bash + run: | + az storage file upload-batch \ + --destination ${{ env.FILE_SHARE_NAME }} \ + --destination-path pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}} \ + --source ${{ steps.generate-dump.outputs.PG_DUMP_PATH }} \ + --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \ + --account-key ${{ env.STORAGE_ACCOUNT_KEY }} diff --git a/.github/workflows/pg_restore.yml b/.github/workflows/pg_restore.yml new file mode 100644 index 0000000..641d200 --- /dev/null +++ b/.github/workflows/pg_restore.yml @@ -0,0 +1,59 @@ +name: 'Postgres: Restore Database' + +on: + workflow_call: + inputs: + pg_dump_name: + description: 'Dump Name' + required: true + type: string + database_name: + description: 'Database Name' + required: true + default: refinery + type: string + +jobs: + pg-restore: + name: 'Postgres: Restore ${{ inputs.database_name }}' + runs-on: [self-hosted, "${{ github.ref_name }}"] + environment: ${{ github.ref_name }} + env: + ENVIRONMENT_NAME: ${{ github.ref_name }} + FILE_SHARE_RESOURCE_GROUP: "${{ vars.FILE_SHARE_RESOURCE_GROUP }}" + STORAGE_ACCOUNT_NAME: "${{ vars.STORAGE_ACCOUNT_NAME }}" + STORAGE_ACCOUNT_KEY: "${{ secrets.STORAGE_ACCOUNT_KEY }}" + FILE_SHARE_NAME: "${{ vars.FILE_SHARE_NAME }}" + PG_HOST: "${{ secrets.PG_HOST }}" + PG_USER: "${{ secrets.PG_USER }}" + PG_PASSWORD: "${{ secrets.PG_PASSWORD }}" + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + with: + repository: '${{ github.repository_owner }}/cicd-deployment-scripts' + ref: 'pg-dump-restore' + + - name: Download Dump + id: download-dump + shell: bash + run: | + az storage file download-batch \ + --destination . \ + --source ${{ env.FILE_SHARE_NAME }} \ + --pattern "pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}/*" \ + --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \ + --account-key ${{ env.STORAGE_ACCOUNT_KEY }} \ + --no-progress + + echo "PG_DUMP_PATH=$(pwd)/pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}" >> $GITHUB_OUTPUT + + - name: Restore Dump + shell: bash + run: | + bash ./pg/restore.sh \ + -h "${{ env.PG_HOST }}" \ + -u "${{ env.PG_USER }}" \ + -d "${{ inputs.database_name }}" \ + -n "${{ steps.download-dump.outputs.PG_DUMP_PATH }}" \ + -p "${{ env.PG_PASSWORD }}" \ No newline at end of file diff --git a/README.md b/README.md index 0c87be1..0b4d31f 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,8 @@ Scripts used for Kern AI CI/CD efforts. - [K8: Release](#k8-release) - [K8: Restart](#k8-restart) - [K8: Test](#k8-test) +- [Postgres: Dump Database](#postgres-dump-database) +- [Postgres: Restore Database](#postgres-restore-database) - [Parent Images: Build](#parent-images-build) - [Parent Images: Matrix](#parent-images-matrix) - [Parent Images: Submodule Merge](#parent-images-submodule-merge) @@ -190,7 +192,7 @@ Outputs: ### Azure: Function App Deployment -Workflow file: `az_fnapp_deploy.yml` +Workflow file: `az_fa_deploy.yml` Triggers: - workflow_dispatch @@ -590,6 +592,66 @@ Inputs: +### Postgres: Dump Database + +Workflow file: `pg_dump.yml` + +Triggers: +- workflow_dispatch + +Inputs: +- pg_dump_name +- database_name + + + +**Description:** + +- generates a PostgreSQL dump of the database specified by the workflow input +- the dump is stored in the Azure File Storage configured by GitHub Actions Environment Variables + + + +**Jobs:** + +- Postgres: Dump ${{ inputs.database_name }} + - `Generate Dump` + - `Upload Dump` + + + + + +### Postgres: Restore Database + +Workflow file: `pg_restore.yml` + +Triggers: +- workflow_dispatch + +Inputs: +- pg_dump_name +- database_name + + + +**Description:** + +- restores a PostgreSQL dump of the database specified by the workflow input +- the dump is downloaded from the Azure File Storage configured by GitHub Actions Environment Variables + + + +**Jobs:** + +- Postgres: Restore ${{ inputs.database_name }} + - `Download Dump` + - `Restore Dump` + + + + + ### Parent Images: Build Workflow file: `pi_build.yml` @@ -618,8 +680,18 @@ Triggers: - `Set up Python` - `Install Dependencies` - `Compile Requirements` - - `Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}` - - `Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}-arm64` + - `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}` + - `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64` + +- Parent Images: App + - `Set up Python` + - `Install Dependencies` + - `Compile Requirements` + - `Clone ${{ matrix.app }}` + - `Compile Requirements (Python)` + - `Compile Requirements (Next)` + - `Perform Edit/Git Operations (Python)` + - `Perform Edit/Git Operations (Next)` @@ -636,6 +708,7 @@ Inputs: - repository - checkout_ref - parent_image_type +- edit_dockerfile Outputs: - parent_image_type @@ -713,7 +786,10 @@ Triggers: - `Install Dependencies` - `Perform Edit/Git Operations` -- GitHub: Delete Branch +- GitHub: Delete Submodule Branch + - `Delete Branch` + +- GitHub: Delete App Branch - `Delete Branch` @@ -743,19 +819,14 @@ Triggers: **Jobs:** -- Configure Head Branch Name - - `Configure branch name` - - pi-matrix - Parent Images: Docker Build - `Set up Python` - `Install Dependencies` - `Compile Requirements` - - `Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}` - - `Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}-arm64` - - `Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}` - - `Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}-arm64` + - `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}` + - `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.sha }}-${{ env.PARENT_IMAGE_TYPE }}` - Parent Images: App - `Set up Python` @@ -769,9 +840,6 @@ Triggers: - GitHub: Delete Branch - `Delete Branch` -- GitHub: Delete Branch - - `Delete Branch` - @@ -798,9 +866,11 @@ Triggers: - pi-matrix -- Parent Images: Dockerfile +- Parent Images: Dockerfile - `Perform Edit/Git Operations` +- call-gh-release + diff --git a/docs/docs.json b/docs/docs.json index a4466e6..584bc2c 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -139,7 +139,7 @@ } ] }, - "az_fnapp_deploy.yml": { + "az_fa_deploy.yml": { "name": "Azure: Function App Deployment", "trigger": [ "workflow_dispatch", @@ -513,6 +513,58 @@ } ] }, + "pg_dump.yml": { + "name": "Postgres: Dump Database", + "trigger": [ + "workflow_dispatch" + ], + "inputs": [ + "pg_dump_name", + "database_name" + ], + "outputs": [], + "description": [ + "generates a PostgreSQL dump of the database specified by the workflow input", + "the dump is stored in the Azure File Storage configured by GitHub Actions Environment Variables" + ], + "troubleshooting": [], + "jobs": [ + { + "name": "pg-dump", + "descriptive_name": "Postgres: Dump ${{ inputs.database_name }}", + "steps": [ + "Generate Dump", + "Upload Dump" + ] + } + ] + }, + "pg_restore.yml": { + "name": "Postgres: Restore Database", + "trigger": [ + "workflow_dispatch" + ], + "inputs": [ + "pg_dump_name", + "database_name" + ], + "outputs": [], + "description": [ + "restores a PostgreSQL dump of the database specified by the workflow input", + "the dump is downloaded from the Azure File Storage configured by GitHub Actions Environment Variables" + ], + "troubleshooting": [], + "jobs": [ + { + "name": "pg-restore", + "descriptive_name": "Postgres: Restore ${{ inputs.database_name }}", + "steps": [ + "Download Dump", + "Restore Dump" + ] + } + ] + }, "pi_build.yml": { "name": "Parent Images: Build", "trigger": [ @@ -544,8 +596,22 @@ "Set up Python", "Install Dependencies", "Compile Requirements", - "Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}", - "Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}-arm64" + "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}", + "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64" + ] + }, + { + "name": "pi-update-app", + "descriptive_name": "Parent Images: App", + "steps": [ + "Set up Python", + "Install Dependencies", + "Compile Requirements", + "Clone ${{ matrix.app }}", + "Compile Requirements (Python)", + "Compile Requirements (Next)", + "Perform Edit/Git Operations (Python)", + "Perform Edit/Git Operations (Next)" ] } ] @@ -558,7 +624,8 @@ "inputs": [ "repository", "checkout_ref", - "parent_image_type" + "parent_image_type", + "edit_dockerfile" ], "outputs": [ "parent_image_type", @@ -614,7 +681,14 @@ }, { "name": "gh-delete-submodule-branches", - "descriptive_name": "GitHub: Delete Branch", + "descriptive_name": "GitHub: Delete Submodule Branch", + "steps": [ + "Delete Branch" + ] + }, + { + "name": "gh-delete-app-branches", + "descriptive_name": "GitHub: Delete App Branch", "steps": [ "Delete Branch" ] @@ -638,13 +712,6 @@ "worked around by manually performing the [requirements compilation](https://www.notion.so/kern-ai/Docker-Base-Images-9d858b002ff840d3b0a3e90ec61d4179?pvs=4#a4450704a486434083710ef071b48cdc)" ], "jobs": [ - { - "name": "configure-branch-name", - "descriptive_name": "Configure Head Branch Name", - "steps": [ - "Configure branch name" - ] - }, { "name": "pi-matrix", "descriptive_name": null, @@ -657,10 +724,8 @@ "Set up Python", "Install Dependencies", "Compile Requirements", - "Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}", - "Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}-arm64", - "Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}", - "Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}-arm64" + "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}", + "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.sha }}-${{ env.PARENT_IMAGE_TYPE }}" ] }, { @@ -682,13 +747,6 @@ "steps": [ "Delete Branch" ] - }, - { - "name": "gh-delete-app-branches", - "descriptive_name": "GitHub: Delete Branch", - "steps": [ - "Delete Branch" - ] } ] }, @@ -712,10 +770,15 @@ }, { "name": "pi-edit", - "descriptive_name": "Parent Images: Dockerfile ", + "descriptive_name": "Parent Images: Dockerfile", "steps": [ "Perform Edit/Git Operations" ] + }, + { + "name": "call-gh-release", + "descriptive_name": null, + "steps": [] } ] }, diff --git a/docs/docs_input.json b/docs/docs_input.json index fcd3f9e..d574722 100644 --- a/docs/docs_input.json +++ b/docs/docs_input.json @@ -199,6 +199,26 @@ "in case of a workflow failure (TBD), ignore the failure and proceed with Pull Request merge" ] }, + "pg_dump.yml": { + "trigger": [ + "workflow_dispatch" + ], + "description": [ + "generates a PostgreSQL dump of the database specified by the workflow input", + "the dump is stored in the Azure File Storage configured by GitHub Actions Environment Variables" + ], + "troubleshooting": [] + }, + "pg_restore.yml": { + "trigger": [ + "workflow_dispatch" + ], + "description": [ + "restores a PostgreSQL dump of the database specified by the workflow input", + "the dump is downloaded from the Azure File Storage configured by GitHub Actions Environment Variables" + ], + "troubleshooting": [] + }, "pi_build.yml": { "trigger": [ "pull_request_opened_synchronized" diff --git a/pg/dump.sh b/pg/dump.sh new file mode 100644 index 0000000..af7ac69 --- /dev/null +++ b/pg/dump.sh @@ -0,0 +1,29 @@ +# !/bin/bash +set -e + +PG_HOST="" +PG_USER="" +PG_PASSWORD="" +PG_DATABASE="" +PG_DUMP_NAME="" + +while getopts h:u:d:n:p: flag +do + case "${flag}" in + h) PG_HOST=${OPTARG};; + u) PG_USER=${OPTARG};; + d) PG_DATABASE=${OPTARG};; + n) PG_DUMP_NAME=${OPTARG};; + p) PG_PASSWORD=${OPTARG};; + esac +done + +# environment variables used to authenticate and configure the postgresql client +# https://www.postgresql.org/docs/8.4/libpq-envars.html +export PGHOST=$PG_HOST +export PGUSER=$PG_USER +export PGPASSWORD=$PG_PASSWORD + +pg_dump --dbname $PG_DATABASE --file pg_dump_${PG_DATABASE}_${PG_DUMP_NAME} --format=d --jobs 2 + +echo "PG_DUMP_PATH=$(pwd)/pg_dump_${PG_DATABASE}_${PG_DUMP_NAME}" >> $GITHUB_OUTPUT diff --git a/pg/restore.sh b/pg/restore.sh new file mode 100644 index 0000000..660582f --- /dev/null +++ b/pg/restore.sh @@ -0,0 +1,37 @@ +# !/bin/bash +set -e + +PG_HOST="" +PG_USER="" +PG_PASSWORD="" +PG_DATABASE="" +PG_DUMP_PATH="" + +while getopts h:u:d:n:p: flag +do + case "${flag}" in + h) PG_HOST=${OPTARG};; + u) PG_USER=${OPTARG};; + d) PG_DATABASE=${OPTARG};; + n) PG_DUMP_PATH=${OPTARG};; + p) PG_PASSWORD=${OPTARG};; + esac +done + +# environment variables used to authenticate and configure the postgresql client +# https://www.postgresql.org/docs/8.4/libpq-envars.html +export PGHOST=$PG_HOST +export PGUSER=$PG_USER +export PGPASSWORD=$PG_PASSWORD + +# terminate existing and block new connections to the database +psql --command "REVOKE CONNECT ON DATABASE $PG_DATABASE FROM PUBLIC, $PG_USER;" +psql --command "SELECT pg_terminate_backend(pid) \ +FROM pg_stat_activity \ +WHERE pid <> pg_backend_pid() AND datname = '$PG_DATABASE'; +" + +dropdb $PG_DATABASE +createdb $PG_DATABASE + +pg_restore --dbname $PG_DATABASE ${PG_DUMP_PATH} --format=d --jobs 2 \ No newline at end of file