Postgres DB backup

.github/workflows/pg_dump.yml

@@ -0,0 +1,56 @@
+name: 'Postgres: Dump Database'
+
+on:
+  workflow_call:
+    inputs:
+      pg_dump_name:
+        description: 'Dump Name'
+        required: true
+        type: string
+      database_name:
+        description: 'Database Name'
+        required: true
+        default: refinery
+        type: string
+
+jobs:
+  pg-dump:
+    name: 'Postgres: Dump ${{ inputs.database_name }}'
+    runs-on: [self-hosted, "${{ github.ref_name }}"]
+    environment: ${{ github.ref_name }}
+    env:
+      ENVIRONMENT_NAME: ${{ github.ref_name }}
+      FILE_SHARE_RESOURCE_GROUP: "${{ vars.FILE_SHARE_RESOURCE_GROUP }}"
+      STORAGE_ACCOUNT_NAME: "${{ vars.STORAGE_ACCOUNT_NAME }}"
+      STORAGE_ACCOUNT_KEY: "${{ secrets.STORAGE_ACCOUNT_KEY }}"
+      FILE_SHARE_NAME: "${{ vars.FILE_SHARE_NAME }}"
+      PG_HOST: "${{ secrets.PG_HOST }}"
+      PG_USER: "${{ secrets.PG_USER }}"
+      PG_PASSWORD: "${{ secrets.PG_PASSWORD }}"
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+      with:
+        repository: '${{ github.repository_owner }}/cicd-deployment-scripts'
+        ref: 'pg-dump-restore'
+
+    - name: Generate Dump
+      id: generate-dump
+      shell: bash
+      run: |
+        bash ./pg/dump.sh \
+          -h "${{ env.PG_HOST }}" \
+          -u "${{ env.PG_USER }}" \
+          -d "${{ inputs.database_name }}" \
+          -n "${{ inputs.pg_dump_name }}" \
+          -p "${{ env.PG_PASSWORD }}"
+
+    - name: Upload Dump
+      shell: bash
+      run: |
+        az storage file upload-batch \
+          --destination ${{ env.FILE_SHARE_NAME }} \
+          --destination-path pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}} \
+          --source ${{ steps.generate-dump.outputs.PG_DUMP_PATH }} \
+          --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \
+          --account-key ${{ env.STORAGE_ACCOUNT_KEY }}

.github/workflows/pg_restore.yml

@@ -0,0 +1,59 @@
+name: 'Postgres: Restore Database'
+
+on:
+  workflow_call:
+    inputs:
+      pg_dump_name:
+        description: 'Dump Name'
+        required: true
+        type: string
+      database_name:
+        description: 'Database Name'
+        required: true
+        default: refinery
+        type: string
+
+jobs:
+  pg-restore:
+    name: 'Postgres: Restore ${{ inputs.database_name }}'
+    runs-on: [self-hosted, "${{ github.ref_name }}"]
+    environment: ${{ github.ref_name }}
+    env:
+      ENVIRONMENT_NAME: ${{ github.ref_name }}
+      FILE_SHARE_RESOURCE_GROUP: "${{ vars.FILE_SHARE_RESOURCE_GROUP }}"
+      STORAGE_ACCOUNT_NAME: "${{ vars.STORAGE_ACCOUNT_NAME }}"
+      STORAGE_ACCOUNT_KEY: "${{ secrets.STORAGE_ACCOUNT_KEY }}"
+      FILE_SHARE_NAME: "${{ vars.FILE_SHARE_NAME }}"
+      PG_HOST: "${{ secrets.PG_HOST }}"
+      PG_USER: "${{ secrets.PG_USER }}"
+      PG_PASSWORD: "${{ secrets.PG_PASSWORD }}"
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+      with:
+        repository: '${{ github.repository_owner }}/cicd-deployment-scripts'
+        ref: 'pg-dump-restore'
+
+    - name: Download Dump
+      id: download-dump
+      shell: bash
+      run: |
+        az storage file download-batch \
+          --destination . \
+          --source ${{ env.FILE_SHARE_NAME }} \
+          --pattern "pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}/*" \
+          --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \
+          --account-key ${{ env.STORAGE_ACCOUNT_KEY }} \
+          --no-progress
+
+        echo "PG_DUMP_PATH=$(pwd)/pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}" >> $GITHUB_OUTPUT
+
+    - name: Restore Dump
+      shell: bash
+      run: |
+        bash ./pg/restore.sh \
+          -h "${{ env.PG_HOST }}" \
+          -u "${{ env.PG_USER }}" \
+          -d "${{ inputs.database_name }}" \
+          -n "${{ steps.download-dump.outputs.PG_DUMP_PATH }}" \
+          -p "${{ env.PG_PASSWORD }}"

README.md

@@ -23,6 +23,8 @@ Scripts used for Kern AI CI/CD efforts.
 - [K8: Release](#k8-release)
 - [K8: Restart](#k8-restart)
 - [K8: Test](#k8-test)
+- [Postgres: Dump Database](#postgres-dump-database)
+- [Postgres: Restore Database](#postgres-restore-database)
 - [Parent Images: Build](#parent-images-build)
 - [Parent Images: Matrix](#parent-images-matrix)
 - [Parent Images: Submodule Merge](#parent-images-submodule-merge)
@@ -190,7 +192,7 @@ Outputs:
 
 ### Azure: Function App Deployment
 
-Workflow file: `az_fnapp_deploy.yml`
+Workflow file: `az_fa_deploy.yml`
 
 Triggers:
 - workflow_dispatch
@@ -590,6 +592,66 @@ Inputs:
 
 
 
+### Postgres: Dump Database
+
+Workflow file: `pg_dump.yml`
+
+Triggers:
+- workflow_dispatch
+
+Inputs:
+- pg_dump_name
+- database_name
+
+
+
+**Description:**
+
+- generates a PostgreSQL dump of the database specified by the workflow input
+- the dump is stored in the Azure File Storage configured by GitHub Actions Environment Variables
+
+
+
+**Jobs:**
+
+- Postgres: Dump ${{ inputs.database_name }}
+	- `Generate Dump`
+	- `Upload Dump`
+
+
+
+
+
+### Postgres: Restore Database
+
+Workflow file: `pg_restore.yml`
+
+Triggers:
+- workflow_dispatch
+
+Inputs:
+- pg_dump_name
+- database_name
+
+
+
+**Description:**
+
+- restores a PostgreSQL dump of the database specified by the workflow input
+- the dump is downloaded from the Azure File Storage configured by GitHub Actions Environment Variables
+
+
+
+**Jobs:**
+
+- Postgres: Restore ${{ inputs.database_name }}
+	- `Download Dump`
+	- `Restore Dump`
+
+
+
+
+
 ### Parent Images: Build
 
 Workflow file: `pi_build.yml`
@@ -618,8 +680,18 @@ Triggers:
 	- `Set up Python`
 	- `Install Dependencies`
 	- `Compile Requirements`
-	- `Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}`
-	- `Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}-arm64`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64`
+
+- Parent Images: App
+	- `Set up Python`
+	- `Install Dependencies`
+	- `Compile Requirements`
+	- `Clone ${{ matrix.app }}`
+	- `Compile Requirements (Python)`
+	- `Compile Requirements (Next)`
+	- `Perform Edit/Git Operations (Python)`
+	- `Perform Edit/Git Operations (Next)`
 
 
 
@@ -636,6 +708,7 @@ Inputs:
 - repository
 - checkout_ref
 - parent_image_type
+- edit_dockerfile
 
 Outputs:
 - parent_image_type
@@ -713,7 +786,10 @@ Triggers:
 	- `Install Dependencies`
 	- `Perform Edit/Git Operations`
 
-- GitHub: Delete Branch
+- GitHub: Delete Submodule Branch
+	- `Delete Branch`
+
+- GitHub: Delete App Branch
 	- `Delete Branch`
 
 
@@ -743,19 +819,14 @@ Triggers:
 
 **Jobs:**
 
-- Configure Head Branch Name
-	- `Configure branch name`
-
 - pi-matrix
 
 - Parent Images: Docker Build
 	- `Set up Python`
 	- `Install Dependencies`
 	- `Compile Requirements`
-	- `Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}`
-	- `Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}-arm64`
-	- `Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}`
-	- `Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}-arm64`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.sha }}-${{ env.PARENT_IMAGE_TYPE }}`
 
 - Parent Images: App
 	- `Set up Python`
@@ -769,9 +840,6 @@ Triggers:
 - GitHub: Delete Branch
 	- `Delete Branch`
 
-- GitHub: Delete Branch
-	- `Delete Branch`
-
 
 
 
@@ -798,9 +866,11 @@ Triggers:
 
 - pi-matrix
 
-- Parent Images: Dockerfile 
+- Parent Images: Dockerfile
 	- `Perform Edit/Git Operations`
 
+- call-gh-release
+