From d9ab81e77f5a3ddaa1fbb028669d75f531d42ff3 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 10:57:31 +0200
Subject: [PATCH 01/11] perf: add pg_dump gh action

---
 .github/workflows/pg_dump.yml | 66 +++++++++++++++++++++++++++++++++++
 pg/dump.sh                    | 47 +++++++++++++++++++++++++
 pg/restore.sh                 | 55 +++++++++++++++++++++++++++++
 3 files changed, 168 insertions(+)
 create mode 100644 .github/workflows/pg_dump.yml
 create mode 100644 pg/dump.sh
 create mode 100644 pg/restore.sh

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
new file mode 100644
index 0000000..51fb72c
--- /dev/null
+++ b/.github/workflows/pg_dump.yml
@@ -0,0 +1,66 @@
+name: 'Postgres: Dump'
+
+on:
+  workflow_call:
+    inputs:
+      pg_dump_name:
+        description: 'Dump Name'
+        required: true
+        type: string
+      database_name:
+        description: 'Database Name'
+        required: true
+        default: refinery
+        type: string
+
+jobs:
+  pg-dump:
+    name: 'Postgres: Dump ${{ inputs.database_name }}'
+    runs-on: [self-hosted, "${{ github.ref_name }}"]
+    environment: ${{ github.ref_name }}
+    env:
+      ENVIRONMENT_NAME: ${{ github.ref_name }}
+      FILE_SHARE_RESOURCE_GROUP: "${{ vars.FILE_SHARE_RESOURCE_GROUP }}"
+      STORAGE_ACCOUNT_NAME: "${{ vars.STORAGE_ACCOUNT_NAME }}"
+      STORAGE_ACCOUNT_KEY: "${{ secrets.STORAGE_ACCOUNT_KEY }}"
+      FILE_SHARE_NAME: "${{ vars.FILE_SHARE_NAME }}"
+      PG_HOST: "${{ secrets.PG_HOST }}"
+      PG_USER: "${{ secrets.PG_USER }}"
+      PG_PASSWORD: "${{ secrets.PG_PASSWORD }}"
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+      with:
+        repository: '${{ github.event.repository.owner }}/cicd-deployment-scripts'
+
+    - name: Azure Cloud Login
+      uses: azure/login@v2
+      with:
+        client-id: ${{ secrets.AZURE_CLIENT_ID }}
+        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+    - name: Generate Dump
+      id: generate-dump
+      shell: bash
+      run: |
+        bash cicd-deployment-scripts/pg/dump.sh \
+          -h ${{ env.PG_HOST }} \
+          -u ${{ env.PG_USER }} \
+          -d ${{ inputs.database_name }} \
+          -n ${{ inputs.pg_dump_name }} \
+          -p ${{ env.PG_PASSWORD }} \
+          -r ${{ env.FILE_SHARE_RESOURCE_GROUP}} \
+          -s ${{ env.STORAGE_ACCOUNT_NAME }} \
+          -f ${{ env.FILE_SHARE_NAME }}
+
+    - name: Upload Dump
+      uses: azure/cli@v2
+      with:
+        azcliversion: latest
+        inlineScript: |
+          az storage file upload-batch \
+            --destination ${{ env.FILE_SHARE_NAME }} \
+            --destination-path pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}} \
+            --source ${{ steps.generate-dump.outputs.PG_DUMP_PATH }} \
+            --account-key ${{ env.STORAGE_ACCOUNT_KEY }}
diff --git a/pg/dump.sh b/pg/dump.sh
new file mode 100644
index 0000000..3a938df
--- /dev/null
+++ b/pg/dump.sh
@@ -0,0 +1,47 @@
+# !/bin/bash
+set -e
+
+PG_HOST=""
+PG_USER=""
+PG_PASSWORD=""
+PG_DATABASE=""
+PG_DUMP_NAME=""
+
+RESOURCE_GROUP_NAME=""
+STORAGE_ACCOUNT_NAME=""
+FILE_SHARE_NAME=""
+
+
+while getopts h:u:d:n:p:r:s:f: flag
+do
+    case "${flag}" in
+        h) PG_HOST=${OPTARG};;
+        u) PG_USER=${OPTARG};;
+        d) PG_DATABASE=${OPTARG};;
+        n) PG_DUMP_NAME=${OPTARG};;
+        p) PG_PASSWORD=${OPTARG};;
+        r) RESOURCE_GROUP_NAME=${OPTARG};;
+        s) STORAGE_ACCOUNT_NAME=${OPTARG};;
+        f) FILE_SHARE_NAME=${OPTARG};;
+    esac
+done
+
+# environment variables used to authenticate and configure the postgresql client
+# https://www.postgresql.org/docs/8.4/libpq-envars.html
+export PGHOST=$PG_HOST
+export PGUSER=$PG_USER
+export PGPASSWORD=$PG_PASSWORD
+
+pg_dump --dbname $PG_DATABASE --file pg_dump_${PG_DATABASE}_${PG_DUMP_NAME} --format=d --jobs 2
+
+echo "PG_DUMP_PATH=$(pwd)/pg_dump_${PG_DATABASE}_${PG_DUMP_NAME}" >> $GITHUB_OUTPUT
+
+# STORAGE_ACCOUNT_KEY=$(su - ${admin_username} -c "az storage account keys list \
+#     --resource-group $RESOURCE_GROUP_NAME \
+#     --account-name $STORAGE_ACCOUNT_NAME \
+#     --query '[0].value' --output tsv | tr -d '\"'")
+# az storage file upload-batch \
+#     --destination ${FILE_SHARE_NAME} \
+#     --destination-path pg_dump/${PG_DATABASE}/${PG_DUMP_NAME} \
+#     --source pg_dump_${PG_DATABASE}_${PG_DUMP_NAME} \
+#     --account-key $STORAGE_ACCOUNT_KEY
diff --git a/pg/restore.sh b/pg/restore.sh
new file mode 100644
index 0000000..006d550
--- /dev/null
+++ b/pg/restore.sh
@@ -0,0 +1,55 @@
+# !/bin/bash
+set -e
+
+PG_HOST=""
+PG_USER=""
+PG_PASSWORD=""
+PG_DATABASE=""
+PG_DUMP_NAME=""
+
+RESOURCE_GROUP_NAME=""
+STORAGE_ACCOUNT_NAME=""
+FILE_SHARE_NAME=""
+
+
+while getopts h:u:d:n:p:r:s:f: flag
+do
+    case "${flag}" in
+        h) PG_HOST=${OPTARG};;
+        u) PG_USER=${OPTARG};;
+        d) PG_DATABASE=${OPTARG};;
+        n) PG_DUMP_NAME=${OPTARG};;
+        p) PG_PASSWORD=${OPTARG};;
+        r) RESOURCE_GROUP_NAME=${OPTARG};;
+        s) STORAGE_ACCOUNT_NAME=${OPTARG};;
+        f) FILE_SHARE_NAME=${OPTARG};;
+    esac
+done
+
+# environment variables used to authenticate and configure the postgresql client
+# https://www.postgresql.org/docs/8.4/libpq-envars.html
+export PGHOST=$PG_HOST
+export PGUSER=$PG_USER
+export PGPASSWORD=$PG_PASSWORD
+
+mkdir pg_dump_${PG_DATABASE}_${PG_DUMP_NAME}
+
+az storage file download-batch \
+    --account-name $STORAGE_ACCOUNT_NAME \
+    --destination . \
+    --source ${FILE_SHARE_NAME} \
+    --pattern "pg_dump/${PG_DATABASE}/${PG_DUMP_NAME}/*" \
+    --account-key $STORAGE_ACCOUNT_KEY \
+    --no-progress
+
+# terminate existing and block new connections to the database
+psql --command "REVOKE CONNECT ON DATABASE $PG_DATABASE FROM PUBLIC, $PG_USER;"
+psql --command "SELECT pg_terminate_backend(pid) \
+FROM pg_stat_activity \
+WHERE pid <> pg_backend_pid() AND datname = '$PG_DATABASE';
+"
+
+dropdb $PG_DATABASE
+createdb $PG_DATABASE
+
+pg_restore --dbname $PG_DATABASE pg_dump/${PG_DATABASE}/${PG_DUMP_NAME} --format=d --jobs 2
\ No newline at end of file

From f026eb8d3aa4e1fb2df1dc09573229bd357f5f6e Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:04:16 +0200
Subject: [PATCH 02/11] fix: repo owner on checkout

---
 .github/workflows/pg_dump.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index 51fb72c..18e98f0 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -1,4 +1,4 @@
-name: 'Postgres: Dump'
+name: 'Postgres: Dump Database'
 
 on:
   workflow_call:
@@ -31,7 +31,7 @@ jobs:
     - name: Checkout Repository
       uses: actions/checkout@v2
       with:
-        repository: '${{ github.event.repository.owner }}/cicd-deployment-scripts'
+        repository: '${{ github.repository_owner }}/cicd-deployment-scripts'
 
     - name: Azure Cloud Login
       uses: azure/login@v2
@@ -44,7 +44,7 @@ jobs:
       id: generate-dump
       shell: bash
       run: |
-        bash cicd-deployment-scripts/pg/dump.sh \
+        bash ./pg/dump.sh \
           -h ${{ env.PG_HOST }} \
           -u ${{ env.PG_USER }} \
           -d ${{ inputs.database_name }} \

From d28134aeab71d94f674cea5d2a81cd9d0c751135 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:05:21 +0200
Subject: [PATCH 03/11] fix: add permissions

---
 .github/workflows/pg_dump.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index 18e98f0..c59d015 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -13,6 +13,12 @@ on:
         default: refinery
         type: string
 
+# Special permissions required for AZ CLI OIDC authentication
+permissions:
+  id-token: write
+  contents: read
+  actions: read
+
 jobs:
   pg-dump:
     name: 'Postgres: Dump ${{ inputs.database_name }}'

From 886615d7325203f266d7abc00352c498729c8ed9 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:13:22 +0200
Subject: [PATCH 04/11] perf: remove az cli to always run on self hosted

---
 .github/workflows/pg_dump.yml | 29 +++++++----------------------
 1 file changed, 7 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index c59d015..c65c70c 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -13,12 +13,6 @@ on:
         default: refinery
         type: string
 
-# Special permissions required for AZ CLI OIDC authentication
-permissions:
-  id-token: write
-  contents: read
-  actions: read
-
 jobs:
   pg-dump:
     name: 'Postgres: Dump ${{ inputs.database_name }}'
@@ -39,13 +33,6 @@ jobs:
       with:
         repository: '${{ github.repository_owner }}/cicd-deployment-scripts'
 
-    - name: Azure Cloud Login
-      uses: azure/login@v2
-      with:
-        client-id: ${{ secrets.AZURE_CLIENT_ID }}
-        tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-
     - name: Generate Dump
       id: generate-dump
       shell: bash
@@ -61,12 +48,10 @@ jobs:
           -f ${{ env.FILE_SHARE_NAME }}
 
     - name: Upload Dump
-      uses: azure/cli@v2
-      with:
-        azcliversion: latest
-        inlineScript: |
-          az storage file upload-batch \
-            --destination ${{ env.FILE_SHARE_NAME }} \
-            --destination-path pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}} \
-            --source ${{ steps.generate-dump.outputs.PG_DUMP_PATH }} \
-            --account-key ${{ env.STORAGE_ACCOUNT_KEY }}
+      shell: bash
+      run: |
+        az storage file upload-batch \
+          --destination ${{ env.FILE_SHARE_NAME }} \
+          --destination-path pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}} \
+          --source ${{ steps.generate-dump.outputs.PG_DUMP_PATH }} \
+          --account-key ${{ env.STORAGE_ACCOUNT_KEY }}

From 844ef1d1bc07967cca4501b35baa6ffad9ac4bf1 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:16:21 +0200
Subject: [PATCH 05/11] test: print ls

---
 .github/workflows/pg_dump.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index c65c70c..a134ff4 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -37,7 +37,8 @@ jobs:
       id: generate-dump
       shell: bash
       run: |
-        bash ./pg/dump.sh \
+        ls -al
+        bash ./cicd-deployment-scripts/pg/dump.sh \
           -h ${{ env.PG_HOST }} \
           -u ${{ env.PG_USER }} \
           -d ${{ inputs.database_name }} \

From 3ddbe496647cfb0a023bb55ec982c5b777aebf21 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:17:33 +0200
Subject: [PATCH 06/11] fix: checkout pg-dump-restore branch

---
 .github/workflows/pg_dump.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index a134ff4..47414ac 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -32,13 +32,13 @@ jobs:
       uses: actions/checkout@v2
       with:
         repository: '${{ github.repository_owner }}/cicd-deployment-scripts'
+        ref: 'pg-dump-restore'
 
     - name: Generate Dump
       id: generate-dump
       shell: bash
       run: |
-        ls -al
-        bash ./cicd-deployment-scripts/pg/dump.sh \
+        bash ./pg/dump.sh \
           -h ${{ env.PG_HOST }} \
           -u ${{ env.PG_USER }} \
           -d ${{ inputs.database_name }} \

From 3d5b7bbb97ae70bf9f993575d48264baafdc67ec Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:25:37 +0200
Subject: [PATCH 07/11] fix: wrap script arguments with quotes

---
 .github/workflows/pg_dump.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index 47414ac..2914aa1 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -39,11 +39,11 @@ jobs:
       shell: bash
       run: |
         bash ./pg/dump.sh \
-          -h ${{ env.PG_HOST }} \
-          -u ${{ env.PG_USER }} \
-          -d ${{ inputs.database_name }} \
-          -n ${{ inputs.pg_dump_name }} \
-          -p ${{ env.PG_PASSWORD }} \
+          -h "${{ env.PG_HOST }}" \
+          -u "${{ env.PG_USER }}" \
+          -d "${{ inputs.database_name }}" \
+          -n "${{ inputs.pg_dump_name }}" \
+          -p "${{ env.PG_PASSWORD }}" \
           -r ${{ env.FILE_SHARE_RESOURCE_GROUP}} \
           -s ${{ env.STORAGE_ACCOUNT_NAME }} \
           -f ${{ env.FILE_SHARE_NAME }}

From 17a627f80ccdaac226daa9fa941effdfb275fd07 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:27:50 +0200
Subject: [PATCH 08/11] fix: az file upload-batch arguments

---
 .github/workflows/pg_dump.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index 2914aa1..5f78db7 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -55,4 +55,5 @@ jobs:
           --destination ${{ env.FILE_SHARE_NAME }} \
           --destination-path pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}} \
           --source ${{ steps.generate-dump.outputs.PG_DUMP_PATH }} \
+          --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \
           --account-key ${{ env.STORAGE_ACCOUNT_KEY }}

From 81fd70728a64c486817264bd7a46df0047fd1c98 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:34:56 +0200
Subject: [PATCH 09/11] perf: add pg_restore gh action

---
 .github/workflows/pg_dump.yml    |  5 +--
 .github/workflows/pg_restore.yml | 59 ++++++++++++++++++++++++++++++++
 pg/dump.sh                       | 20 +----------
 pg/restore.sh                    | 26 +++-----------
 4 files changed, 65 insertions(+), 45 deletions(-)
 create mode 100644 .github/workflows/pg_restore.yml

diff --git a/.github/workflows/pg_dump.yml b/.github/workflows/pg_dump.yml
index 5f78db7..ded55c9 100644
--- a/.github/workflows/pg_dump.yml
+++ b/.github/workflows/pg_dump.yml
@@ -43,10 +43,7 @@ jobs:
           -u "${{ env.PG_USER }}" \
           -d "${{ inputs.database_name }}" \
           -n "${{ inputs.pg_dump_name }}" \
-          -p "${{ env.PG_PASSWORD }}" \
-          -r ${{ env.FILE_SHARE_RESOURCE_GROUP}} \
-          -s ${{ env.STORAGE_ACCOUNT_NAME }} \
-          -f ${{ env.FILE_SHARE_NAME }}
+          -p "${{ env.PG_PASSWORD }}"
 
     - name: Upload Dump
       shell: bash
diff --git a/.github/workflows/pg_restore.yml b/.github/workflows/pg_restore.yml
new file mode 100644
index 0000000..8b50280
--- /dev/null
+++ b/.github/workflows/pg_restore.yml
@@ -0,0 +1,59 @@
+name: 'Postgres: Restore Database'
+
+on:
+  workflow_call:
+    inputs:
+      pg_dump_name:
+        description: 'Dump Name'
+        required: true
+        type: string
+      database_name:
+        description: 'Database Name'
+        required: true
+        default: refinery
+        type: string
+
+jobs:
+  pg-restore:
+    name: 'Postgres: Restore ${{ inputs.database_name }}'
+    runs-on: [self-hosted, "${{ github.ref_name }}"]
+    environment: ${{ github.ref_name }}
+    env:
+      ENVIRONMENT_NAME: ${{ github.ref_name }}
+      FILE_SHARE_RESOURCE_GROUP: "${{ vars.FILE_SHARE_RESOURCE_GROUP }}"
+      STORAGE_ACCOUNT_NAME: "${{ vars.STORAGE_ACCOUNT_NAME }}"
+      STORAGE_ACCOUNT_KEY: "${{ secrets.STORAGE_ACCOUNT_KEY }}"
+      FILE_SHARE_NAME: "${{ vars.FILE_SHARE_NAME }}"
+      PG_HOST: "${{ secrets.PG_HOST }}"
+      PG_USER: "${{ secrets.PG_USER }}"
+      PG_PASSWORD: "${{ secrets.PG_PASSWORD }}"
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+      with:
+        repository: '${{ github.repository_owner }}/cicd-deployment-scripts'
+        ref: 'pg-dump-restore'
+
+    - name: Download Dump
+      id: download-dump
+      shell: bash
+      run: |
+        az storage file download-batch \
+          --destination . \
+          --source ${{ env.FILE_SHARE_NAME }} \
+          --pattern "pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}/*" \
+          --account-name ${{ env.STORAGE_ACCOUNT_NAME }} \
+          --account-key ${{ env.STORAGE_ACCOUNT_KEY }} \
+          --no-progress
+
+        echo "PG_DUMP_PATH=$(pwd)/pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}"
+
+    - name: Restore Dump
+      shell: bash
+      run: |
+        bash ./pg/restore.sh \
+          -h "${{ env.PG_HOST }}" \
+          -u "${{ env.PG_USER }}" \
+          -d "${{ inputs.database_name }}" \
+          -n "${{ steps.download-dump.outputs.PG_DUMP_PATH }}" \
+          -p "${{ env.PG_PASSWORD }}"
\ No newline at end of file
diff --git a/pg/dump.sh b/pg/dump.sh
index 3a938df..af7ac69 100644
--- a/pg/dump.sh
+++ b/pg/dump.sh
@@ -7,12 +7,7 @@ PG_PASSWORD=""
 PG_DATABASE=""
 PG_DUMP_NAME=""
 
-RESOURCE_GROUP_NAME=""
-STORAGE_ACCOUNT_NAME=""
-FILE_SHARE_NAME=""
-
-
-while getopts h:u:d:n:p:r:s:f: flag
+while getopts h:u:d:n:p: flag
 do
     case "${flag}" in
         h) PG_HOST=${OPTARG};;
@@ -20,9 +15,6 @@ do
         d) PG_DATABASE=${OPTARG};;
         n) PG_DUMP_NAME=${OPTARG};;
         p) PG_PASSWORD=${OPTARG};;
-        r) RESOURCE_GROUP_NAME=${OPTARG};;
-        s) STORAGE_ACCOUNT_NAME=${OPTARG};;
-        f) FILE_SHARE_NAME=${OPTARG};;
     esac
 done
 
@@ -35,13 +27,3 @@ export PGPASSWORD=$PG_PASSWORD
 pg_dump --dbname $PG_DATABASE --file pg_dump_${PG_DATABASE}_${PG_DUMP_NAME} --format=d --jobs 2
 
 echo "PG_DUMP_PATH=$(pwd)/pg_dump_${PG_DATABASE}_${PG_DUMP_NAME}" >> $GITHUB_OUTPUT
-
-# STORAGE_ACCOUNT_KEY=$(su - ${admin_username} -c "az storage account keys list \
-#     --resource-group $RESOURCE_GROUP_NAME \
-#     --account-name $STORAGE_ACCOUNT_NAME \
-#     --query '[0].value' --output tsv | tr -d '\"'")
-# az storage file upload-batch \
-#     --destination ${FILE_SHARE_NAME} \
-#     --destination-path pg_dump/${PG_DATABASE}/${PG_DUMP_NAME} \
-#     --source pg_dump_${PG_DATABASE}_${PG_DUMP_NAME} \
-#     --account-key $STORAGE_ACCOUNT_KEY
diff --git a/pg/restore.sh b/pg/restore.sh
index 006d550..660582f 100644
--- a/pg/restore.sh
+++ b/pg/restore.sh
@@ -5,24 +5,16 @@ PG_HOST=""
 PG_USER=""
 PG_PASSWORD=""
 PG_DATABASE=""
-PG_DUMP_NAME=""
+PG_DUMP_PATH=""
 
-RESOURCE_GROUP_NAME=""
-STORAGE_ACCOUNT_NAME=""
-FILE_SHARE_NAME=""
-
-
-while getopts h:u:d:n:p:r:s:f: flag
+while getopts h:u:d:n:p: flag
 do
     case "${flag}" in
         h) PG_HOST=${OPTARG};;
         u) PG_USER=${OPTARG};;
         d) PG_DATABASE=${OPTARG};;
-        n) PG_DUMP_NAME=${OPTARG};;
+        n) PG_DUMP_PATH=${OPTARG};;
         p) PG_PASSWORD=${OPTARG};;
-        r) RESOURCE_GROUP_NAME=${OPTARG};;
-        s) STORAGE_ACCOUNT_NAME=${OPTARG};;
-        f) FILE_SHARE_NAME=${OPTARG};;
     esac
 done
 
@@ -32,16 +24,6 @@ export PGHOST=$PG_HOST
 export PGUSER=$PG_USER
 export PGPASSWORD=$PG_PASSWORD
 
-mkdir pg_dump_${PG_DATABASE}_${PG_DUMP_NAME}
-
-az storage file download-batch \
-    --account-name $STORAGE_ACCOUNT_NAME \
-    --destination . \
-    --source ${FILE_SHARE_NAME} \
-    --pattern "pg_dump/${PG_DATABASE}/${PG_DUMP_NAME}/*" \
-    --account-key $STORAGE_ACCOUNT_KEY \
-    --no-progress
-
 # terminate existing and block new connections to the database
 psql --command "REVOKE CONNECT ON DATABASE $PG_DATABASE FROM PUBLIC, $PG_USER;"
 psql --command "SELECT pg_terminate_backend(pid) \
@@ -52,4 +34,4 @@ WHERE pid <> pg_backend_pid() AND datname = '$PG_DATABASE';
 dropdb $PG_DATABASE
 createdb $PG_DATABASE
 
-pg_restore --dbname $PG_DATABASE pg_dump/${PG_DATABASE}/${PG_DUMP_NAME} --format=d --jobs 2
\ No newline at end of file
+pg_restore --dbname $PG_DATABASE ${PG_DUMP_PATH} --format=d --jobs 2
\ No newline at end of file

From 24210f452dd96bff1e557673d474c63f3baf640d Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Wed, 2 Apr 2025 11:38:18 +0200
Subject: [PATCH 10/11] fix: download-dump outputs

---
 .github/workflows/pg_restore.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pg_restore.yml b/.github/workflows/pg_restore.yml
index 8b50280..641d200 100644
--- a/.github/workflows/pg_restore.yml
+++ b/.github/workflows/pg_restore.yml
@@ -46,7 +46,7 @@ jobs:
           --account-key ${{ env.STORAGE_ACCOUNT_KEY }} \
           --no-progress
 
-        echo "PG_DUMP_PATH=$(pwd)/pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}"
+        echo "PG_DUMP_PATH=$(pwd)/pg_dump/${{ inputs.database_name }}/${{ inputs.pg_dump_name}}" >> $GITHUB_OUTPUT
 
     - name: Restore Dump
       shell: bash

From aaf438c2be5a3c7df6df4abb98c183e2f6e60845 Mon Sep 17 00:00:00 2001
From: andhreljaKern <andrea.hrelja@kern.ai>
Date: Tue, 22 Apr 2025 11:07:12 +0200
Subject: [PATCH 11/11] docs: update pg dump/restore docs

---
 README.md            | 100 ++++++++++++++++++++++++++++++++------
 docs/docs.json       | 111 +++++++++++++++++++++++++++++++++----------
 docs/docs_input.json |  20 ++++++++
 3 files changed, 192 insertions(+), 39 deletions(-)

diff --git a/README.md b/README.md
index 0c87be1..0b4d31f 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,8 @@ Scripts used for Kern AI CI/CD efforts.
 - [K8: Release](#k8-release)
 - [K8: Restart](#k8-restart)
 - [K8: Test](#k8-test)
+- [Postgres: Dump Database](#postgres-dump-database)
+- [Postgres: Restore Database](#postgres-restore-database)
 - [Parent Images: Build](#parent-images-build)
 - [Parent Images: Matrix](#parent-images-matrix)
 - [Parent Images: Submodule Merge](#parent-images-submodule-merge)
@@ -190,7 +192,7 @@ Outputs:
 
 ### Azure: Function App Deployment
 
-Workflow file: `az_fnapp_deploy.yml`
+Workflow file: `az_fa_deploy.yml`
 
 Triggers:
 - workflow_dispatch
@@ -590,6 +592,66 @@ Inputs:
 
 
 
+### Postgres: Dump Database
+
+Workflow file: `pg_dump.yml`
+
+Triggers:
+- workflow_dispatch
+
+Inputs:
+- pg_dump_name
+- database_name
+
+
+
+**Description:**
+
+- generates a PostgreSQL dump of the database specified by the workflow input
+- the dump is stored in the Azure File Storage configured by GitHub Actions Environment Variables
+
+
+
+**Jobs:**
+
+- Postgres: Dump ${{ inputs.database_name }}
+	- `Generate Dump`
+	- `Upload Dump`
+
+
+
+
+
+### Postgres: Restore Database
+
+Workflow file: `pg_restore.yml`
+
+Triggers:
+- workflow_dispatch
+
+Inputs:
+- pg_dump_name
+- database_name
+
+
+
+**Description:**
+
+- restores a PostgreSQL dump of the database specified by the workflow input
+- the dump is downloaded from the Azure File Storage configured by GitHub Actions Environment Variables
+
+
+
+**Jobs:**
+
+- Postgres: Restore ${{ inputs.database_name }}
+	- `Download Dump`
+	- `Restore Dump`
+
+
+
+
+
 ### Parent Images: Build
 
 Workflow file: `pi_build.yml`
@@ -618,8 +680,18 @@ Triggers:
 	- `Set up Python`
 	- `Install Dependencies`
 	- `Compile Requirements`
-	- `Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}`
-	- `Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}-arm64`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64`
+
+- Parent Images: App
+	- `Set up Python`
+	- `Install Dependencies`
+	- `Compile Requirements`
+	- `Clone ${{ matrix.app }}`
+	- `Compile Requirements (Python)`
+	- `Compile Requirements (Next)`
+	- `Perform Edit/Git Operations (Python)`
+	- `Perform Edit/Git Operations (Next)`
 
 
 
@@ -636,6 +708,7 @@ Inputs:
 - repository
 - checkout_ref
 - parent_image_type
+- edit_dockerfile
 
 Outputs:
 - parent_image_type
@@ -713,7 +786,10 @@ Triggers:
 	- `Install Dependencies`
 	- `Perform Edit/Git Operations`
 
-- GitHub: Delete Branch
+- GitHub: Delete Submodule Branch
+	- `Delete Branch`
+
+- GitHub: Delete App Branch
 	- `Delete Branch`
 
 
@@ -743,19 +819,14 @@ Triggers:
 
 **Jobs:**
 
-- Configure Head Branch Name
-	- `Configure branch name`
-
 - pi-matrix
 
 - Parent Images: Docker Build
 	- `Set up Python`
 	- `Install Dependencies`
 	- `Compile Requirements`
-	- `Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}`
-	- `Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}-arm64`
-	- `Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}`
-	- `Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}-arm64`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}`
+	- `Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.sha }}-${{ env.PARENT_IMAGE_TYPE }}`
 
 - Parent Images: App
 	- `Set up Python`
@@ -769,9 +840,6 @@ Triggers:
 - GitHub: Delete Branch
 	- `Delete Branch`
 
-- GitHub: Delete Branch
-	- `Delete Branch`
-
 
 
 
@@ -798,9 +866,11 @@ Triggers:
 
 - pi-matrix
 
-- Parent Images: Dockerfile 
+- Parent Images: Dockerfile
 	- `Perform Edit/Git Operations`
 
+- call-gh-release
+
 
 
 
diff --git a/docs/docs.json b/docs/docs.json
index a4466e6..584bc2c 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -139,7 +139,7 @@
       }
     ]
   },
-  "az_fnapp_deploy.yml": {
+  "az_fa_deploy.yml": {
     "name": "Azure: Function App Deployment",
     "trigger": [
       "workflow_dispatch",
@@ -513,6 +513,58 @@
       }
     ]
   },
+  "pg_dump.yml": {
+    "name": "Postgres: Dump Database",
+    "trigger": [
+      "workflow_dispatch"
+    ],
+    "inputs": [
+      "pg_dump_name",
+      "database_name"
+    ],
+    "outputs": [],
+    "description": [
+      "generates a PostgreSQL dump of the database specified by the workflow input",
+      "the dump is stored in the Azure File Storage configured by GitHub Actions Environment Variables"
+    ],
+    "troubleshooting": [],
+    "jobs": [
+      {
+        "name": "pg-dump",
+        "descriptive_name": "Postgres: Dump ${{ inputs.database_name }}",
+        "steps": [
+          "Generate Dump",
+          "Upload Dump"
+        ]
+      }
+    ]
+  },
+  "pg_restore.yml": {
+    "name": "Postgres: Restore Database",
+    "trigger": [
+      "workflow_dispatch"
+    ],
+    "inputs": [
+      "pg_dump_name",
+      "database_name"
+    ],
+    "outputs": [],
+    "description": [
+      "restores a PostgreSQL dump of the database specified by the workflow input",
+      "the dump is downloaded from the Azure File Storage configured by GitHub Actions Environment Variables"
+    ],
+    "troubleshooting": [],
+    "jobs": [
+      {
+        "name": "pg-restore",
+        "descriptive_name": "Postgres: Restore ${{ inputs.database_name }}",
+        "steps": [
+          "Download Dump",
+          "Restore Dump"
+        ]
+      }
+    ]
+  },
   "pi_build.yml": {
     "name": "Parent Images: Build",
     "trigger": [
@@ -544,8 +596,22 @@
           "Set up Python",
           "Install Dependencies",
           "Compile Requirements",
-          "Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}",
-          "Build & Push refinery-parent-images:${{ needs.configure-branch-name.outputs.gh_head_ref }}-${{ matrix.parent_image_type }}-arm64"
+          "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}",
+          "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ env.HEAD_REF }}-${{ matrix.parent_image_type }}-arm64"
+        ]
+      },
+      {
+        "name": "pi-update-app",
+        "descriptive_name": "Parent Images: App",
+        "steps": [
+          "Set up Python",
+          "Install Dependencies",
+          "Compile Requirements",
+          "Clone ${{ matrix.app }}",
+          "Compile Requirements (Python)",
+          "Compile Requirements (Next)",
+          "Perform Edit/Git Operations (Python)",
+          "Perform Edit/Git Operations (Next)"
         ]
       }
     ]
@@ -558,7 +624,8 @@
     "inputs": [
       "repository",
       "checkout_ref",
-      "parent_image_type"
+      "parent_image_type",
+      "edit_dockerfile"
     ],
     "outputs": [
       "parent_image_type",
@@ -614,7 +681,14 @@
       },
       {
         "name": "gh-delete-submodule-branches",
-        "descriptive_name": "GitHub: Delete Branch",
+        "descriptive_name": "GitHub: Delete Submodule Branch",
+        "steps": [
+          "Delete Branch"
+        ]
+      },
+      {
+        "name": "gh-delete-app-branches",
+        "descriptive_name": "GitHub: Delete App Branch",
         "steps": [
           "Delete Branch"
         ]
@@ -638,13 +712,6 @@
       "worked around by manually performing the [requirements compilation](https://www.notion.so/kern-ai/Docker-Base-Images-9d858b002ff840d3b0a3e90ec61d4179?pvs=4#a4450704a486434083710ef071b48cdc)"
     ],
     "jobs": [
-      {
-        "name": "configure-branch-name",
-        "descriptive_name": "Configure Head Branch Name",
-        "steps": [
-          "Configure branch name"
-        ]
-      },
       {
         "name": "pi-matrix",
         "descriptive_name": null,
@@ -657,10 +724,8 @@
           "Set up Python",
           "Install Dependencies",
           "Compile Requirements",
-          "Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}",
-          "Build & Push refinery-parent-images:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}-arm64",
-          "Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}",
-          "Build & Push refinery-parent-images:sha-${{ env.PARENT_IMAGE_TYPE }}-arm64"
+          "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.event.pull_request.base.ref }}-${{ env.PARENT_IMAGE_TYPE }}",
+          "Build & Push ${{ env.PARENT_IMAGE_NAME }}:${{ github.sha }}-${{ env.PARENT_IMAGE_TYPE }}"
         ]
       },
       {
@@ -682,13 +747,6 @@
         "steps": [
           "Delete Branch"
         ]
-      },
-      {
-        "name": "gh-delete-app-branches",
-        "descriptive_name": "GitHub: Delete Branch",
-        "steps": [
-          "Delete Branch"
-        ]
       }
     ]
   },
@@ -712,10 +770,15 @@
       },
       {
         "name": "pi-edit",
-        "descriptive_name": "Parent Images: Dockerfile ",
+        "descriptive_name": "Parent Images: Dockerfile",
         "steps": [
           "Perform Edit/Git Operations"
         ]
+      },
+      {
+        "name": "call-gh-release",
+        "descriptive_name": null,
+        "steps": []
       }
     ]
   },
diff --git a/docs/docs_input.json b/docs/docs_input.json
index fcd3f9e..d574722 100644
--- a/docs/docs_input.json
+++ b/docs/docs_input.json
@@ -199,6 +199,26 @@
       "in case of a workflow failure (TBD), ignore the failure and proceed with Pull Request merge"
     ]
   },
+  "pg_dump.yml": {
+    "trigger": [
+      "workflow_dispatch"
+    ],
+    "description": [
+      "generates a PostgreSQL dump of the database specified by the workflow input",
+      "the dump is stored in the Azure File Storage configured by GitHub Actions Environment Variables"
+    ],
+    "troubleshooting": []
+  },
+  "pg_restore.yml": {
+    "trigger": [
+      "workflow_dispatch"
+    ],
+    "description": [
+      "restores a PostgreSQL dump of the database specified by the workflow input",
+      "the dump is downloaded from the Azure File Storage configured by GitHub Actions Environment Variables"
+    ],
+    "troubleshooting": []
+  },
   "pi_build.yml": {
     "trigger": [
       "pull_request_opened_synchronized"