Add patch stats (#22)

JelteF · JelteF · commit e7150a5b54d2 · 2025-02-16T12:12:06.000+01:00
This syncs and displays some of the new patch statistics that the CFbot
gathers on the commitfest patch page. These stats are:
1. Version of the patch
2. Number of patches in the last patchset
3. Additions/deletions in the first patch of the last patchset
4. Total additions/deletions when combining all patches in the last
patchset

It also displays the "Total additions/deletions" in the commitfest
overview.
diff --git a/media/commitfest/css/commitfest.css b/media/commitfest/css/commitfest.css
@@ -73,3 +73,13 @@ div.form-group div.controls input.threadpick-input {
 .github-logo {
     height: 20px;
 }
+
+.additions {
+    font-weight: bold;
+    color: green;
+}
+
+.deletions {
+    font-weight: bold;
+    color: red;
+}
diff --git a/pgcommitfest/commitfest/fixtures/commitfest_data.json b/pgcommitfest/commitfest/fixtures/commitfest_data.json
@@ -132,6 +132,27 @@
         ]
     }
 },
+{
+    "model": "commitfest.patch",
+    "pk": 4,
+    "fields": {
+        "name": "create subscription with (origin = none, copy_data = none)",
+        "topic": 3,
+        "wikilink": "",
+        "gitlink": "",
+        "targetversion": null,
+        "committer": null,
+        "created": "2025-01-31T13:30:19.744",
+        "modified": "2025-01-31T13:30:21.305",
+        "lastmail": "2025-01-20T12:44:40",
+        "authors": [],
+        "reviewers": [],
+        "subscribers": [],
+        "mailthread_set": [
+            4
+        ]
+    }
+},
 {
     "model": "commitfest.patchoncommitfest",
     "pk": 1,
@@ -176,6 +197,17 @@
         "status": 1
     }
 },
+{
+    "model": "commitfest.patchoncommitfest",
+    "pk": 5,
+    "fields": {
+        "patch": 4,
+        "commitfest": 2,
+        "enterdate": "2025-01-31T13:30:19.745",
+        "leavedate": null,
+        "status": 1
+    }
+},
 {
     "model": "commitfest.patchhistory",
     "pk": 1,
@@ -264,6 +296,28 @@
         "what": "Attached mail thread example@message-3"
     }
 },
+{
+    "model": "commitfest.patchhistory",
+    "pk": 9,
+    "fields": {
+        "patch": 4,
+        "date": "2025-01-31T13:30:19.745",
+        "by": 1,
+        "by_cfbot": false,
+        "what": "Created patch record"
+    }
+},
+{
+    "model": "commitfest.patchhistory",
+    "pk": 10,
+    "fields": {
+        "patch": 4,
+        "date": "2025-01-31T13:30:19.748",
+        "by": 1,
+        "by_cfbot": false,
+        "what": "Attached mail thread example@message-4"
+    }
+},
 {
     "model": "commitfest.mailthread",
     "pk": 1,
@@ -306,6 +360,20 @@
         "latestmsgid": "example@message-3"
     }
 },
+{
+    "model": "commitfest.mailthread",
+    "pk": 4,
+    "fields": {
+        "messageid": "example@message-4",
+        "subject": "Re: create subscription with (origin = none, copy_data = on)",
+        "firstmessage": "2025-01-20T12:44:40",
+        "firstauthor": "test@test.com",
+        "latestmessage": "2025-01-20T12:44:40",
+        "latestauthor": "test@test.com",
+        "latestsubject": "Re: create subscription with (origin = none, copy_data = on)",
+        "latestmsgid": "example@message-4"
+    }
+},
 {
     "model": "commitfest.patchstatus",
     "pk": 1,
@@ -381,7 +449,13 @@
         "status": "finished",
         "needs_rebase_since": null,
         "created": "2025-01-26T22:06:02.980",
-        "modified": "2025-01-26T22:06:02.980"
+        "modified": "2025-01-29T22:50:37.805",
+        "version": "",
+        "patch_count": 5,
+        "first_additions": 10,
+        "first_deletions": 5,
+        "all_additions": 834,
+        "all_deletions": 128
     }
 },
 {
@@ -395,7 +469,13 @@
         "status": "failed",
         "needs_rebase_since": null,
         "created": "2025-01-26T22:11:09.961",
-        "modified": "2025-01-26T22:20:39.372"
+        "modified": "2025-01-26T22:20:39.372",
+        "version": null,
+        "patch_count": null,
+        "first_additions": null,
+        "first_deletions": null,
+        "all_additions": null,
+        "all_deletions": null
     }
 },
 {
@@ -409,7 +489,33 @@
         "status": "failed",
         "needs_rebase_since": null,
         "created": "2025-01-26T22:22:46.602",
-        "modified": "2025-01-26T22:26:41.587"
+        "modified": "2025-01-29T22:58:51.032",
+        "version": "",
+        "patch_count": 3,
+        "first_additions": 345,
+        "first_deletions": 158,
+        "all_additions": 8764,
+        "all_deletions": 345
+    }
+},
+{
+    "model": "commitfest.cfbotbranch",
+    "pk": 4,
+    "fields": {
+        "branch_id": 76,
+        "branch_name": "cf/4",
+        "commit_id": "abcggg",
+        "apply_url": "http://cfbot.cputube.org/patch_4748.log",
+        "status": "testing",
+        "needs_rebase_since": null,
+        "created": "2025-01-31T13:32:22.017",
+        "modified": "2025-01-31T13:32:22.017",
+        "version": "",
+        "patch_count": 1,
+        "first_additions": 123,
+        "first_deletions": 14,
+        "all_additions": 123,
+        "all_deletions": 14
     }
 },
 {
diff --git a/pgcommitfest/commitfest/migrations/0009_extra_branch_fields.py b/pgcommitfest/commitfest/migrations/0009_extra_branch_fields.py
@@ -0,0 +1,43 @@
+# Generated by Django 4.2.17 on 2025-01-31 11:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("commitfest", "0008_move_mail_thread_many_to_many"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='cfbotbranch',
+            name='all_additions',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='cfbotbranch',
+            name='all_deletions',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='cfbotbranch',
+            name='first_additions',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='cfbotbranch',
+            name='first_deletions',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='cfbotbranch',
+            name='patch_count',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='cfbotbranch',
+            name='version',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]
diff --git a/pgcommitfest/commitfest/models.py b/pgcommitfest/commitfest/models.py
@@ -380,6 +380,12 @@ class CfbotBranch(models.Model):
     needs_rebase_since = models.DateTimeField(null=True, blank=True)
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
+    version = models.TextField(null=True, blank=True)
+    patch_count = models.IntegerField(null=True, blank=True)
+    first_additions = models.IntegerField(null=True, blank=True)
+    first_deletions = models.IntegerField(null=True, blank=True)
+    all_additions = models.IntegerField(null=True, blank=True)
+    all_deletions = models.IntegerField(null=True, blank=True)
 
     def save(self, *args, **kwargs):
         """Only used by the admin panel to save empty commit id as NULL
diff --git a/pgcommitfest/commitfest/templates/commitfest.html b/pgcommitfest/commitfest/templates/commitfest.html
@@ -65,6 +65,7 @@ <h3>{{p.is_open|yesno:"Active patches,Closed patches"}}</h3>
    <th>Status</th>
    <th>Ver</th>
    <th>CI status</th>
+   <th><a href="#" style="color:#333333;" onclick="return sortpatches(6);">Stats</a>{%if sortkey == 6%}<div style="float:right;"><i class="glyphicon glyphicon-arrow-down"></i></div>{%endif%}</th>
    <th>Author</th>
    <th>Reviewers</th>
    <th>Committer</th>
@@ -113,6 +114,11 @@ <h3>{{p.is_open|yesno:"Active patches,Closed patches"}}</h3>
     </a>
     {%endif%}
    </td>
+   <td>
+    {%if p.cfbot_results and p.cfbot_results.all_additions is not none %}
+    <span class="additions">+{{ p.cfbot_results.all_additions }}</span><span class="deletions">&#8722;{{ p.cfbot_results.all_deletions }}</span>
+    {%endif%}
+   </td>
    <td>{{p.author_names|default:''}}</td>
    <td>{{p.reviewer_names|default:''}}</td>
    <td>{{p.committer|default:''}}</td>
diff --git a/pgcommitfest/commitfest/templates/patch.html b/pgcommitfest/commitfest/templates/patch.html
@@ -49,6 +49,20 @@
         </a>
       </td>
     </tr>
+    <tr>
+     <th>Stats (from CFBot)</th>
+     <td>
+      {%if cfbot_branch and cfbot_branch.commit_id %}
+      {%if cfbot_branch.version %}
+      Patch version: {{ cfbot_branch.version }},
+      {%endif%}
+      Patch count: {{ cfbot_branch.patch_count }},
+      First patch: <span class="additions">+{{ cfbot_branch.first_additions }}</span><span class="deletions">&#8722;{{ cfbot_branch.first_deletions }}</span>,
+      All patches: <span class="additions">+{{ cfbot_branch.all_additions }}</span><span class="deletions">&#8722;{{ cfbot_branch.all_deletions }}</span>
+      {%else%}
+      Unknown
+      {%endif%}
+    </tr>
     <tr>
       <th>Topic</th>
       <td>{{patch.topic}}</td>
diff --git a/pgcommitfest/commitfest/views.py b/pgcommitfest/commitfest/views.py
@@ -191,6 +191,8 @@ def commitfest(request, cfid):
             orderby_str = 'p.id'
         elif sortkey == 5:
             orderby_str = 'p.name, created'
+        elif sortkey == 6:
+            orderby_str = "branch.all_additions + branch.all_deletions NULLS LAST, created"
         else:
             orderby_str = 'p.id'
             sortkey = 0
@@ -229,7 +231,12 @@ def commitfest(request, cfid):
             count(*) total,
             string_agg(task.task_name, ', ') FILTER (WHERE task.status in ('ABORTED', 'ERRORED', 'FAILED')) as failed_task_names,
             branch.commit_id IS NULL as needs_rebase,
-            branch.apply_url
+            branch.apply_url,
+            branch.patch_count,
+            branch.first_additions,
+            branch.first_deletions,
+            branch.all_additions,
+            branch.all_deletions
         FROM commitfest_cfbotbranch branch
         LEFT JOIN commitfest_cfbottask task ON task.branch_id = branch.branch_id
         WHERE branch.patch_id=p.id
@@ -241,8 +248,9 @@ def commitfest(request, cfid):
 INNER JOIN commitfest_topic t ON t.id=p.topic_id
 LEFT JOIN auth_user committer ON committer.id=p.committer_id
 LEFT JOIN commitfest_targetversion v ON p.targetversion_id=v.id
+LEFT JOIN commitfest_cfbotbranch branch ON branch.patch_id=p.id
 WHERE poc.commitfest_id=%(cid)s {0}
-GROUP BY p.id, poc.id, committer.id, t.id, v.version
+GROUP BY p.id, poc.id, committer.id, t.id, v.version, branch.patch_id
 ORDER BY is_open DESC, {1}""".format(where_str, orderby_str), params)
     patches = [dict(zip([col[0] for col in curs.description], row)) for row in curs.fetchall()]
 
@@ -842,16 +850,26 @@ def cfbot_ingest(message):
     cursor.execute("""INSERT INTO commitfest_cfbotbranch (patch_id, branch_id,
                                                 branch_name, commit_id,
                                                 apply_url, status,
-                                                created, modified)
-                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                                                created, modified,
+                                                version, patch_count,
+                                                first_additions, first_deletions,
+                                                all_additions, all_deletions
+                                                )
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
                     ON CONFLICT (patch_id) DO UPDATE
                         SET status = EXCLUDED.status,
                             modified = EXCLUDED.modified,
                             branch_id = EXCLUDED.branch_id,
                             branch_name = EXCLUDED.branch_name,
                             commit_id = EXCLUDED.commit_id,
                             apply_url = EXCLUDED.apply_url,
-                            created = EXCLUDED.created
+                            created = EXCLUDED.created,
+                            version = EXCLUDED.version,
+                            patch_count = EXCLUDED.patch_count,
+                            first_additions = EXCLUDED.first_additions,
+                            first_deletions = EXCLUDED.first_deletions,
+                            all_additions = EXCLUDED.all_additions,
+                            all_deletions = EXCLUDED.all_deletions
                         WHERE commitfest_cfbotbranch.created < EXCLUDED.created
                             OR (commitfest_cfbotbranch.branch_id = EXCLUDED.branch_id
                                 AND commitfest_cfbotbranch.modified < EXCLUDED.modified)
@@ -864,7 +882,13 @@ def cfbot_ingest(message):
                        branch_status["apply_url"],
                        branch_status["status"],
                        branch_status["created"],
-                       branch_status["modified"])
+                       branch_status["modified"],
+                       branch_status["version"],
+                       branch_status["patch_count"],
+                       branch_status["first_additions"],
+                       branch_status["first_deletions"],
+                       branch_status["all_additions"],
+                       branch_status["all_deletions"])
                    )
 
     # Now we check what we have in our database. If that contains a different
