Big refactor to make things cleaner + enable retries properly on infra flakes for local upload

Author: Ubuntu

nucleus/dataset_item_uploader.py

@@ -150,12 +150,22 @@ def get_form_data_and_file_pointers_fn(
         """Constructs a function that will generate form data on each retry."""
 
         def fn():
-            payload = construct_append_payload(items, update)
+
+            # For some reason, our backend only accepts this reformatting of items when
+            # doing local upload.
+            # TODO: make it just accept the same exact format as a normal append request
+            # i.e. the output of construct_append_payload(items, update)
+            json_data = []
+            for item in items:
+                item_payload = item.to_payload()
+                item_payload[UPDATE_KEY] = update
+                json_data.append(item_payload)
+
             form_data = [
                 FileFormField(
                     name=ITEMS_KEY,
                     filename=None,
-                    value=json.dumps(payload, allow_nan=False),
+                    value=json.dumps(json_data, allow_nan=False),
                     content_type="application/json",
                 )
             ]

nucleus/segmentation_uploader.py

@@ -112,3 +112,8 @@
 #                     (MASK_TYPE, (filename, mask_fp, "image/png"))
 #                 )
 #             return request_payload, file_pointers
+
+
+# {"items": [{"metadata": {"test": 0}, "reference_id": "test_img.jpg", "image_url": "tests/test_img.jpg", "upload_to_scale": true}]
+
+# [{"metadata": {"test": 0}, "reference_id": "test_img.jpg", "image_url": "tests/test_img.jpg", "upload_to_scale": true, "update": false}]