Merge pull request #66 from HackSoftware/file-uploads

File uploads - locally and on s3

Author: RadoRado

.env.example

@@ -1,2 +1,10 @@
 PYTHONBREAKPOINT=ipdb.set_trace
 SENTRY_DSN=""
+
+FILE_UPLOAD_STRATEGY="direct" # pass-thru
+FILE_UPLOAD_STORAGE="local" # s3
+
+AWS_S3_ACCESS_KEY_ID=""
+AWS_S3_SECRET_ACCESS_KEY=""
+AWS_STORAGE_BUCKET_NAME="django-styleguide-example"
+AWS_S3_REGION_NAME="eu-central-1"

.github/workflows/django.yml

@@ -8,7 +8,7 @@ jobs:
     - name: Build docker
       run: docker-compose build
     - name: Type check
-      run: docker-compose run django mypy styleguide_example/
+      run: docker-compose run django mypy --config mypy.ini styleguide_example/
     - name: Run migrations
       run: docker-compose run django python manage.py migrate
     - name: Run tests
@@ -38,7 +38,9 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements/local.txt
     - name: Type check
-      run: mypy styleguide_example/
+      run: |
+        mypy --version
+        mypy --config mypy.ini styleguide_example/
     - name: Run migrations
       run: python manage.py migrate
     - name: Run tests

.gitignore

@@ -127,3 +127,7 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+
+# media files
+/media

config/django/base.py

@@ -12,10 +12,7 @@
 
 import os
 
-from config.env import env, environ
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = environ.Path(__file__) - 3
+from config.env import env, BASE_DIR
 
 env.read_env(os.path.join(BASE_DIR, ".env"))
 
@@ -41,6 +38,8 @@
     'styleguide_example.users.apps.UsersConfig',
     'styleguide_example.errors.apps.ErrorsConfig',
     'styleguide_example.testing_examples.apps.TestingExamplesConfig',
+    'styleguide_example.integrations.apps.IntegrationsConfig',
+    'styleguide_example.files.apps.FilesConfig',
 ]
 
 THIRD_PARTY_APPS = [
@@ -171,8 +170,12 @@
     'DEFAULT_AUTHENTICATION_CLASSES': []
 }
 
+APP_DOMAIN = env("APP_DOMAIN", default="http://localhost:8000")
+
 from config.settings.cors import *  # noqa
 from config.settings.jwt import *  # noqa
 from config.settings.sessions import *  # noqa
 from config.settings.celery import *  # noqa
 from config.settings.sentry import *  # noqa
+
+from config.settings.files_and_storages import *  # noqa

config/env.py

@@ -1,3 +1,15 @@
+from django.core.exceptions import ImproperlyConfigured
+
 import environ
 
 env = environ.Env()
+
+BASE_DIR = environ.Path(__file__) - 2
+
+
+def env_to_enum(enum_cls, value):
+    for x in enum_cls:
+        if x.value == value:
+            return x
+
+    raise ImproperlyConfigured(f"Env value {repr(value)} could not be found in {repr(enum_cls)}")

config/settings/files_and_storages.py

@@ -0,0 +1,36 @@
+import os
+
+from config.env import BASE_DIR, env, env_to_enum
+
+from styleguide_example.files.enums import FileUploadStrategy, FileUploadStorage
+
+
+FILE_UPLOAD_STRATEGY = env_to_enum(
+    FileUploadStrategy,
+    env("FILE_UPLOAD_STRATEGY", default="direct")
+)
+FILE_UPLOAD_STORAGE = env_to_enum(
+    FileUploadStorage,
+    env("FILE_UPLOAD_STORAGE", default="local")
+)
+
+if FILE_UPLOAD_STORAGE == FileUploadStorage.LOCAL:
+    MEDIA_ROOT_NAME = "media"
+    MEDIA_ROOT = os.path.join(BASE_DIR, MEDIA_ROOT_NAME)
+    MEDIA_URL = f"/{MEDIA_ROOT_NAME}/"
+
+if FILE_UPLOAD_STORAGE == FileUploadStorage.S3:
+    # Using django-storages
+    # https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html
+    DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
+
+    AWS_S3_ACCESS_KEY_ID = env("AWS_S3_ACCESS_KEY_ID")
+    AWS_S3_SECRET_ACCESS_KEY = env("AWS_S3_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME")
+    AWS_S3_SIGNATURE_VERSION = env("AWS_S3_SIGNATURE_VERSION", default="s3v4")
+
+    # https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
+    AWS_DEFAULT_ACL = env("AWS_DEFAULT_ACL", default="private")
+
+    AWS_PRESIGNED_EXPIRY = env.int("AWS_PRESIGNED_EXPIRY", default=10)  # seconds

config/urls.py

@@ -14,9 +14,11 @@
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
+from django.conf import settings
 from django.urls import path, include
+from django.conf.urls.static import static
 
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('api/', include(('styleguide_example.api.urls', 'api'))),
-]
+] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)

mypy.ini

@@ -0,0 +1,35 @@
+[mypy]
+plugins =
+    mypy_django_plugin.main,
+    mypy_drf_plugin.main
+
+[mypy.plugins.django-stubs]
+django_settings_module = "config.django.base"
+
+[mypy-config.*]
+# Ignore everything related to Django config
+ignore_errors = true
+
+[mypy-styleguide_example.*.migrations.*]
+# Ignore Django migrations
+ignore_errors = true
+
+[mypy-celery.*]
+# Remove this when celery stubs are present
+ignore_missing_imports = True
+
+[mypy-django_celery_beat.*]
+# Remove this when django_celery_beat stubs are present
+ignore_missing_imports = True
+
+[mypy-django_filters.*]
+# Remove this when django_filters stubs are present
+ignore_missing_imports = True
+
+[mypy-factory.*]
+# Remove this when factory stubs are present
+ignore_missing_imports = True
+
+[mypy-rest_framework_jwt.*]
+# Remove this when rest_framework_jwt stubs are present
+ignore_missing_imports = True

requirements/base.txt

@@ -10,7 +10,11 @@ django-celery-beat==2.2.1
 whitenoise==6.0.0
 
 django-filter==21.1
-django-cors-headers==3.11.0
 django-extensions==3.1.5
+django-cors-headers==3.10.0
+django-storages==1.12.3
 
 drf-jwt==1.19.2
+
+boto3==1.20.20
+attrs==21.4.0

requirements/local.txt

@@ -14,3 +14,4 @@ ipython==8.2.0
 mypy==0.942
 django-stubs==1.9.0
 djangorestframework-stubs==1.4.0
+boto3-stubs==1.21.32

setup.cfg

@@ -4,39 +4,3 @@ exclude =
     .git,
     __pycache__,
     */migrations/*
-
-[mypy]
-plugins =
-    mypy_django_plugin.main,
-    mypy_drf_plugin.main
-
-[mypy.plugins.django-stubs]
-django_settings_module = "config.django.base"
-
-[mypy-config.*]
-# Ignore everything related to Django config
-ignore_errors = true
-
-[mypy-styleguide_example.*.migrations.*]
-# Ignore Django migrations
-ignore_errors = true
-
-[mypy-celery.*]
-# Remove this when celery stubs are present
-ignore_missing_imports = True
-
-[mypy-django_celery_beat.*]
-# Remove this when django_celery_beat stubs are present
-ignore_missing_imports = True
-
-[mypy-django_filters.*]
-# Remove this when django_filters stubs are present
-ignore_missing_imports = True
-
-[mypy-factory.*]
-# Remove this when factory stubs are present
-ignore_missing_imports = True
-
-[mypy-rest_framework_jwt.*]
-# Remove this when rest_framework_jwt stubs are present
-ignore_missing_imports = True

styleguide_example/api/urls.py

@@ -6,4 +6,5 @@
     ),
     path('users/', include(('styleguide_example.users.urls', 'users'))),
     path('errors/', include(('styleguide_example.errors.urls', 'errors'))),
+    path('files/', include(('styleguide_example.files.urls', 'files'))),
 ]

styleguide_example/common/utils.py

@@ -1,7 +1,9 @@
-from rest_framework import serializers
-
+from django.conf import settings
 from django.shortcuts import get_object_or_404
 from django.http import Http404
+from django.core.exceptions import ImproperlyConfigured
+
+from rest_framework import serializers
 
 
 def make_mock_object(**kwargs):
@@ -30,3 +32,28 @@ def inline_serializer(*, fields, data=None, **kwargs):
         return serializer_class(data=data, **kwargs)
 
     return serializer_class(**kwargs)
+
+
+def assert_settings(required_settings, error_message_prefix=""):
+    """
+    Checks if each item from `required_settings` is present in Django settings
+    """
+    not_present = []
+    values = {}
+
+    for required_setting in required_settings:
+        if not hasattr(settings, required_setting):
+            not_present.append(required_setting)
+            continue
+
+        values[required_setting] = getattr(settings, required_setting)
+
+    if not_present:
+        if not error_message_prefix:
+            error_message_prefix = "Required settings not found."
+
+        stringified_not_present = ", ".join(not_present)
+
+        raise ImproperlyConfigured(f"{error_message_prefix} Could not find: {stringified_not_present}")
+
+    return values

styleguide_example/files/__init__.py

@@ -0,0 +1,77 @@
+from django import forms
+
+from django.contrib import admin, messages
+from django.core.exceptions import ValidationError
+
+from styleguide_example.files.models import File
+from styleguide_example.files.services import (
+    FileDirectUploadService
+)
+
+
+class FileForm(forms.ModelForm):
+    class Meta:
+        model = File
+        fields = ["file", "uploaded_by"]
+
+
+@admin.register(File)
+class FileAdmin(admin.ModelAdmin):
+    list_display = [
+        "id",
+        "original_file_name",
+        "file_name",
+        "file_type",
+        "url",
+        "uploaded_by",
+        "created_at",
+        "upload_finished_at",
+        "is_valid",
+    ]
+    list_select_related = ["uploaded_by"]
+
+    ordering = ["-created_at"]
+
+    def get_form(self, request, obj=None, **kwargs):
+        """
+        That's a bit of a hack
+        Dynamically change self.form, before delegating to the actual ModelAdmin.get_form
+        Proper kwargs are form, fields, exclude, formfield_callback
+        """
+        if obj is None:
+            self.form = FileForm
+
+        return super().get_form(request, obj, **kwargs)
+
+    def get_readonly_fields(self, request, obj=None):
+        """
+        We want to show those fields only when we have an existing object.
+        """
+
+        if obj is not None:
+            return [
+                "original_file_name",
+                "file_name",
+                "file_type",
+                "created_at",
+                "updated_at",
+                "upload_finished_at"
+            ]
+
+        return []
+
+    def save_model(self, request, obj, form, change):
+        try:
+            cleaned_data = form.cleaned_data
+
+            service = FileDirectUploadService(
+                file_obj=cleaned_data["file"],
+                user=cleaned_data["uploaded_by"]
+            )
+
+            if change:
+                service.update(file=obj)
+            else:
+                service.create()
+        except ValidationError as exc:
+            self.message_user(request, str(exc), messages.ERROR)