#158 - BE - Submit with git

devops/lms.yml

@@ -97,6 +97,7 @@ services:
     volumes:
       - ../lms:/app_dir/lms
       - ../../notebooks-tests:/app_dir/notebooks-tests
+      - repositories-data-volume:/repositories
     environment:
       - DB_NAME=${DB_NAME:-db}
       - DB_USER=${DB_USERNAME:-postgres}
@@ -109,6 +110,7 @@ services:
       - CELERY_CHECKS_PUBLIC_VHOST=lmstests-public
       - CELERY_RABBITMQ_HOST=rabbitmq
       - CELERY_RABBITMQ_PORT=5672
+      - REPOSITORY_FOLDER=/repositories
     links:
       - db
       - rabbitmq
@@ -126,6 +128,7 @@ volumes:
   docker-engine-lib:
   db-data-volume:
   rabbit-data-volume:
+  repositories-data-volume:
 
 
 networks:

lms/lmsweb/__init__.py

@@ -1,8 +1,10 @@
 import pathlib
 import shutil
+import typing
 
 from flask import Flask
 from flask_babel import Babel  # type: ignore
+from flask_httpauth import HTTPBasicAuth
 from flask_limiter import Limiter  # type: ignore
 from flask_limiter.util import get_remote_address  # type: ignore
 from flask_mail import Mail  # type: ignore
@@ -28,6 +30,8 @@
     static_folder=str(static_dir),
 )
 
+http_basic_auth = HTTPBasicAuth()
+
 limiter = Limiter(webapp, key_func=get_remote_address)
 
 
@@ -52,3 +56,18 @@
 
 # gunicorn search for application
 application = webapp
+
+
+@http_basic_auth.get_password
+def get_password(username: str) -> typing.Optional[str]:
+    user = models.User.get_or_none(models.User.username == username)
+    return user.password if user else None
+
+
+@http_basic_auth.verify_password
+def verify_password(username: str, client_password: str):
+    username_username = models.User.username == username
+    login_user = models.User.get_or_none(username_username)
+    if login_user is None or not login_user.is_password_valid(client_password):
+        return False
+    return login_user

lms/lmsweb/config.py.example

@@ -67,3 +67,5 @@ LIMITS_PER_HOUR = 50
 
 # Change password settings
 MAX_INVALID_PASSWORD_TRIES = 5
+
+REPOSITORY_FOLDER = os.getenv("REPOSITORY_FOLDER", os.path.abspath(os.path.join(os.curdir, "repositories")))

lms/lmsweb/git_service.py

@@ -0,0 +1,229 @@
+import os
+import shutil
+import subprocess  # noqa: S404
+import tempfile
+import typing
+import pathlib
+
+import flask
+
+from lms.lmsdb import models
+from lms.models import upload
+from lms.utils import hashing
+from lms.utils.log import log
+
+
+class _GitOperation(typing.NamedTuple):
+    response_content_type: str
+    service_command: typing.List[str]
+    supported: bool
+    format_response: typing.Optional[typing.Callable]
+    contain_new_commits: bool
+
+
+class GitService:
+    _GIT_PROCESS_TIMEOUT = 20
+    _GIT_VALID_EXIT_CODE = 0
+    _STATELESS_RPC = '--stateless-rpc'
+    _ADVERTISE_REFS = '--advertise-refs'
+    _UPLOAD_COMMAND = 'git-upload-pack'
+    _RECEIVE_COMMAND = 'git-receive-pack'
+    _REFS_COMMAND = '/info/refs'
+
+    def __init__(
+            self,
+            user: models.User,
+            exercise_number: int,
+            course_id: int,
+            request: flask.Request,
+            base_repository_folder: str,
+    ):
+        self._base_repository_folder = base_repository_folder
+        self._user = user
+        self._exercise_number = exercise_number
+        self._course_id = course_id
+        self._request = request
+
+    @property
+    def project_name(self) -> str:
+        return f'{self._course_id}-{self._exercise_number}-{self._user.id}'
+
+    @property
+    def repository_folder(self) -> pathlib.Path:
+        return pathlib.Path(self._base_repository_folder) / self.project_name
+
+    def handle_operation(self) -> flask.Response:
+        git_operation = self._extract_git_operation()
+        repository_folder = self.repository_folder / 'config'
+
+        new_repository = not repository_folder.exists()
+        if new_repository:
+            self._initialize_bare_repository()
+
+        if not git_operation.supported:
+            raise OSError
+
+        data_out = self._execute_git_operation(git_operation)
+
+        if git_operation.format_response:
+            data_out = git_operation.format_response(data_out)
+
+        if git_operation.contain_new_commits:
+            files = self._load_files_from_repository()
+            solution_hash = hashing.by_content(str(files))
+            upload.upload_solution(
+                course_id=self._course_id,
+                exercise_number=self._exercise_number,
+                files=files,
+                solution_hash=solution_hash,
+                user_id=self._user.id,
+            )
+
+        return self.build_response(data_out, git_operation)
+
+    def _execute_command(
+            self,
+            args: typing.List[str],
+            cwd: typing.Union[str, pathlib.Path],
+            proc_input: typing.Optional[bytes] = None,
+    ):
+        proc = subprocess.Popen(  # noqa: S603
+            args=args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            cwd=cwd,
+        )
+        data_out, _ = proc.communicate(proc_input, self._GIT_PROCESS_TIMEOUT)
+        operation_failed = proc.wait() != self._GIT_VALID_EXIT_CODE
+        if operation_failed:
+            log.error(
+                'Failed to execute command %s. stdout=%s\nstderr=%s',
+                args, proc.stdout.read(), proc.stderr.read(),
+            )
+            raise OSError
+        return data_out
+
+    def _execute_git_operation(self, git_operation: _GitOperation) -> bytes:
+        return self._execute_command(
+            args=git_operation.service_command,
+            cwd=self._base_repository_folder,
+            proc_input=self._request.data,
+        )
+
+    def _initialize_bare_repository(self) -> None:
+        os.makedirs(self.repository_folder, exist_ok=True)
+        self._execute_command(
+            args=['git', 'init', '--bare'],
+            cwd=self.repository_folder,
+        )
+
+    @staticmethod
+    def build_response(
+            data_out: bytes,
+            git_operation: _GitOperation,
+    ) -> flask.Response:
+        res = flask.make_response(data_out)
+        res.headers['Pragma'] = 'no-cache'
+        res.headers['Cache-Control'] = 'no-cache, max-age=0, must-revalidate'
+        res.headers['Content-Type'] = git_operation.response_content_type
+        return res
+
+    def _extract_git_operation(self) -> _GitOperation:
+        if self._request.path.endswith(self._UPLOAD_COMMAND):
+            return self._build_upload_operation()
+
+        elif self._request.path.endswith(self._RECEIVE_COMMAND):
+            return self._build_receive_operation()
+
+        elif self._request.path.endswith(self._REFS_COMMAND):
+            return self._build_refs_operation()
+
+        else:
+            log.error(
+                'Failed to find the git command for route %s',
+                self._request.path,
+            )
+            raise NotImplementedError
+
+    def _build_refs_operation(self) -> _GitOperation:
+        allowed_commands = [self._UPLOAD_COMMAND, self._RECEIVE_COMMAND]
+        service_name = self._request.args.get('service')
+        content_type = f'application/x-{service_name}-advertisement'
+        supported = service_name in allowed_commands
+
+        def format_response_callback(response_bytes: bytes) -> bytes:
+            packet = f'# service={service_name}\n'
+            length = len(packet) + 4
+            prefix = '{:04x}'.format(length & 0xFFFF)
+
+            data = (prefix + packet + '0000').encode()
+            data += response_bytes
+            return data
+
+        return _GitOperation(
+            response_content_type=content_type,
+            service_command=[
+                service_name,
+                self._STATELESS_RPC,
+                self._ADVERTISE_REFS,
+                self.project_name,
+            ],
+            supported=supported,
+            format_response=format_response_callback,
+            contain_new_commits=False,
+        )
+
+    def _build_receive_operation(self) -> _GitOperation:
+        return _GitOperation(
+            response_content_type='application/x-git-receive-pack-result',
+            service_command=[
+                self._RECEIVE_COMMAND,
+                self._STATELESS_RPC,
+                self.project_name,
+            ],
+            supported=True,
+            format_response=None,
+            contain_new_commits=True,
+        )
+
+    def _build_upload_operation(self) -> _GitOperation:
+        return _GitOperation(
+            response_content_type='application/x-git-upload-pack-result',
+            service_command=[
+                self._UPLOAD_COMMAND,
+                self._STATELESS_RPC,
+                self.project_name,
+            ],
+            supported=True,
+            format_response=None,
+            contain_new_commits=False,
+        )
+
+    def _load_files_from_repository(self) -> typing.List[upload.File]:
+        """
+        Since the remote server is a git bare repository
+        we need to 'clone' the bare repository to resolve the files.
+        We are not changing the remote at any end - that is why we
+        don't care about git files here.
+        """
+        with tempfile.TemporaryDirectory() as tempdir:
+            self._execute_command(
+                args=['git', 'clone', self.repository_folder, '.'],
+                cwd=tempdir,
+            )
+            to_return = []
+            # remove git internal files
+            shutil.rmtree(pathlib.Path(tempdir) / '.git')
+            for root, _, files in os.walk(tempdir):
+                for file in files:
+                    upload_file = self._load_file(file, root, tempdir)
+                    to_return.append(upload_file)
+            return to_return
+
+    @staticmethod
+    def _load_file(file_name: str, root: str, tempdir: str) -> upload.File:
+        file_path = str(pathlib.Path(root).relative_to(tempdir) / file_name)
+        with open(pathlib.Path(root) / file_name) as f:
+            upload_file = upload.File(path=file_path, code=f.read())
+        return upload_file

lms/lmsweb/routes.py

@@ -2,3 +2,4 @@
 STATUS = '/status'
 DOWNLOADS = '/download'
 SHARED = '/shared'
+GIT = '/git/<int:course_id>/<int:exercise_number>.git'

lms/lmsweb/views.py

@@ -2,7 +2,7 @@
 
 import arrow  # type: ignore
 from flask import (
-    jsonify, make_response, render_template, request,
+    Response, jsonify, make_response, render_template, request,
     send_from_directory, session, url_for,
 )
 from flask_babel import gettext as _  # type: ignore
@@ -18,17 +18,18 @@
     ALL_MODELS, Comment, Course, Note, Role, RoleOptions, SharedSolution,
     Solution, SolutionFile, User, UserCourse, database,
 )
-from lms.lmsweb import babel, limiter, routes, webapp
+from lms.lmsweb import babel, http_basic_auth, limiter, routes, webapp
 from lms.lmsweb.admin import (
     AdminModelView, SPECIAL_MAPPING, admin, managers_only,
 )
 from lms.lmsweb.config import (
     CONFIRMATION_TIME, LANGUAGES, LIMITS_PER_HOUR,
-    LIMITS_PER_MINUTE, LOCALE, MAX_UPLOAD_SIZE,
+    LIMITS_PER_MINUTE, LOCALE, MAX_UPLOAD_SIZE, REPOSITORY_FOLDER,
 )
 from lms.lmsweb.forms.change_password import ChangePasswordForm
 from lms.lmsweb.forms.register import RegisterForm
 from lms.lmsweb.forms.reset_password import RecoverPassForm, ResetPassForm
+from lms.lmsweb.git_service import GitService
 from lms.lmsweb.manifest import MANIFEST
 from lms.lmsweb.redirections import (
     PERMISSIVE_CORS, get_next_url, login_manager,
@@ -570,6 +571,21 @@ def download(download_id: str):
     return response
 
 
+@webapp.route(f'{routes.GIT}/info/refs')
+@webapp.route(f'{routes.GIT}/git-receive-pack', methods=['POST'])
+@webapp.route(f'{routes.GIT}/git-upload-pack', methods=['POST'])
+@http_basic_auth.login_required
+def git_handler(course_id: int, exercise_number: int) -> Response:
+    git_service = GitService(
+        user=http_basic_auth.current_user(),
+        exercise_number=exercise_number,
+        course_id=course_id,
+        request=request,
+        base_repository_folder=REPOSITORY_FOLDER,
+    )
+    return git_service.handle_operation()
+
+
 @webapp.route(f'{routes.SOLUTIONS}/<int:solution_id>')
 @webapp.route(f'{routes.SOLUTIONS}/<int:solution_id>/<int:file_id>')
 @login_required

lms/models/upload.py

@@ -72,10 +72,13 @@ def new(
     errors: List[Union[UploadError, AlreadyExists]] = []
     for exercise_number, files, solution_hash in Extractor(file):
         try:
-            solution = _upload_to_db(
-                exercise_number, course_id, user_id, files, solution_hash,
+            upload_solution(
+                course_id=course_id,
+                exercise_number=exercise_number,
+                files=files,
+                solution_hash=solution_hash,
+                user_id=user_id,
             )
-            _run_auto_checks(solution)
         except (UploadError, AlreadyExists) as e:
             log.debug(e)
             errors.append(e)
@@ -87,3 +90,20 @@ def new(
         raise UploadError(errors)
 
     return matches, misses
+
+
+def upload_solution(
+        course_id: int,
+        exercise_number: int,
+        files: List[File],
+        solution_hash: str,
+        user_id: int,
+):
+    solution = _upload_to_db(
+        exercise_number=exercise_number,
+        course_id=course_id,
+        user_id=user_id,
+        files=files,
+        solution_hash=solution_hash,
+    )
+    _run_auto_checks(solution)