Implement relevant functionality for bot, pagination!

felixmulder · felixmulder · commit 13788d1f9366 · 2017-02-09T22:01:57.000+01:00
diff --git a/bot/src/dotty/tools/bot/BotServer.scala b/bot/src/dotty/tools/bot/BotServer.scala
@@ -7,12 +7,16 @@ import scalaz.concurrent.Task
 
 object Main extends ServerApp with PullRequestService {
 
+  val user = sys.env("USER")
+  val token = sys.env("TOKEN")
+
   /** Services mounted to the server */
   final val services = prService
 
-  override def server(args: List[String]): Task[Server] =
+  override def server(args: List[String]): Task[Server] = {
     BlazeBuilder
       .bindHttp(8080, "localhost")
       .mountService(services, "/api")
       .start
+  }
 }
diff --git a/bot/src/dotty/tools/bot/PullRequestService.scala b/bot/src/dotty/tools/bot/PullRequestService.scala
@@ -1,115 +1,186 @@
 package dotty.tools.bot
 
-import org.http4s._
+import org.http4s.{ Status => _, _ }
 import org.http4s.client.blaze._
 import org.http4s.client.Client
+import org.http4s.headers.Authorization
 
 import scalaz.concurrent.Task
+import scala.util.control.NonFatal
 
 import io.circe._
 import io.circe.generic.auto._
 import io.circe.syntax._
 import org.http4s.circe._
 import org.http4s.dsl._
+import org.http4s.util._
 
-import github4s.Github
-import github4s.jvm.Implicits._
-import github4s.free.domain.{ Commit, Issue }
+import model.Github._
 
 trait PullRequestService {
 
+  /** Username for authorized admin */
+  def user: String
+
+  /** OAuth token needed for user to create statuses */
+  def token: String
+
+  /** Pull Request HTTP service */
   val prService = HttpService {
     case request @ POST -> Root =>
       request.as(jsonOf[Issue]).flatMap(checkPullRequest)
   }
 
-  private case class CLASignature(
+  private[this] lazy val authHeader = {
+    val creds = BasicCredentials(user, token)
+    new Authorization(creds)
+  }
+
+  private final case class CLASignature(
     user: String,
     signed: Boolean,
     version: String,
     currentVersion: String
   )
 
-  private case class Status(
-    state: String,
-    target_url: String,
-    description: String,
-    context: String = "continuous-integration/CLA"
-  )
-
   def claUrl(userName: String): String =
    s"https://www.lightbend.com/contribute/cla/scala/check/$userName"
 
   def commitsUrl(prNumber: Int): String =
-    s"https://api.github.com/repos/lampepfl/dotty/pulls/$prNumber/commits"
+    s"https://api.github.com/repos/lampepfl/dotty/pulls/$prNumber/commits?per_page=100"
+
+  def statusUrl(sha: String): String =
+    s"https://api.github.com/repos/lampepfl/dotty/statuses/$sha"
 
   def toUri(url: String): Task[Uri] =
     Uri.fromString(url).fold(Task.fail, Task.now)
 
   def getRequest(endpoint: Uri): Task[Request] = Task.now {
-    Request(uri = endpoint, method = Method.GET)
+    Request(uri = endpoint, method = Method.GET).putHeaders(authHeader)
   }
 
   def postRequest(endpoint: Uri): Task[Request] = Task.now {
-    Request(uri = endpoint, method = Method.POST)
+    Request(uri = endpoint, method = Method.POST).putHeaders(authHeader)
   }
 
   def shutdownClient(client: Client): Task[Unit] = Task.now {
     client.shutdownNow()
   }
 
   def users(xs: List[Commit]): Task[Set[String]] = Task.now {
-    xs.map(_.login).flatten.toSet
+    xs
+      .flatMap { commit =>
+        List(commit.author.login, commit.committer.login).flatten
+      }
+      .toSet
   }
 
   sealed trait CommitStatus {
     def commit: Commit
     def isValid: Boolean
   }
-  final case class Valid(commit: Commit) extends CommitStatus { def isValid = true }
-  final case class Invalid(commit: Commit) extends CommitStatus { def isValid = false }
+  final case class Valid(user: String, commit: Commit) extends CommitStatus { def isValid = true }
+  final case class Invalid(user: String, commit: Commit) extends CommitStatus { def isValid = false }
+  final case class CLAServiceDown(user: String, commit: Commit) extends CommitStatus { def isValid = false }
+  final case class MissingUser(commit: Commit) extends CommitStatus { def isValid = false }
 
   /** Partitions invalid and valid commits */
   def checkCLA(xs: List[Commit], httpClient: Client): Task[List[CommitStatus]] = {
-    def checkUser(commit: Commit): Task[CommitStatus] = for {
-      endpoint <- toUri(claUrl(commit.login.get))
-      claReq   <- getRequest(endpoint)
-      claRes   <- httpClient.expect(claReq)(jsonOf[CLASignature])
-      res = if (claRes.signed) Valid(commit) else Invalid(commit)
-    } yield res
-
-    Task.gatherUnordered(xs.filter(_.login.isDefined).map(checkUser))
+    def checkUser(user: String, commit: Commit): Task[CommitStatus] = {
+      val claStatus = for {
+        endpoint <- toUri(claUrl(user))
+        claReq   <- getRequest(endpoint)
+        claRes   <- httpClient.expect(claReq)(jsonOf[CLASignature])
+        res = if (claRes.signed) Valid(user, commit) else Invalid(user, commit)
+      } yield res
+
+      claStatus.handleWith {
+        case NonFatal(e) =>
+          println(e)
+          Task.now(CLAServiceDown(user, commit))
+      }
+    }
+
+    def checkCommit(commit: Commit, author: Author): Task[CommitStatus] =
+      author.login.map(checkUser(_, commit)).getOrElse(Task.now(MissingUser(commit)))
+
+    Task.gatherUnordered {
+      xs.flatMap {
+        case c @ Commit(_, author, commiter, _) =>
+          if (author == commiter) List(checkCommit(c, author))
+          else List(
+            checkCommit(c, author),
+            checkCommit(c, commiter)
+          )
+      }
+    }
   }
 
-  def sendStatuses(xs: List[CommitStatus], httpClient: Client): Task[Unit] = {
-    def setStatus(cm: CommitStatus): Task[Unit] = for {
-      endpoint <- toUri(cm.commit.url.replaceAll("git\\/commits", "statuses"))
-
-      target = claUrl(cm.commit.login.getOrElse("<invalid-user>"))
-      state = if (cm.isValid) "success" else "failure"
-      desc =
+  def sendStatuses(xs: List[CommitStatus], httpClient: Client): Task[List[StatusResponse]] = {
+    def setStatus(cm: CommitStatus): Task[StatusResponse] = {
+      val desc =
         if (cm.isValid) "User signed CLA"
         else "User needs to sign cla: https://www.lightbend.com/contribute/cla/scala"
 
-      statusReq <- postRequest(endpoint).map(_.withBody(Status(state, target, desc).asJson))
-      statusRes <- httpClient.expect(statusReq)(jsonOf[String])
-      print     <- Task.now(println(statusRes))
-    } yield print
+      val stat = cm match {
+        case Valid(user, commit) =>
+          Status("success", claUrl(user), desc)
+        case Invalid(user, commit) =>
+          Status("failure", claUrl(user), desc)
+        case MissingUser(commit) =>
+          Status("failure", "", "Missing valid github user for this PR")
+        case CLAServiceDown(user, commit) =>
+          Status("pending", claUrl(user), "CLA Service is down")
+      }
+
+      for {
+        endpoint <- toUri(statusUrl(cm.commit.sha))
+        req      <- postRequest(endpoint).map(_.withBody(stat.asJson))
+        res      <- httpClient.expect(req)(jsonOf[StatusResponse])
+      } yield res
+    }
+
+    Task.gatherUnordered(xs.map(setStatus))
+  }
+
+  private[this] val ExtractLink = """<([^>]+)>; rel="([^"]+)"""".r
+  def findNext(header: Option[Header]): Option[String] = header.flatMap { header =>
+    val value = header.value
+
+    value
+      .split(',')
+      .collect {
+        case ExtractLink(url, kind) if kind == "next" =>
+          url
+      }
+      .headOption
+  }
+
+  def getCommits(issueNbr: Int, httpClient: Client): Task[List[Commit]] = {
+    def makeRequest(url: String): Task[List[Commit]] =
+      for {
+        endpoint <- toUri(url)
+        req <- getRequest(endpoint)
+        res <- httpClient.fetch(req){ res =>
+          val link = CaseInsensitiveString("Link")
+          val next = findNext(res.headers.get(link)).map(makeRequest).getOrElse(Task.now(Nil))
+
+          res.as[List[Commit]](jsonOf[List[Commit]]).flatMap(commits => next.map(commits ++ _))
+        }
+      } yield res
 
-    Task.gatherUnordered(xs.map(setStatus)).map(_ => ())
+    makeRequest(commitsUrl(issueNbr))
   }
 
   def checkPullRequest(issue: Issue): Task[Response] = {
     val httpClient = PooledHttp1Client()
 
     for {
       // First get all the commits from the PR
-      endpoint   <- toUri(commitsUrl(issue.number))
-      commitsReq <- getRequest(endpoint)
-      commitsRes <- httpClient.expect(commitsReq)(jsonOf[List[Commit]])
+      commits <- getCommits(issue.number, httpClient)
 
       // Then get check the CLA of each commit
-      statuses   <- checkCLA(commitsRes, httpClient)
+      statuses   <- checkCLA(commits, httpClient)
 
       // Send statuses to Github and exit
       _          <- sendStatuses(statuses, httpClient)
diff --git a/bot/src/dotty/tools/bot/model/Github.scala b/bot/src/dotty/tools/bot/model/Github.scala
@@ -12,4 +12,32 @@ object Github {
     number: Int,
     pull_request: Option[PullRequest]
   )
+
+  case class CommitInfo(
+    message: String
+  )
+
+  case class Commit(
+    sha: String,
+    author: Author,
+    committer: Author,
+    commit: CommitInfo
+  )
+
+  case class Author(
+    login: Option[String]
+  )
+
+  case class Status(
+    state: String,
+    target_url: String,
+    description: String,
+    context: String = "CLA"
+  )
+
+  case class StatusResponse(
+    url: String,
+    id: Long,
+    state: String
+  )
 }
diff --git a/bot/test/PRServiceTests.scala b/bot/test/PRServiceTests.scala
@@ -31,4 +31,62 @@ class PRServiceTests extends PullRequestService {
 
     assert(issue.pull_request.isDefined, "missing pull request")
   }
+
+  @Test def canGetAllCommitsFromPR = {
+    val httpClient = PooledHttp1Client()
+    val issueNbr = 1941 // has 2 commits: https://github.com/lampepfl/dotty/pull/1941/commits
+
+    val List(c1, c2) = getCommits(issueNbr, httpClient).run
+
+    assertEquals(
+      "Represent untyped operators as Ident instead of Name",
+      c1.commit.message.takeWhile(_ != '\n')
+    )
+
+    assertEquals(
+      "Better positions for infix term operations.",
+      c2.commit.message.takeWhile(_ != '\n')
+    )
+  }
+
+  @Test def canGetMoreThan100Commits = {
+    val httpClient = PooledHttp1Client()
+    val issueNbr = 1840 // has >100 commits: https://github.com/lampepfl/dotty/pull/1840/commits
+
+    val numberOfCommits = getCommits(issueNbr, httpClient).run.length
+
+    assert(
+      numberOfCommits > 100,
+      s"PR 1840, should have a number of commits greater than 100, but was: $numberOfCommits"
+    )
+  }
+
+  @Test def canCheckCLA = {
+    val httpClient = PooledHttp1Client()
+    val validUserCommit = Commit("sha-here", Author(Some("felixmulder")), Author(Some("felixmulder")), CommitInfo(""))
+    val statuses: List[CommitStatus] = checkCLA(validUserCommit :: Nil, httpClient).run
+
+    assert(statuses.length == 1, s"wrong number of valid statuses: got ${statuses.length}, expected 1")
+    httpClient.shutdownNow()
+  }
+
+  @Test def canSetStatus = {
+    val httpClient = PooledHttp1Client()
+    val sha = "fa64b4b613fe5e78a5b4185b4aeda89e2f1446ff"
+    val status = Invalid("smarter", Commit(sha, Author(Some("smarter")), Author(Some("smarter")), CommitInfo("")))
+
+    val statuses: List[StatusResponse] = sendStatuses(status :: Nil, httpClient).run
+
+    assert(
+      statuses.length == 1,
+      s"assumed one status response would be returned, got: ${statuses.length}"
+    )
+
+    assert(
+      statuses.head.state == "failure",
+      s"status set had wrong state, expected 'failure', got: ${statuses.head.state}"
+    )
+
+    httpClient.shutdownNow()
+  }
 }
