Merge pull request #7858 from Turbo87/sync-admins

Implement admin account syncing

Author: Turbo87

Cargo.lock

@@ -90,6 +90,7 @@ ipnetwork = "=0.20.0"
 tikv-jemallocator = { version = "=0.5.4", features = ['unprefixed_malloc_on_supported_platforms', 'profiling'] }
 lettre = { version = "=0.11.3", default-features = false, features = ["file-transport", "smtp-transport", "native-tls", "hostname", "builder"] }
 minijinja = "=1.0.11"
+mockall = "=0.12.1"
 moka = { version = "=0.12.2", features = ["future"]  }
 oauth2 = { version = "=4.4.2", default-features = false, features = ["reqwest"] }
 object_store = { version = "=0.9.0", features = ["aws"] }
@@ -128,4 +129,5 @@ crates_io_test_db = { path = "crates_io_test_db" }
 claims = "=0.7.1"
 googletest = "=0.10.0"
 insta = { version = "=1.34.0", features = ["json", "redactions"] }
+regex = "=1.10.2"
 tokio = "=1.35.1"

Cargo.toml

@@ -3,6 +3,7 @@ use crate::schema::{background_jobs, crates};
 use crate::worker::jobs;
 use anyhow::Result;
 use crates_io_worker::BackgroundJob;
+use diesel::dsl::exists;
 use diesel::prelude::*;
 use secrecy::{ExposeSecret, SecretString};
 
@@ -30,6 +31,11 @@ pub enum Command {
         #[arg()]
         name: String,
     },
+    SyncAdmins {
+        /// Force a sync even if one is already in progress
+        #[arg(long)]
+        force: bool,
+    },
 }
 
 pub fn run(command: Command) -> Result<()> {
@@ -58,6 +64,28 @@ pub fn run(command: Command) -> Result<()> {
         } => {
             jobs::DumpDb::new(database_url.expose_secret(), target_name).enqueue(conn)?;
         }
+        Command::SyncAdmins { force } => {
+            if !force {
+                // By default, we don't want to enqueue a sync if one is already
+                // in progress. If a sync fails due to e.g. an expired pinned
+                // certificate we don't want to keep adding new jobs to the
+                // queue, since the existing job will be retried until it
+                // succeeds.
+
+                let query = background_jobs::table
+                    .filter(background_jobs::job_type.eq(jobs::SyncAdmins::JOB_NAME));
+
+                if diesel::select(exists(query)).get_result(conn)? {
+                    info!(
+                        "Did not enqueue {}, existing job already in progress",
+                        jobs::SyncAdmins::JOB_NAME
+                    );
+                    return Ok(());
+                }
+            }
+
+            jobs::SyncAdmins.enqueue(conn)?;
+        }
         Command::DailyDbMaintenance => {
             jobs::DailyDbMaintenance.enqueue(conn)?;
         }

src/admin/enqueue_job.rs

@@ -18,6 +18,7 @@ use crates_io::cloudfront::CloudFront;
 use crates_io::db::DieselPool;
 use crates_io::fastly::Fastly;
 use crates_io::storage::Storage;
+use crates_io::team_repo::TeamRepoImpl;
 use crates_io::worker::{Environment, RunnerExt};
 use crates_io::{config, Emails};
 use crates_io::{db, ssh};
@@ -76,7 +77,8 @@ fn main() -> anyhow::Result<()> {
         .expect("Couldn't build client");
 
     let emails = Emails::from_environment(&config);
-    let fastly = Fastly::from_environment(client);
+    let fastly = Fastly::from_environment(client.clone());
+    let team_repo = TeamRepoImpl::default();
 
     let connection_pool = r2d2::Pool::builder()
         .max_size(10)
@@ -90,6 +92,7 @@ fn main() -> anyhow::Result<()> {
         .storage(storage)
         .connection_pool(DieselPool::new_background_worker(connection_pool.clone()))
         .emails(emails)
+        .team_repo(Box::new(team_repo))
         .build()?;
 
     let environment = Arc::new(environment);

src/bin/background-worker.rs

@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFjCCAv6gAwIBAgIRAJErCErPDBinU/bWLiWnX1owDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMjAwOTA0MDAwMDAw
+WhcNMjUwOTE1MTYwMDAwWjAyMQswCQYDVQQGEwJVUzEWMBQGA1UEChMNTGV0J3Mg
+RW5jcnlwdDELMAkGA1UEAxMCUjMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQC7AhUozPaglNMPEuyNVZLD+ILxmaZ6QoinXSaqtSu5xUyxr45r+XXIo9cP
+R5QUVTVXjJ6oojkZ9YI8QqlObvU7wy7bjcCwXPNZOOftz2nwWgsbvsCUJCWH+jdx
+sxPnHKzhm+/b5DtFUkWWqcFTzjTIUu61ru2P3mBw4qVUq7ZtDpelQDRrK9O8Zutm
+NHz6a4uPVymZ+DAXXbpyb/uBxa3Shlg9F8fnCbvxK/eG3MHacV3URuPMrSXBiLxg
+Z3Vms/EY96Jc5lP/Ooi2R6X/ExjqmAl3P51T+c8B5fWmcBcUr2Ok/5mzk53cU6cG
+/kiFHaFpriV1uxPMUgP17VGhi9sVAgMBAAGjggEIMIIBBDAOBgNVHQ8BAf8EBAMC
+AYYwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYB
+Af8CAQAwHQYDVR0OBBYEFBQusxe3WFbLrlAJQOYfr52LFMLGMB8GA1UdIwQYMBaA
+FHm0WeZ7tuXkAXOACIjIGlj26ZtuMDIGCCsGAQUFBwEBBCYwJDAiBggrBgEFBQcw
+AoYWaHR0cDovL3gxLmkubGVuY3Iub3JnLzAnBgNVHR8EIDAeMBygGqAYhhZodHRw
+Oi8veDEuYy5sZW5jci5vcmcvMCIGA1UdIAQbMBkwCAYGZ4EMAQIBMA0GCysGAQQB
+gt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCFyk5HPqP3hUSFvNVneLKYY611TR6W
+PTNlclQtgaDqw+34IL9fzLdwALduO/ZelN7kIJ+m74uyA+eitRY8kc607TkC53wl
+ikfmZW4/RvTZ8M6UK+5UzhK8jCdLuMGYL6KvzXGRSgi3yLgjewQtCPkIVz6D2QQz
+CkcheAmCJ8MqyJu5zlzyZMjAvnnAT45tRAxekrsu94sQ4egdRCnbWSDtY7kh+BIm
+lJNXoB1lBMEKIq4QDUOXoRgffuDghje1WrG9ML+Hbisq/yFOGwXD9RiX8F6sw6W4
+avAuvDszue5L3sz85K+EC4Y/wFVDNvZo4TYXao6Z0f+lQKc0t8DQYzk1OXVu8rp2
+yJMC6alLbBfODALZvYH7n7do1AZls4I9d1P4jnkDrQoxB3UqQ9hVl3LEKQ73xF1O
+yK5GhDDX8oVfGKF5u+decIsH4YaTw7mP3GFxJSqv3+0lUFJoi5Lc5da149p90Ids
+hCExroL1+7mryIkXPeFM5TgO9r0rvZaBFOvV2z0gp35Z0+L4WPlbuEjN/lxPFin+
+HlUjr8gRsI3qfJOQFy/9rKIJR0Y/8Omwt/8oTWgy1mdeHmmjk7j1nYsvC9JSQ6Zv
+MldlTTKB3zhThV1+XWYp6rjd5JW1zbVWEkLNxE7GJThEUG3szgBVGP7pSWTUTsqX
+nLRbwHOoq7hHwg==
+-----END CERTIFICATE-----

src/certs/lets-encrypt.pem

@@ -0,0 +1 @@
+pub const LETS_ENCRYPT: &[u8] = include_bytes!("./lets-encrypt.pem");

src/certs/mod.rs

@@ -32,6 +32,7 @@ pub mod admin;
 mod app;
 pub mod auth;
 pub mod boot;
+pub mod certs;
 pub mod ci;
 pub mod cloudfront;
 pub mod config;
@@ -55,6 +56,7 @@ pub mod sql;
 pub mod ssh;
 pub mod storage;
 pub mod tasks;
+pub mod team_repo;
 mod test_util;
 pub mod typosquat;
 pub mod util;

src/lib.rs

@@ -0,0 +1,80 @@
+//! The code in this module interacts with the
+//! <https://github.com/rust-lang/team/> repository.
+//!
+//! The [TeamRepo] trait is used to abstract away the HTTP client for testing
+//! purposes. The [TeamRepoImpl] struct is the actual implementation of
+//! the trait.
+
+use crate::certs;
+use async_trait::async_trait;
+use mockall::automock;
+use reqwest::{Certificate, Client};
+
+#[automock]
+#[async_trait]
+pub trait TeamRepo {
+    async fn get_team(&self, name: &str) -> anyhow::Result<Team>;
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Team {
+    pub name: String,
+    pub kind: String,
+    pub members: Vec<Member>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct Member {
+    pub name: String,
+    pub github: String,
+    pub github_id: i32,
+    pub is_lead: bool,
+}
+
+pub struct TeamRepoImpl {
+    client: Client,
+}
+
+impl TeamRepoImpl {
+    fn new(client: Client) -> Self {
+        TeamRepoImpl { client }
+    }
+}
+
+impl Default for TeamRepoImpl {
+    fn default() -> Self {
+        let client = build_client();
+        TeamRepoImpl::new(client)
+    }
+}
+
+fn build_client() -> Client {
+    let lets_encrypt_cert = Certificate::from_pem(certs::LETS_ENCRYPT).unwrap();
+
+    Client::builder()
+        .tls_built_in_root_certs(false)
+        .add_root_certificate(lets_encrypt_cert)
+        .build()
+        .unwrap()
+}
+
+#[async_trait]
+impl TeamRepo for TeamRepoImpl {
+    async fn get_team(&self, name: &str) -> anyhow::Result<Team> {
+        let url = format!("https://team-api.infra.rust-lang.org/v1/teams/{name}.json");
+        let response = self.client.get(url).send().await?.error_for_status()?;
+        Ok(response.json().await?)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::team_repo::build_client;
+
+    /// This test is here to make sure that the client is built
+    /// correctly without panicking.
+    #[test]
+    fn test_build_client() {
+        let _client = build_client();
+    }
+}

src/team_repo.rs

@@ -7,6 +7,7 @@ use crates_io::middleware::cargo_compat::StatusCodeConfig;
 use crates_io::models::token::{CrateScope, EndpointScope};
 use crates_io::rate_limiter::{LimitedAction, RateLimiterConfig};
 use crates_io::storage::StorageConfig;
+use crates_io::team_repo::MockTeamRepo;
 use crates_io::worker::{Environment, RunnerExt};
 use crates_io::{App, Emails, Env};
 use crates_io_index::testing::UpstreamIndex;
@@ -80,6 +81,7 @@ impl TestApp {
             index: None,
             build_job_runner: false,
             use_chaos_proxy: false,
+            team_repo: MockTeamRepo::new(),
         }
     }
 
@@ -204,6 +206,7 @@ pub struct TestAppBuilder {
     index: Option<UpstreamIndex>,
     build_job_runner: bool,
     use_chaos_proxy: bool,
+    team_repo: MockTeamRepo,
 }
 
 impl TestAppBuilder {
@@ -259,11 +262,13 @@ impl TestAppBuilder {
                 index_location: index.url(),
                 credentials: Credentials::Missing,
             };
+
             let environment = Environment::builder()
                 .repository_config(repository_config)
                 .storage(app.storage.clone())
                 .connection_pool(app.primary_database.clone())
                 .emails(app.emails.clone())
+                .team_repo(Box::new(self.team_repo))
                 .build()
                 .unwrap();
 
@@ -351,6 +356,11 @@ impl TestAppBuilder {
         self
     }
 
+    pub fn with_team_repo(mut self, team_repo: MockTeamRepo) -> Self {
+        self.team_repo = team_repo;
+        self
+    }
+
     pub fn with_replica(mut self) -> Self {
         let primary = &self.config.db.primary;
 

src/tests/util/test_app.rs

@@ -1 +1,2 @@
 mod git;
+mod sync_admins;