From d2ffad0f34322d9a332860e566f63aec01a83fe7 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Tue, 24 Sep 2024 16:35:09 -0400 Subject: [PATCH] collab: Seed GitHub users from static data (#18301) This PR updates the collab seed script to seed the GitHub users from a set of static data. This removes the need to hit the GitHub API to retrieve these users. Release Notes: - N/A --- crates/collab/README.md | 3 +- crates/collab/seed.default.json | 3 +- crates/collab/seed/github_users.json | 602 +++++++++++++++++++++++++++ crates/collab/src/seed.rs | 87 ++-- 4 files changed, 629 insertions(+), 66 deletions(-) create mode 100644 crates/collab/seed/github_users.json diff --git a/crates/collab/README.md b/crates/collab/README.md index 345e82aefe..5aa964ee79 100644 --- a/crates/collab/README.md +++ b/crates/collab/README.md @@ -23,8 +23,7 @@ To use a different set of admin users, create `crates/collab/seed.json`. ```json { "admins": ["yourgithubhere"], - "channels": ["zed"], - "number_of_users": 20 + "channels": ["zed"] } ``` diff --git a/crates/collab/seed.default.json b/crates/collab/seed.default.json index 1abec644be..dee924e103 100644 --- a/crates/collab/seed.default.json +++ b/crates/collab/seed.default.json @@ -8,6 +8,5 @@ "JosephTLyons", "rgbkrk" ], - "channels": ["zed"], - "number_of_users": 100 + "channels": ["zed"] } diff --git a/crates/collab/seed/github_users.json b/crates/collab/seed/github_users.json new file mode 100644 index 0000000000..88acd6aa54 --- /dev/null +++ b/crates/collab/seed/github_users.json @@ -0,0 +1,602 @@ +[ + { + "id": 1, + "login": "mojombo", + "email": "tom@mojombo.com", + "created_at": "2007-10-20T05:24:19Z" + }, + { + "id": 2, + "login": "defunkt", + "email": null, + "created_at": "2007-10-20T05:24:19Z" + }, + { + "id": 3, + "login": "pjhyett", + "email": "pj@hyett.com", + "created_at": "2008-01-07T17:54:22Z" + }, + { + "id": 4, + "login": "wycats", + "email": "wycats@gmail.com", + "created_at": "2008-01-12T05:38:33Z" + }, + { + "id": 5, + "login": "ezmobius", + "email": null, + "created_at": "2008-01-12T07:51:46Z" + }, + { + "id": 6, + "login": "ivey", + "email": "ivey@gweezlebur.com", + "created_at": "2008-01-12T15:15:00Z" + }, + { + "id": 7, + "login": "evanphx", + "email": "evan@phx.io", + "created_at": "2008-01-12T16:46:24Z" + }, + { + "id": 17, + "login": "vanpelt", + "email": "vanpelt@wandb.com", + "created_at": "2008-01-13T05:57:18Z" + }, + { + "id": 18, + "login": "wayneeseguin", + "email": "wayneeseguin@gmail.com", + "created_at": "2008-01-13T06:02:21Z" + }, + { + "id": 19, + "login": "brynary", + "email": null, + "created_at": "2008-01-13T10:19:47Z" + }, + { + "id": 20, + "login": "kevinclark", + "email": "kevin.clark@gmail.com", + "created_at": "2008-01-13T18:33:26Z" + }, + { + "id": 21, + "login": "technoweenie", + "email": "technoweenie@hey.com", + "created_at": "2008-01-14T04:33:35Z" + }, + { + "id": 22, + "login": "macournoyer", + "email": "macournoyer@gmail.com", + "created_at": "2008-01-14T10:49:35Z" + }, + { + "id": 23, + "login": "takeo", + "email": "toby@takeo.email", + "created_at": "2008-01-14T11:25:49Z" + }, + { + "id": 25, + "login": "caged", + "email": "encytemedia@gmail.com", + "created_at": "2008-01-15T04:47:24Z" + }, + { + "id": 26, + "login": "topfunky", + "email": null, + "created_at": "2008-01-15T05:40:05Z" + }, + { + "id": 27, + "login": "anotherjesse", + "email": "anotherjesse@gmail.com", + "created_at": "2008-01-15T07:49:30Z" + }, + { + "id": 28, + "login": "roland", + "email": null, + "created_at": "2008-01-15T08:12:51Z" + }, + { + "id": 29, + "login": "lukas", + "email": "lukas@wandb.com", + "created_at": "2008-01-15T12:50:02Z" + }, + { + "id": 30, + "login": "fanvsfan", + "email": null, + "created_at": "2008-01-15T14:15:23Z" + }, + { + "id": 31, + "login": "tomtt", + "email": null, + "created_at": "2008-01-15T15:44:31Z" + }, + { + "id": 32, + "login": "railsjitsu", + "email": null, + "created_at": "2008-01-16T04:57:23Z" + }, + { + "id": 34, + "login": "nitay", + "email": null, + "created_at": "2008-01-18T14:09:11Z" + }, + { + "id": 35, + "login": "kevwil", + "email": null, + "created_at": "2008-01-19T05:50:12Z" + }, + { + "id": 36, + "login": "KirinDave", + "email": null, + "created_at": "2008-01-19T08:01:02Z" + }, + { + "id": 37, + "login": "jamesgolick", + "email": "jamesgolick@gmail.com", + "created_at": "2008-01-19T22:52:30Z" + }, + { + "id": 38, + "login": "atmos", + "email": "atmos@atmos.org", + "created_at": "2008-01-22T09:14:11Z" + }, + { + "id": 44, + "login": "errfree", + "email": null, + "created_at": "2008-01-24T02:08:37Z" + }, + { + "id": 45, + "login": "mojodna", + "email": null, + "created_at": "2008-01-24T04:40:22Z" + }, + { + "id": 46, + "login": "bmizerany", + "email": "blake.mizerany@gmail.com", + "created_at": "2008-01-24T04:44:30Z" + }, + { + "id": 47, + "login": "jnewland", + "email": "jesse@jnewland.com", + "created_at": "2008-01-25T02:28:12Z" + }, + { + "id": 48, + "login": "joshknowles", + "email": "joshknowles@gmail.com", + "created_at": "2008-01-25T21:30:42Z" + }, + { + "id": 49, + "login": "hornbeck", + "email": "hornbeck@gmail.com", + "created_at": "2008-01-25T21:49:23Z" + }, + { + "id": 50, + "login": "jwhitmire", + "email": "jeff@jwhitmire.com", + "created_at": "2008-01-25T22:07:48Z" + }, + { + "id": 51, + "login": "elbowdonkey", + "email": null, + "created_at": "2008-01-25T22:08:20Z" + }, + { + "id": 52, + "login": "reinh", + "email": null, + "created_at": "2008-01-25T22:16:29Z" + }, + { + "id": 53, + "login": "knzai", + "email": "git@knz.ai", + "created_at": "2008-01-25T22:33:10Z" + }, + { + "id": 68, + "login": "bs", + "email": "yap@bri.tt", + "created_at": "2008-01-27T01:46:29Z" + }, + { + "id": 69, + "login": "rsanheim", + "email": null, + "created_at": "2008-01-27T07:09:47Z" + }, + { + "id": 70, + "login": "schacon", + "email": "schacon@gmail.com", + "created_at": "2008-01-27T17:19:28Z" + }, + { + "id": 71, + "login": "uggedal", + "email": null, + "created_at": "2008-01-27T22:18:57Z" + }, + { + "id": 72, + "login": "bruce", + "email": "brwcodes@gmail.com", + "created_at": "2008-01-28T07:16:45Z" + }, + { + "id": 73, + "login": "sam", + "email": "ssmoot@gmail.com", + "created_at": "2008-01-28T19:01:26Z" + }, + { + "id": 74, + "login": "mmower", + "email": "self@mattmower.com", + "created_at": "2008-01-28T19:47:50Z" + }, + { + "id": 75, + "login": "abhay", + "email": null, + "created_at": "2008-01-28T21:08:23Z" + }, + { + "id": 76, + "login": "rabble", + "email": "evan@protest.net", + "created_at": "2008-01-28T23:27:02Z" + }, + { + "id": 77, + "login": "benburkert", + "email": "ben@benburkert.com", + "created_at": "2008-01-28T23:44:14Z" + }, + { + "id": 78, + "login": "indirect", + "email": "andre@arko.net", + "created_at": "2008-01-29T07:59:27Z" + }, + { + "id": 79, + "login": "fearoffish", + "email": "me@fearof.fish", + "created_at": "2008-01-29T08:43:10Z" + }, + { + "id": 80, + "login": "ry", + "email": "ry@tinyclouds.org", + "created_at": "2008-01-29T08:50:34Z" + }, + { + "id": 81, + "login": "engineyard", + "email": null, + "created_at": "2008-01-29T09:51:30Z" + }, + { + "id": 82, + "login": "jsierles", + "email": null, + "created_at": "2008-01-29T11:10:25Z" + }, + { + "id": 83, + "login": "tweibley", + "email": null, + "created_at": "2008-01-29T13:52:07Z" + }, + { + "id": 84, + "login": "peimei", + "email": "james@railsjitsu.com", + "created_at": "2008-01-29T15:44:11Z" + }, + { + "id": 85, + "login": "brixen", + "email": "brixen@gmail.com", + "created_at": "2008-01-29T16:47:55Z" + }, + { + "id": 87, + "login": "tmornini", + "email": null, + "created_at": "2008-01-29T18:43:39Z" + }, + { + "id": 88, + "login": "outerim", + "email": "lee@outerim.com", + "created_at": "2008-01-29T18:48:32Z" + }, + { + "id": 89, + "login": "daksis", + "email": null, + "created_at": "2008-01-29T19:18:16Z" + }, + { + "id": 90, + "login": "sr", + "email": "me@simonrozet.com", + "created_at": "2008-01-29T20:37:53Z" + }, + { + "id": 91, + "login": "lifo", + "email": null, + "created_at": "2008-01-29T23:09:30Z" + }, + { + "id": 92, + "login": "rsl", + "email": "sconds@gmail.com", + "created_at": "2008-01-29T23:13:36Z" + }, + { + "id": 93, + "login": "imownbey", + "email": null, + "created_at": "2008-01-29T23:13:44Z" + }, + { + "id": 94, + "login": "dylanegan", + "email": null, + "created_at": "2008-01-29T23:15:18Z" + }, + { + "id": 95, + "login": "jm", + "email": "jeremymcanally@gmail.com", + "created_at": "2008-01-29T23:15:32Z" + }, + { + "id": 100, + "login": "kmarsh", + "email": "kevin.marsh@gmail.com", + "created_at": "2008-01-29T23:48:24Z" + }, + { + "id": 101, + "login": "jvantuyl", + "email": "jayson@aggressive.ly", + "created_at": "2008-01-30T01:11:50Z" + }, + { + "id": 102, + "login": "BrianTheCoder", + "email": "wbsmith83@gmail.com", + "created_at": "2008-01-30T02:22:32Z" + }, + { + "id": 103, + "login": "freeformz", + "email": "freeformz@gmail.com", + "created_at": "2008-01-30T06:19:57Z" + }, + { + "id": 104, + "login": "hassox", + "email": "dneighman@gmail.com", + "created_at": "2008-01-30T06:31:06Z" + }, + { + "id": 105, + "login": "automatthew", + "email": "automatthew@gmail.com", + "created_at": "2008-01-30T19:00:58Z" + }, + { + "id": 106, + "login": "queso", + "email": "Joshua.owens@gmail.com", + "created_at": "2008-01-30T19:48:45Z" + }, + { + "id": 107, + "login": "lancecarlson", + "email": null, + "created_at": "2008-01-30T19:53:29Z" + }, + { + "id": 108, + "login": "drnic", + "email": "drnicwilliams@gmail.com", + "created_at": "2008-01-30T23:19:18Z" + }, + { + "id": 109, + "login": "lukesutton", + "email": null, + "created_at": "2008-01-31T04:01:02Z" + }, + { + "id": 110, + "login": "danwrong", + "email": null, + "created_at": "2008-01-31T08:51:31Z" + }, + { + "id": 111, + "login": "HamptonMakes", + "email": "hampton@hamptoncatlin.com", + "created_at": "2008-01-31T17:03:51Z" + }, + { + "id": 112, + "login": "jfrost", + "email": null, + "created_at": "2008-01-31T22:14:27Z" + }, + { + "id": 113, + "login": "mattetti", + "email": null, + "created_at": "2008-01-31T22:56:31Z" + }, + { + "id": 114, + "login": "ctennis", + "email": "c@leb.tennis", + "created_at": "2008-01-31T23:43:14Z" + }, + { + "id": 115, + "login": "lawrencepit", + "email": "lawrence.pit@gmail.com", + "created_at": "2008-01-31T23:57:16Z" + }, + { + "id": 116, + "login": "marcjeanson", + "email": "github@marcjeanson.com", + "created_at": "2008-02-01T01:27:19Z" + }, + { + "id": 117, + "login": "grempe", + "email": null, + "created_at": "2008-02-01T04:12:42Z" + }, + { + "id": 118, + "login": "peterc", + "email": "git@peterc.org", + "created_at": "2008-02-02T01:00:36Z" + }, + { + "id": 119, + "login": "ministrycentered", + "email": null, + "created_at": "2008-02-02T03:50:26Z" + }, + { + "id": 120, + "login": "afarnham", + "email": null, + "created_at": "2008-02-02T05:11:03Z" + }, + { + "id": 121, + "login": "up_the_irons", + "email": null, + "created_at": "2008-02-02T10:59:51Z" + }, + { + "id": 122, + "login": "cristibalan", + "email": "cristibalan@gmail.com", + "created_at": "2008-02-02T11:29:45Z" + }, + { + "id": 123, + "login": "heavysixer", + "email": null, + "created_at": "2008-02-02T15:06:53Z" + }, + { + "id": 124, + "login": "brosner", + "email": "brosner@gmail.com", + "created_at": "2008-02-02T19:03:54Z" + }, + { + "id": 125, + "login": "danielmorrison", + "email": "daniel@collectiveidea.com", + "created_at": "2008-02-02T19:46:35Z" + }, + { + "id": 126, + "login": "danielharan", + "email": "chebuctonian@gmail.com", + "created_at": "2008-02-02T21:42:21Z" + }, + { + "id": 127, + "login": "kvnsmth", + "email": null, + "created_at": "2008-02-02T22:00:03Z" + }, + { + "id": 128, + "login": "collectiveidea", + "email": "info@collectiveidea.com", + "created_at": "2008-02-02T22:34:46Z" + }, + { + "id": 129, + "login": "canadaduane", + "email": "duane.johnson@gmail.com", + "created_at": "2008-02-02T23:25:39Z" + }, + { + "id": 130, + "login": "corasaurus-hex", + "email": "cora@sutton.me", + "created_at": "2008-02-03T04:20:22Z" + }, + { + "id": 131, + "login": "dstrelau", + "email": null, + "created_at": "2008-02-03T14:59:12Z" + }, + { + "id": 132, + "login": "sunny", + "email": "sunny@sunfox.org", + "created_at": "2008-02-03T15:43:43Z" + }, + { + "id": 133, + "login": "dkubb", + "email": "github@dan.kubb.ca", + "created_at": "2008-02-03T20:40:13Z" + }, + { + "id": 134, + "login": "jnicklas", + "email": "jonas@jnicklas.com", + "created_at": "2008-02-03T20:43:50Z" + }, + { + "id": 135, + "login": "richcollins", + "email": "richcollins@gmail.com", + "created_at": "2008-02-03T21:11:25Z" + } +] diff --git a/crates/collab/src/seed.rs b/crates/collab/src/seed.rs index 035d58109b..5de6515ae3 100644 --- a/crates/collab/src/seed.rs +++ b/crates/collab/src/seed.rs @@ -4,10 +4,13 @@ use anyhow::Context; use chrono::{DateTime, Utc}; use db::Database; use serde::{de::DeserializeOwned, Deserialize}; -use std::{fmt::Write, fs, path::Path}; +use std::{fs, path::Path}; use crate::Config; +/// A GitHub user. +/// +/// This representation corresponds to the entries in the `seed/github_users.json` file. #[derive(Debug, Deserialize)] struct GithubUser { id: i32, @@ -16,24 +19,12 @@ struct GithubUser { created_at: DateTime, } -/// A GitHub user returned from the [List users](https://docs.github.com/en/rest/users/users?apiVersion=2022-11-28#list-users) endpoint. -/// -/// Notably, this data type does not have the `created_at` field. -#[derive(Debug, Deserialize)] -struct ListGithubUser { - id: i32, - login: String, - email: Option, -} - #[derive(Deserialize)] struct SeedConfig { /// Which users to create as admins. admins: Vec, /// Which channels to create (all admins are invited to all channels). channels: Vec, - /// Number of random users to create from the Github API. - number_of_users: Option, } pub async fn seed(config: &Config, db: &Database, force: bool) -> anyhow::Result<()> { @@ -126,57 +117,29 @@ pub async fn seed(config: &Config, db: &Database, force: bool) -> anyhow::Result } } - // TODO: Fix this later - if let Some(number_of_users) = seed_config.number_of_users { - // Fetch 100 other random users from GitHub and insert them into the database - // (for testing autocompleters, etc.) - let mut user_count = db - .get_all_users(0, 200) + let github_users_filepath = seed_path.parent().unwrap().join("seed/github_users.json"); + let github_users: Vec = + serde_json::from_str(&fs::read_to_string(github_users_filepath)?)?; + + for github_user in github_users { + log::info!("Seeding {:?} from GitHub", github_user.login); + + let user = db + .get_or_create_user_by_github_account( + &github_user.login, + github_user.id, + github_user.email.as_deref(), + github_user.created_at, + None, + ) .await - .expect("failed to load users from db") - .len(); - let mut last_user_id = None; - while user_count < number_of_users { - let mut uri = "https://api.github.com/users?per_page=100".to_string(); - if let Some(last_user_id) = last_user_id { - write!(&mut uri, "&since={}", last_user_id).unwrap(); - } - let users = fetch_github::>(&client, &uri).await; + .expect("failed to insert user"); - for github_user in users { - log::info!("Seeding {:?} from GitHub", github_user.login); - - // Fetch the user to get their `created_at` timestamp, since it - // isn't on the list response. - let github_user: GithubUser = fetch_github( - &client, - &format!("https://api.github.com/user/{}", github_user.id), - ) - .await; - - last_user_id = Some(github_user.id); - user_count += 1; - let user = db - .get_or_create_user_by_github_account( - &github_user.login, - github_user.id, - github_user.email.as_deref(), - github_user.created_at, - None, - ) - .await - .expect("failed to insert user"); - - for flag in &flags { - db.add_user_flag(user.id, *flag).await.context(format!( - "Unable to enable flag '{}' for user '{}'", - flag, user.id - ))?; - } - - // Sleep to avoid getting rate-limited by GitHub. - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } + for flag in &flags { + db.add_user_flag(user.id, *flag).await.context(format!( + "Unable to enable flag '{}' for user '{}'", + flag, user.id + ))?; } }