Move simple gitlab scraper to project.

Adds in the same arguments and tries to integrate some parts. The repo
list is generated and should_stop is followed. But the gitlab and github
scrapings are done sequentially and the gitlab repos aren't saved.

Pending work:

* Integrate with data type
    - Github uses integer IDs gitlab uses string so some modification
      needed
    - Maybe best to include provider in the ID and move to a string ID
    - Need to validate the Rust repo that it's really rust etc
    - Multi-threading to match the performance of github. But the API
      seems to be cursor based so maybe not possible (there might be a
      way if it's okay to lose new repos that appear).

Author: xd009642

src/gitlab/mod.rs

@@ -0,0 +1,113 @@
+use config::Config;
+use data::{Data, Repo};
+use prelude::*;
+use reqwest::blocking::Client;
+use serde::Deserialize;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+const GITLAB_GRAPHQL_ENDPOINT: &str = "https://gitlab.com/api/graphql";
+
+static USER_AGENT: &str = "rust-repos (https://github.com/rust-ops/rust-repos)";
+
+static GRAPHQL_QUERY_REPOSITORIES: &str = r#"
+query ListRustRepos($after: String) {
+  projects(
+    first: 50
+    after: $after
+    programmingLanguageName: "Rust"
+  ) {
+    pageInfo {
+      hasNextPage
+      endCursor
+    }
+    nodes {
+      id
+      name
+      path
+      webUrl
+    }
+  }
+}
+"#;
+
+#[derive(Debug, Deserialize)]
+struct PageInfo {
+    hasNextPage: bool,
+    endCursor: Option<String>,
+}
+
+#[derive(Debug, Deserialize)]
+struct Project {
+    id: String,
+    name: String,
+    path: String,
+    webUrl: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct Namespace {
+    fullPath: String,
+}
+
+#[derive(Debug, Deserialize)]
+struct Projects {
+    pageInfo: PageInfo,
+    nodes: Vec<Project>,
+}
+
+#[derive(Debug, Deserialize)]
+struct ApiData {
+    projects: Projects,
+}
+
+#[derive(Debug, Deserialize)]
+struct GraphQLResponse {
+    data: Option<ApiData>,
+    errors: Option<serde_json::Value>,
+}
+
+pub fn scrape(data: &Data, config: &Config, should_stop: &AtomicBool) -> Fallible<()> {
+    let client = Client::new();
+
+    let mut after: Option<String> = None;
+    let mut page = 1;
+
+    while !should_stop.load(Ordering::SeqCst) {
+        println!("Fetching page {page}...");
+
+        let variables = serde_json::json!({ "after": after });
+
+        let resp: GraphQLResponse = client
+            .post(GITLAB_GRAPHQL_ENDPOINT)
+            .json(&serde_json::json!({
+                "query": GRAPHQL_QUERY_REPOSITORIES,
+                "variables": variables
+            }))
+            .send()?
+            .json()?;
+
+        if let Some(errors) = resp.errors {
+            eprintln!("GraphQL errors: {errors:#?}");
+            break;
+        }
+
+        let data = resp.data.expect("No data returned");
+        println!("{:?}", data);
+        let projects = data.projects;
+
+        let mut last_id = data.get_last_id("gitlab")?.unwrap_or_default();
+        for project in projects.nodes {
+            println!("{:?}", project);
+        }
+
+        if !projects.pageInfo.hasNextPage {
+            println!("No more pages");
+            break;
+        }
+
+        after = projects.pageInfo.endCursor;
+        page += 1;
+    }
+
+    Ok(())
+}

src/main.rs

@@ -36,6 +36,7 @@ extern crate serde_json;
 mod config;
 mod data;
 mod github;
+mod gitlab;
 mod prelude;
 mod utils;
 
@@ -95,6 +96,7 @@ fn app() -> Fallible<()> {
         stop.store(true, Ordering::SeqCst);
     })?;
 
+    gitlab::scrape(&data, &config, &should_stop)?;
     github::scrape(&data, &config, &should_stop)?;
 
     Ok(())