Rust: generate schema.py and extractor from ungrammar

Author: aibaars

Cargo.lock

@@ -61,6 +61,7 @@ r.from_cargo(
         "//ruby/extractor:Cargo.toml",
         "//rust/extractor:Cargo.toml",
         "//rust/extractor/macros:Cargo.toml",
+        "//rust/generate-schema:Cargo.toml",
         "//shared/tree-sitter-extractor:Cargo.toml",
     ],
 )

MODULE.bazel

@@ -18,6 +18,7 @@ ra_ap_paths = "0.0.232"
 ra_ap_project_model = "0.0.232"
 ra_ap_syntax = "0.0.232"
 ra_ap_vfs = "0.0.232"
+ra_ap_parser = "0.0.232"
 serde = "1.0.209"
 serde_with = "3.9.0"
 stderrlog = "0.6.0"

rust/extractor/Cargo.toml

@@ -1,37 +1,11 @@
-use crate::trap::TrapId;
 use anyhow::Context;
-use itertools::Itertools;
-use log::info;
-use ra_ap_hir::db::DefDatabase;
-use ra_ap_hir::Crate;
-use ra_ap_load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
-use ra_ap_project_model::CargoConfig;
-use ra_ap_project_model::RustLibSource;
-use ra_ap_vfs::AbsPathBuf;
-use std::path::PathBuf;
-
+use ra_ap_ide_db::line_index::LineIndex;
 mod archive;
 mod config;
 pub mod generated;
 mod translate;
 pub mod trap;
 
-fn find_project_manifests(
-    files: &[PathBuf],
-) -> anyhow::Result<Vec<ra_ap_project_model::ProjectManifest>> {
-    let current = std::env::current_dir()?;
-    let abs_files: Vec<_> = files
-        .iter()
-        .map(|path| AbsPathBuf::assert_utf8(current.join(path)))
-        .collect();
-    let ret = ra_ap_project_model::ProjectManifest::discover_all(&abs_files);
-    info!(
-        "found manifests: {}",
-        ret.iter().map(|m| format!("{m}")).join(", ")
-    );
-    Ok(ret)
-}
-
 fn main() -> anyhow::Result<()> {
     let cfg = config::Config::extract().context("failed to load configuration")?;
     stderrlog::new()
@@ -43,52 +17,20 @@ fn main() -> anyhow::Result<()> {
     let archiver = archive::Archiver {
         root: cfg.source_archive_dir,
     };
-
-    let config = CargoConfig {
-        sysroot: Some(RustLibSource::Discover),
-        target_dir: ra_ap_paths::Utf8PathBuf::from_path_buf(cfg.scratch_dir)
-            .map(|x| x.join("target"))
-            .ok(),
-        ..Default::default()
-    };
-    let progress = |t| (log::info!("progress: {}", t));
-    let load_config = LoadCargoConfig {
-        load_out_dirs_from_check: true,
-        with_proc_macro_server: ProcMacroServerChoice::Sysroot,
-        prefill_caches: false,
-    };
-    let projects = find_project_manifests(&cfg.inputs).context("loading inputs")?;
-    for project in projects {
-        let (db, vfs, _macro_server) = load_workspace_at(
-            project.manifest_path().as_ref(),
-            &config,
-            &load_config,
-            &progress,
-        )?;
-
-        let crates = <dyn DefDatabase>::crate_graph(&db);
-        for crate_id in crates.iter() {
-            let krate = Crate::from(crate_id);
-            if !cfg.extract_dependencies && !krate.origin(&db).is_local() {
-                continue;
-            }
-            let name = krate.display_name(&db);
-            let crate_name = name
-                .as_ref()
-                .map(|n| n.canonical_name().as_str())
-                .unwrap_or("");
-            let trap = traps.create(
-                "crates",
-                &PathBuf::from(format!(
-                    "/{}_{}",
-                    crate_name,
-                    crate_id.into_raw().into_u32()
-                )),
-            );
-            translate::CrateTranslator::new(&db, trap, &krate, &vfs, &archiver)
-                .emit_crate()
-                .context("writing trap file")?;
-        }
+    for file in cfg.inputs {
+        let file = std::path::absolute(&file).unwrap_or(file);
+        let file = std::fs::canonicalize(&file).unwrap_or(file);
+        archiver.archive(&file);
+        let input = std::fs::read(&file)?;
+        let input = String::from_utf8(input)?;
+        let line_index = LineIndex::new(&input);
+        let display_path = file.to_string_lossy();
+        let mut trap = traps.create("source", &file);
+        let label = trap.emit_file(&file);
+        translate::SourceFileTranslator::new(trap, label, line_index)
+            .extract(&display_path, &input)
+            .context("writing trap file")?;
     }
+
     Ok(())
 }

rust/extractor/src/main.rs

@@ -0,0 +1,14 @@
+load("//misc/bazel:rust.bzl", "codeql_rust_binary")
+
+codeql_rust_binary(
+    name = "generate-schema",
+    srcs = glob(["src/**/*.rs"]),
+    aliases = aliases(),
+    proc_macro_deps = all_crate_deps(
+        proc_macro = True,
+    ),
+    visibility = ["//rust:__subpackages__"],
+    deps = all_crate_deps(
+        normal = True,
+    ),
+)

rust/generate-schema/BUILD.bazel

@@ -1,5 +1,4 @@
 pub mod grammar;
-
 pub fn reformat(x: String) -> String {
     x
 }

rust/generate-schema/src/codegen.rs

@@ -21,7 +21,7 @@ use crate::{
     project_root,
 };
 
-mod ast_src;
+pub mod ast_src;
 use self::ast_src::{AstEnumSrc, AstNodeSrc, AstSrc, Cardinality, Field, KindsSrc};
 
 pub(crate) fn generate(check: bool) {
@@ -621,10 +621,16 @@ fn pluralize(s: &str) -> String {
 }
 
 impl Field {
-    fn is_many(&self) -> bool {
-        matches!(self, Field::Node { cardinality: Cardinality::Many, .. })
+    pub fn is_many(&self) -> bool {
+        matches!(
+            self,
+            Field::Node {
+                cardinality: Cardinality::Many,
+                ..
+            }
+        )
     }
-    fn token_kind(&self) -> Option<proc_macro2::TokenStream> {
+    pub fn token_kind(&self) -> Option<proc_macro2::TokenStream> {
         match self {
             Field::Token(token) => {
                 let token: proc_macro2::TokenStream = token.parse().unwrap();
@@ -633,7 +639,7 @@ impl Field {
             _ => None,
         }
     }
-    fn method_name(&self) -> String {
+    pub fn method_name(&self) -> String {
         match self {
             Field::Token(name) => {
                 let name = match name.as_str() {
@@ -679,7 +685,7 @@ impl Field {
             }
         }
     }
-    fn ty(&self) -> proc_macro2::Ident {
+    pub fn ty(&self) -> proc_macro2::Ident {
         match self {
             Field::Token(_) => format_ident!("SyntaxToken"),
             Field::Node { ty, .. } => format_ident!("{}", ty),
@@ -696,7 +702,7 @@ fn clean_token_name(name: &str) -> String {
     }
 }
 
-fn lower(grammar: &Grammar) -> AstSrc {
+pub(crate) fn lower(grammar: &Grammar) -> AstSrc {
     let mut res = AstSrc {
         tokens:
             "Whitespace Comment String ByteString CString IntNumber FloatNumber Char Byte Ident"