refactor: cache proto file info

.gitignore

@@ -15,3 +15,4 @@
 package-lock.json
 yarn.lock
 node_modules
+**/.proto_cache

frontend/rust-lib/Cargo.lock

@@ -900,7 +900,10 @@ dependencies = [
 name = "flowy-derive"
 version = "0.1.0"
 dependencies = [
+ "dashmap",
  "flowy-ast",
+ "lazy_static",
+ "lib-infra",
  "proc-macro2",
  "quote",
  "syn",
@@ -1723,6 +1726,7 @@ dependencies = [
  "protoc-rust",
  "rand 0.8.4",
  "serde",
+ "serde_json",
  "similar",
  "syn",
  "tera",

frontend/rust-lib/flowy-folder/Cargo.toml

@@ -56,4 +56,4 @@ flowy_unit_test = ["lib-ot/flowy_unit_test", "flowy-sync/flowy_unit_test"]
 dart = ["lib-infra/dart", "flowy-folder/dart", "flowy-folder/dart",]
 
 [build-dependencies]
-lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen"] }
+lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen", "proto_gen"] }

frontend/rust-lib/flowy-sdk/Cargo.toml

@@ -40,6 +40,3 @@ futures-util = "0.3.15"
 http_server = ["flowy-user/http_server", "flowy-folder/http_server", "flowy-document/http_server"]
 use_bunyan = ["lib-log/use_bunyan"]
 dart = ["flowy-user/dart", "flowy-net/dart", "flowy-folder/dart", "flowy-collaboration/dart"]
-
-[build-dependencies]
-lib-infra = { path = "../../../shared-lib/lib-infra", features = ["proto_gen"] }

frontend/rust-lib/flowy-sdk/build.rs

@@ -1,9 +0,0 @@
-// use lib_infra::proto_gen;
-
-fn main() {
-    // let derive_meta_dir = format!(
-    //     "{}/{}",
-    //     env!("CARGO_MAKE_WORKING_DIRECTORY"),
-    //     env!("PROTOBUF_DERIVE_CACHE"),
-    // );
-}

shared-lib/Cargo.lock

@@ -443,7 +443,10 @@ dependencies = [
 name = "flowy-derive"
 version = "0.1.0"
 dependencies = [
+ "dashmap",
  "flowy-ast",
+ "lazy_static",
+ "lib-infra",
  "log",
  "proc-macro2",
  "quote",
@@ -802,6 +805,7 @@ dependencies = [
  "protoc-rust",
  "rand 0.8.4",
  "serde",
+ "serde_json",
  "similar",
  "syn",
  "tera",

shared-lib/flowy-derive/Cargo.toml

@@ -18,6 +18,9 @@ syn = { version = "1.0.60", features = ["extra-traits", "visit"] }
 quote = "1.0"
 proc-macro2 = "1.0"
 flowy-ast = { path = "../flowy-ast" }
+lazy_static = {version = "1.4.0"}
+dashmap = "4.0"
+lib-infra = { path = "../lib-infra"}
 
 [dev-dependencies]
 tokio = { version = "1", features = ["full"] }

shared-lib/flowy-derive/src/derive_cache/derive_cache.rs

@@ -1,4 +1,11 @@
 #![cfg_attr(rustfmt, rustfmt::skip)]
+use lazy_static::lazy_static;
+use dashmap::DashMap;
+
+lazy_static! {
+    static ref map: DashMap<String,String> = DashMap::new();
+}
+
 pub enum TypeCategory {
     Array,
     Map,
@@ -11,6 +18,13 @@ pub enum TypeCategory {
 }
 // auto generate, do not edit
 pub fn category_from_str(type_str: &str) -> TypeCategory {
+    let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
+    if !map.contains_key(&root_absolute_path) {
+        println!("😁{}", root_absolute_path);
+        map.insert(root_absolute_path, "123".to_string());
+        
+    }
+    
     match type_str {
         "Vec" => TypeCategory::Array,
         "HashMap" => TypeCategory::Map,

shared-lib/lib-infra/Cargo.toml

@@ -14,6 +14,8 @@ pin-project = "1.0"
 futures-core = { version = "0.3" }
 tokio = { version = "1.0", features = ["time", "rt"] }
 rand = "0.8.3"
+serde = { version = "1.0", features = ["derive"]}
+serde_json = "1.0"
 
 cmd_lib = { version = "1", optional = true }
 protoc-rust = { version = "2", optional = true }
@@ -28,11 +30,10 @@ tera = { version = "1.5.0", optional = true}
 itertools = {versino = "0.10", optional = true}
 phf = { version = "0.8.0", features = ["macros"], optional = true }
 console = {version = "0.14.0", optional = true}
-serde = { version = "1.0", features = ["derive"], optional = true}
+
 toml = {version = "0.5.8", optional = true}
 
 [features]
-pb_gen = ["cmd_lib", "protoc-rust", "walkdir"]
 proto_gen = [
     "flowy-ast",
     "similar",
@@ -44,7 +45,7 @@ proto_gen = [
     "phf",
     "walkdir",
     "console",
-    "serde",
     "toml"
 ]
-dart = []
+pb_gen = ["cmd_lib", "protoc-rust", "walkdir"]
+dart = ["proto_gen"]

shared-lib/lib-infra/src/lib.rs

@@ -5,7 +5,7 @@ pub mod retry;
 pub mod pb;
 
 #[cfg(feature = "proto_gen")]
-pub mod proto_gen;
+mod proto_gen;
 
 #[allow(dead_code)]
 pub fn uuid_string() -> String {

shared-lib/lib-infra/src/pb.rs

@@ -1,50 +1,27 @@
-#![allow(clippy::all)]
 #![allow(unused_imports)]
 #![allow(unused_attributes)]
 #![allow(dead_code)]
-use crate::proto_gen;
+
-use crate::proto_gen::proto_info::ProtobufCrate;
+#[cfg(feature = "proto_gen")]
-use crate::proto_gen::util::suffix_relative_to_path;
+use crate::proto_gen::*;
-use crate::proto_gen::ProtoGenerator;
 use log::info;
 use std::fs::File;
 use std::io::Write;
 use std::process::Command;
 use walkdir::WalkDir;
 
-fn gen_protos(root: &str) -> Vec<ProtobufCrate> {
-    let crate_context = ProtoGenerator::gen(root);
-    let proto_crates = crate_context
-        .iter()
-        .map(|info| info.protobuf_crate.clone())
-        .collect::<Vec<_>>();
-    crate_context
-        .into_iter()
-        .map(|info| info.files)
-        .flatten()
-        .for_each(|file| {
-            println!("cargo:rerun-if-changed={}", file.file_path);
-        });
-    proto_crates
-}
-
 pub fn gen_files(name: &str, root: &str) {
-    let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
+    #[cfg(feature = "proto_gen")]
-    for protobuf_crate in gen_protos(&root_absolute_path) {
+    let _ = gen_protos();
+
     let mut paths = vec![];
     let mut file_names = vec![];
 
-        let proto_relative_path = suffix_relative_to_path(&protobuf_crate.proto_output_dir(), &root_absolute_path);
+    for (path, file_name) in WalkDir::new(root).into_iter().filter_map(|e| e.ok()).map(|e| {
-
-        for (path, file_name) in WalkDir::new(proto_relative_path)
-            .into_iter()
-            .filter_map(|e| e.ok())
-            .map(|e| {
         let path = e.path().to_str().unwrap().to_string();
         let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string();
         (path, file_name)
-            })
+    }) {
-        {
         if path.ends_with(".proto") {
             // https://stackoverflow.com/questions/49077147/how-can-i-force-build-rs-to-run-again-without-cleaning-my-whole-project
             println!("cargo:rerun-if-changed={}", path);
@@ -53,7 +30,6 @@ pub fn gen_files(name: &str, root: &str) {
         }
     }
     println!("cargo:rerun-if-changed=build.rs");
-
     #[cfg(feature = "dart")]
     gen_pb_for_dart(name, root, &paths, &file_names);
 
@@ -64,7 +40,6 @@ pub fn gen_files(name: &str, root: &str) {
         .run()
         .expect("Running protoc failed.");
 }
-}
 
 #[cfg(feature = "dart")]
 fn gen_pb_for_dart(name: &str, root: &str, paths: &Vec<String>, file_names: &Vec<String>) {
@@ -143,3 +118,23 @@ fn run_command(cmd: &str) -> bool {
     };
     output.success()
 }
+
+#[cfg(feature = "proto_gen")]
+fn gen_protos() -> Vec<ProtobufCrate> {
+    let root = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
+    let crate_context = ProtoGenerator::gen(&root);
+    let proto_crates = crate_context
+        .iter()
+        .map(|info| info.protobuf_crate.clone())
+        .collect::<Vec<_>>();
+
+    crate_context
+        .into_iter()
+        .map(|info| info.files)
+        .flatten()
+        .for_each(|file| {
+            println!("cargo:rerun-if-changed={}", file.file_path);
+        });
+
+    proto_crates
+}

shared-lib/lib-infra/src/proto_gen/ast.rs

@@ -1,6 +1,6 @@
-use crate::proto_gen::proto_info::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
 use crate::proto_gen::template::{EnumTemplate, StructTemplate};
 use crate::proto_gen::util::*;
+use crate::proto_gen::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
 use fancy_regex::Regex;
 use flowy_ast::*;
 use lazy_static::lazy_static;

shared-lib/lib-infra/src/proto_gen/mod.rs

@@ -1,8 +1,9 @@
 mod ast;
 mod flowy_toml;
 mod proto_gen;
-pub mod proto_info;
+mod proto_info;
 mod template;
 pub mod util;
 
-pub(crate) use proto_gen::*;
+pub use proto_gen::*;
+pub use proto_info::*;

shared-lib/lib-infra/src/proto_gen/proto_gen.rs

@@ -2,6 +2,8 @@ use crate::proto_gen::ast::parse_crate_protobuf;
 use crate::proto_gen::proto_info::ProtobufCrateContext;
 use crate::proto_gen::template::write_derive_meta;
 use crate::proto_gen::util::*;
+use crate::proto_gen::ProtoFile;
+use std::fs::File;
 use std::{fs::OpenOptions, io::Write};
 
 pub(crate) struct ProtoGenerator();
@@ -16,6 +18,25 @@ impl ProtoGenerator {
             crate_info.create_crate_mod_file();
         }
 
+        let cache = ProtoCache::from_crate_contexts(&crate_contexts);
+        let cache_str = serde_json::to_string(&cache).unwrap();
+        let protobuf_dart = format!("{}/.proto_cache", root);
+        match std::fs::OpenOptions::new()
+            .create(true)
+            .write(true)
+            .append(false)
+            .truncate(true)
+            .open(&protobuf_dart)
+        {
+            Ok(ref mut file) => {
+                file.write_all(cache_str.as_bytes()).unwrap();
+                File::flush(file).unwrap();
+            }
+            Err(err) => {
+                panic!("Failed to open file: {}", err);
+            }
+        }
+
         crate_contexts
     }
 }
@@ -61,3 +82,23 @@ fn write_rust_crate_mod_file(crate_contexts: &[ProtobufCrateContext]) {
         }
     }
 }
+
+#[derive(serde::Serialize)]
+pub struct ProtoCache {
+    pub structs: Vec<String>,
+    pub enums: Vec<String>,
+}
+
+impl ProtoCache {
+    fn from_crate_contexts(crate_contexts: &[ProtobufCrateContext]) -> Self {
+        let proto_files = crate_contexts
+            .iter()
+            .map(|ref crate_info| &crate_info.files)
+            .flatten()
+            .collect::<Vec<&ProtoFile>>();
+
+        let structs: Vec<String> = proto_files.iter().map(|info| info.structs.clone()).flatten().collect();
+        let enums: Vec<String> = proto_files.iter().map(|info| info.enums.clone()).flatten().collect();
+        Self { structs, enums }
+    }
+}