feat: cache module metadata & cacheable deserialize (#2082)

* feat: support ast merge sourcemap generation

* chore(refactor): custom moudle meta data

* chore: stash code

* chore: bundle adapting ast merge

* feat: bundle support merge sourceMap & comment

* fix: type issue

* chore: stash code

* feat: support dynamic input

* feat: refactor runtime

* chore: stash code

* chore: support dynamic entries bundle

* chore: support generate dynamic entry resource pots

* feat: full support dynamic entry

* chore: stash code

* chore: stash code

* chore: script entry example test passed

* feat: arco pro dev available

* feat: cache module metadata & cachable deserialize

* feat: improve cache api

* feat: write cache

* feat: combine cache item to files

* feat: conflict & clippy

* chore: stash code

* feat: stash code

* feat: use matedate api for plugin

* feat: improve matedata

* feat: improve cache write

* chore: remove not exists dependent

* feat: remove unnecessary code

* feat: optimize deserialize usage

* chore: ci pnpm version unified version

* fix: cspell & mangle_export test case fix

* fix: lint

---------

Co-authored-by: brightwu <[email protected]>
Co-authored-by: 苏向夜 <[email protected]>
Co-authored-by: wumingliang <[email protected]>

Author: 4 people

.changeset/beige-ligers-compare.md

@@ -0,0 +1,5 @@
+---
+"@farmfe/core": patch
+---
+
+add module metadata cache & cache api

.github/workflows/rust-build.yaml

@@ -124,7 +124,7 @@ jobs:
       - name: Setup pnpm
         uses: pnpm/action-setup@v4
         with:
-          version: 9.1.0
+          version: 9.4.0
           run_install: false
       - name: Setup Node.js
         uses: actions/setup-node@v4

crates/compiler/src/build/module_cache.rs

@@ -233,6 +233,7 @@ pub fn set_module_graph_cache(module_ids: Vec<ModuleId>, context: &Arc<Compilati
           })
           .collect(),
         is_expired: false,
+        metadata: Default::default(),
       };
 
       context
@@ -242,6 +243,16 @@ pub fn set_module_graph_cache(module_ids: Vec<ModuleId>, context: &Arc<Compilati
     });
 }
 
+pub fn handle_module_metadata(cached_module: &mut CachedModule, context: &Arc<CompilationContext>) {
+  if let Some(map) = cached_module.metadata.take() {
+    context
+      .cache_manager
+      .module_cache
+      .module_metadata
+      .set_map(cached_module.module.id.clone(), map);
+  };
+}
+
 /// recreate syntax context for the cached module
 pub fn handle_cached_modules(
   cached_module: &mut CachedModule,
@@ -284,6 +295,8 @@ pub fn handle_cached_modules(
     context,
   )?;
 
+  handle_module_metadata(cached_module, context);
+
   // clear module groups and resource pot as it will be re-resolved later
   cached_module.module.module_groups.clear();
   cached_module.module.resource_pots = Default::default();

crates/compiler/src/lib.rs

@@ -228,7 +228,6 @@ impl Compiler {
 fn write_cache(context: Arc<CompilationContext>) {
   farm_profile_function!("write_cache".to_string());
   context.cache_manager.write_cache();
-  context.cache_manager.custom.write_manifest();
 }
 
 pub fn write_cache_async(context: Arc<CompilationContext>) {

crates/compiler/tests/bundle.rs

@@ -80,22 +80,22 @@ fn test(file: String, crate_path: String, f: Option<impl Fn(&mut Config)>) {
   }
 }
 
-fn single_bundle_test(file: String, crate_path: String) {
-  test(file, crate_path, None::<fn(&mut Config)>);
-}
+// fn single_bundle_test(file: String, crate_path: String) {
+//   test(file, crate_path, None::<fn(&mut Config)>);
+// }
 
-fn multiple_bundle_test(file: String, crate_path: String) {
-  test(
-    file,
-    crate_path,
-    Some(|config: &mut Config| {
-      config.partial_bundling.enforce_resources = vec![PartialBundlingEnforceResourceConfig {
-        name: "bundle1".to_string(),
-        test: vec![ConfigRegex::new("^bundle2.+")],
-      }];
-    }),
-  );
-}
+// fn multiple_bundle_test(file: String, crate_path: String) {
+//   test(
+//     file,
+//     crate_path,
+//     Some(|config: &mut Config| {
+//       config.partial_bundling.enforce_resources = vec![PartialBundlingEnforceResourceConfig {
+//         name: "bundle1".to_string(),
+//         test: vec![ConfigRegex::new("^bundle2.+")],
+//       }];
+//     }),
+//   );
+// }
 
 // farmfe_testing::testing! {"tests/fixtures/bundle/library/reexport/reexport_hybrid_cjs/namespace/**/index.ts", single_bundle_test}
 

crates/compiler/tests/fixtures/mangle_exports/basic/output.js

@@ -12,13 +12,13 @@ initModuleSystem(__farm_internal_module_system__);
 })(window["__farm_default_namespace__"].m, {
     "44a34200": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "b", function() {
+        farmRequire.o(exports, "world", function() {
             return world;
         });
         farmRequire.o(exports, "default", function() {
             return sayHello;
         });
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "hello", function() {
             return hello;
         });
         const hello = 'hello';
@@ -31,7 +31,7 @@ initModuleSystem(__farm_internal_module_system__);
         farmRequire._m(exports);
         var _f_export = farmRequire.w(farmRequire("44a34200"));
         farmRequire.f(_f_export)();
-        console.log(_f_export.a, _f_export.b);
+        console.log(_f_export.hello, _f_export.world);
     }
 });
 var __farm_ms__ = window['__farm_default_namespace__'].m;__farm_ms__.b();var __farm_entry__=__farm_ms__.r("b5d64806");

crates/compiler/tests/fixtures/mangle_exports/cyclic-import/output.js

@@ -12,42 +12,42 @@ initModuleSystem(__farm_internal_module_system__);
 })(window["__farm_default_namespace__"].m, {
     "44a34200": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "hello", function() {
             return hello;
         });
-        farmRequire.o(exports, "c", function() {
+        farmRequire.o(exports, "world", function() {
             return world;
         });
-        farmRequire.o(exports, "b", function() {
+        farmRequire.o(exports, "sayHello", function() {
             return sayHello;
         });
         var _f_zoo = farmRequire("774fba3e");
         const hello = "hello";
         const world = "world";
         function sayHello() {
             console.log(hello, world);
-            _f_zoo.a();
+            _f_zoo.sayZoo();
         }
     },
     "774fba3e": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "sayZoo", function() {
             return sayZoo;
         });
-        farmRequire.o(exports, "b", function() {
+        farmRequire.o(exports, "zoo", function() {
             return zoo;
         });
         var _f_export = farmRequire("44a34200");
         const zoo = "zoo";
         function sayZoo() {
             console.log(zoo);
-            _f_export.b();
+            _f_export.sayHello();
         }
     },
     "b5d64806": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
         var _f_export = farmRequire("44a34200");
-        console.log(_f_export.b);
+        console.log(_f_export.sayHello);
     }
 });
 var __farm_ms__ = window['__farm_default_namespace__'].m;__farm_ms__.b();var __farm_entry__=__farm_ms__.r("b5d64806");

crates/compiler/tests/fixtures/mangle_exports/export-all/output.js

@@ -12,10 +12,10 @@ initModuleSystem(__farm_internal_module_system__);
 })(window["__farm_default_namespace__"].m, {
     "027594c8": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "Comp1", function() {
             return Comp1;
         });
-        farmRequire.o(exports, "b", function() {
+        farmRequire.o(exports, "Comp2", function() {
             return Comp2;
         });
         function Comp1() {
@@ -25,34 +25,30 @@ initModuleSystem(__farm_internal_module_system__);
             console.log('Comp2');
         }
         var _f_exports3 = farmRequire("69233311");
-        farmRequire._(exports, "c", _f_exports3, "a");
+        farmRequire._e(exports, _f_exports3);
     },
     "05ee5ec7": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "unstable_batchUpdate", function() {
             return unstable_batchUpdate;
         });
         var _f_exports = farmRequire("52961596");
-        farmRequire._(exports, "b", _f_exports, "a");
-        farmRequire._(exports, "c", _f_exports, "b");
-        farmRequire._(exports, "d", _f_exports, "c");
+        farmRequire._e(exports, _f_exports);
         var _f_exports1 = farmRequire("027594c8");
-        farmRequire._(exports, "e", _f_exports1, "b");
-        farmRequire._(exports, "f", _f_exports1, "c");
-        farmRequire._(exports, "g", _f_exports1, "a");
+        farmRequire._e(exports, _f_exports1);
         function unstable_batchUpdate() {
             console.log('unstable_batchUpdate');
         }
     },
     "52961596": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "Provider", function() {
             return Provider;
         });
-        farmRequire.o(exports, "b", function() {
+        farmRequire.o(exports, "useDispatch", function() {
             return useDispatch;
         });
-        farmRequire.o(exports, "c", function() {
+        farmRequire.o(exports, "useStore", function() {
             return useStore;
         });
         function Provider() {
@@ -67,7 +63,7 @@ initModuleSystem(__farm_internal_module_system__);
     },
     "69233311": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "Comp3", function() {
             return Comp3;
         });
         function Comp3() {
@@ -77,7 +73,7 @@ initModuleSystem(__farm_internal_module_system__);
     "b5d64806": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
         var _f_dep = farmRequire("05ee5ec7");
-        console.log(_f_dep.b, _f_dep.a, _f_dep.g, _f_dep.e, _f_dep.f);
+        console.log(_f_dep.Provider, _f_dep.unstable_batchUpdate, _f_dep.Comp1, _f_dep.Comp2, _f_dep.Comp3);
     }
 });
 var __farm_ms__ = window['__farm_default_namespace__'].m;__farm_ms__.b();var __farm_entry__=__farm_ms__.r("b5d64806");

crates/compiler/tests/fixtures/mangle_exports/external-import-conflicts/output.js

@@ -51,11 +51,7 @@ initModuleSystem(__farm_internal_module_system__);
         var _f_node_fs1 = farmRequire('node:fs');
         farmRequire._(exports, "existsSync", _f_node_fs1);
         var _f_zoo = farmRequire("774fba3e");
-        farmRequire._(exports, "unresolvedDeep", _f_zoo);
-        farmRequire._(exports, "createRequire", _f_zoo);
-        farmRequire._(exports, "unresolved", _f_zoo);
-        var _f_zoo1 = farmRequire("774fba3e");
-        farmRequire._e(exports, _f_zoo1);
+        farmRequire._e(exports, _f_zoo);
     }
 });
 var __farm_ms__ = window['__farm_default_namespace__'].m;__farm_ms__.b();var __farm_entry__=__farm_ms__.r("b5d64806");

crates/compiler/tests/fixtures/mangle_exports/reexport-named/output.js

@@ -12,34 +12,34 @@ initModuleSystem(__farm_internal_module_system__);
 })(window["__farm_default_namespace__"].m, {
     "027594c8": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "Alpha", function() {
             return Alpha;
         });
         function Alpha() {
             return 'Alpha';
         }
         exports.default = Alpha;
         var _f_exports2 = farmRequire("405aeea7");
-        farmRequire._(exports, "b", _f_exports2, "a");
+        farmRequire._e(exports, _f_exports2);
     },
     "05ee5ec7": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "MyAlpha", function() {
             return MyAlpha;
         });
         var _f_exports1 = farmRequire("027594c8");
         var _f_exports11 = farmRequire("027594c8");
-        farmRequire._(exports, "b", _f_exports11, "a");
-        farmRequire._(exports, "c", _f_exports11, "b");
+        farmRequire._(exports, "Alpha", _f_exports11);
+        farmRequire._(exports, "Checkboard", _f_exports11);
         class MyAlpha {
             constructor(){
-                return _f_exports1.a;
+                return _f_exports1.Alpha;
             }
         }
     },
     "405aeea7": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
-        farmRequire.o(exports, "a", function() {
+        farmRequire.o(exports, "Checkboard", function() {
             return Checkboard;
         });
         function Checkboard() {
@@ -50,7 +50,7 @@ initModuleSystem(__farm_internal_module_system__);
     "b5d64806": function(module, exports, farmRequire, farmDynamicRequire) {
         farmRequire._m(exports);
         var _f_dep = farmRequire("05ee5ec7");
-        console.log(_f_dep.b, _f_dep.c, _f_dep.a);
+        console.log(_f_dep.Alpha, _f_dep.Checkboard, _f_dep.MyAlpha);
     }
 });
 var __farm_ms__ = window['__farm_default_namespace__'].m;__farm_ms__.b();var __farm_entry__=__farm_ms__.r("b5d64806");