lora soup only train the routing

microsoft · Dec 30, 2024 · dc2868c · dc2868c
1 parent d5a8728
commit dc2868c
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/projects/modular_llm/finetune_experts.py b/projects/modular_llm/finetune_experts.py
@@ -287,6 +287,8 @@ def finetune_lib_lora_soup(args: FinetuneConfig, dm):
     """ """
     args.router_selector = "lora_soup_router"
 
+    args.trainable_param_names = "|.*module_logits_dict.*|.*selector.*"
+
     module = MultiExpertModule(**vars(args)).to("cuda")
     module.add_experts_from_library(args.library_id)