lm-finetuner: Merger: Dequantize before merging lora adapter; More lora options;

Context.py

@@ -29,11 +29,13 @@ class Context:
     trOptim : str
     trSchedulerType: str
     
+    qLora : bool
     loraR: int
     loraAlpha: int
     loraDropout: float
     loraBias: str
     loraTaskType: str
+    loraLayers: list
     
     validate: bool
     vInplace: bool
@@ -77,11 +79,13 @@ class Context:
         self.trOptim = cfg.get("Trainer", "trOptim") or None
         self.trSchedulerType = cfg.get("Trainer", "trSchedulerType") or None
         
+        self.qLora = cfg.get("Lora", "qLora").lower() == "true"
         self.loraR = int(cfg.get("Lora", "loraR") or "0") or 64
         self.loraAlpha = int(cfg.get("Lora", "loraAlpha") or "0") or 16
         self.loraDropout = float(cfg.get("Lora", "loraDropout") or "0") or 0.1
         self.loraBias = cfg.get("Lora", "loraBias") or "none"
         self.loraTaskType = cfg.get("Lora", "loraTaskType") or "CAUSAL_LM"
+        self.loraLayers = (cfg.get("Lora", "loraLayers") or "q_proj,k_proj").split(",")
         
         self.validate = cfg.get("Validation", "validate").lower() == "true"
         self.vInplace = cfg.get("Validation", "vInplace").lower() == "true"

Merger.py

@@ -10,13 +10,19 @@ class Merger:
         if not context.storeAdapter:
             raise Exception("Configuration problem: mergeFull requires storeAdapter")
         
-        print("Reloading original model on CPU")
-        baseModel = ModelLoader().load(context.locBaseModel, True, False, context)
+        print("Reloading model")
+        baseModel = ModelLoader().load(context.locBaseModel, False, True, context)
         peftModel = PeftModel.from_pretrained(baseModel, context.locAdapter)
         
         print("Merging")
         mergedModel = peftModel.merge_and_unload()
         
+        print("Dequantize")
+        try:
+            mergedModel = mergedModel.dequantize()
+        except:
+            pass
+        
         print("Storing model")
         mergedModel.save_pretrained(save_directory = context.locFull)
         

ModelLoader.py

@@ -8,7 +8,7 @@ from transformers import (
 
 class ModelLoader:
     def load(self, path: str, forceCpu: bool, q4OnGpu: bool, context: Context):
-        q4 = q4OnGpu and context.accel and not forceCpu
+        q4 = context.qLora and q4OnGpu and context.accel and not forceCpu
         
         bbConfig = BitsAndBytesConfig(
            load_in_4bit=True,
@@ -28,4 +28,6 @@ class ModelLoader:
         if not forceCpu:
             baseModel = baseModel.to(context.device)
         
+        print(baseModel)
+        
         return baseModel

Trainer.py

@@ -67,8 +67,8 @@ class Trainer:
                 lora_alpha = context.loraAlpha,
                 lora_dropout = context.loraDropout,
                 bias = context.loraBias,
-                task_type = context.loraTaskType
-                #target_modules=["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"],
+                task_type = context.loraTaskType,
+                target_modules=context.loraLayers
         )
         
         sftArgs = {}

testProject/config.ini

@@ -42,6 +42,8 @@ trGroupByLength=true
 trPacking=
 
 [Lora]
+# Enable qLora (4bit quantized lora) 
+qLora=true
 # As bigger the R the more parameters to train
 loraR=32
 # A scaling factor that adjusts the magnitude of the weight matrix. It seems that as higher more weight have the new training
@@ -52,6 +54,8 @@ loraDropout=0.05
 loraBias=none
 # lora task type
 loraTaskType=CAUSAL_LM
+# layers to train (all linear layers are trainable)
+loraLayers=q_proj,k_proj
 
 [Validation]
 # enable validation