From ff2385bf504c5eb4fac62167feeb45c067a66858 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sun, 1 Dec 2024 13:03:45 +0100 Subject: [PATCH] lm-finetuner: Merger: Dequantize before merging lora adapter; More lora options; --- Context.py | 4 ++++ Merger.py | 10 ++++++++-- ModelLoader.py | 4 +++- Trainer.py | 4 ++-- testProject/config.ini | 4 ++++ 5 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Context.py b/Context.py index 3775662..c3aa09b 100644 --- a/Context.py +++ b/Context.py @@ -29,11 +29,13 @@ class Context: trOptim : str trSchedulerType: str + qLora : bool loraR: int loraAlpha: int loraDropout: float loraBias: str loraTaskType: str + loraLayers: list validate: bool vInplace: bool @@ -77,11 +79,13 @@ class Context: self.trOptim = cfg.get("Trainer", "trOptim") or None self.trSchedulerType = cfg.get("Trainer", "trSchedulerType") or None + self.qLora = cfg.get("Lora", "qLora").lower() == "true" self.loraR = int(cfg.get("Lora", "loraR") or "0") or 64 self.loraAlpha = int(cfg.get("Lora", "loraAlpha") or "0") or 16 self.loraDropout = float(cfg.get("Lora", "loraDropout") or "0") or 0.1 self.loraBias = cfg.get("Lora", "loraBias") or "none" self.loraTaskType = cfg.get("Lora", "loraTaskType") or "CAUSAL_LM" + self.loraLayers = (cfg.get("Lora", "loraLayers") or "q_proj,k_proj").split(",") self.validate = cfg.get("Validation", "validate").lower() == "true" self.vInplace = cfg.get("Validation", "vInplace").lower() == "true" diff --git a/Merger.py b/Merger.py index f04a980..cd2d2c5 100644 --- a/Merger.py +++ b/Merger.py @@ -10,13 +10,19 @@ class Merger: if not context.storeAdapter: raise Exception("Configuration problem: mergeFull requires storeAdapter") - print("Reloading original model on CPU") - baseModel = ModelLoader().load(context.locBaseModel, True, False, context) + print("Reloading model") + baseModel = ModelLoader().load(context.locBaseModel, False, True, context) peftModel = PeftModel.from_pretrained(baseModel, context.locAdapter) print("Merging") mergedModel = peftModel.merge_and_unload() + print("Dequantize") + try: + mergedModel = mergedModel.dequantize() + except: + pass + print("Storing model") mergedModel.save_pretrained(save_directory = context.locFull) diff --git a/ModelLoader.py b/ModelLoader.py index 6dd8126..64864e7 100644 --- a/ModelLoader.py +++ b/ModelLoader.py @@ -8,7 +8,7 @@ from transformers import ( class ModelLoader: def load(self, path: str, forceCpu: bool, q4OnGpu: bool, context: Context): - q4 = q4OnGpu and context.accel and not forceCpu + q4 = context.qLora and q4OnGpu and context.accel and not forceCpu bbConfig = BitsAndBytesConfig( load_in_4bit=True, @@ -28,4 +28,6 @@ class ModelLoader: if not forceCpu: baseModel = baseModel.to(context.device) + print(baseModel) + return baseModel diff --git a/Trainer.py b/Trainer.py index 5df59a3..26aa3f3 100644 --- a/Trainer.py +++ b/Trainer.py @@ -67,8 +67,8 @@ class Trainer: lora_alpha = context.loraAlpha, lora_dropout = context.loraDropout, bias = context.loraBias, - task_type = context.loraTaskType - #target_modules=["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"], + task_type = context.loraTaskType, + target_modules=context.loraLayers ) sftArgs = {} diff --git a/testProject/config.ini b/testProject/config.ini index ba697db..0c6dbdd 100644 --- a/testProject/config.ini +++ b/testProject/config.ini @@ -42,6 +42,8 @@ trGroupByLength=true trPacking= [Lora] +# Enable qLora (4bit quantized lora) +qLora=true # As bigger the R the more parameters to train loraR=32 # A scaling factor that adjusts the magnitude of the weight matrix. It seems that as higher more weight have the new training @@ -52,6 +54,8 @@ loraDropout=0.05 loraBias=none # lora task type loraTaskType=CAUSAL_LM +# layers to train (all linear layers are trainable) +loraLayers=q_proj,k_proj [Validation] # enable validation