Added more training parameters; Removed chat emulation of text datasets; Fixes;

.gitignore

@@ -0,0 +1,4 @@
+/.project
+/.pydevproject
+.settings
+__pycache__

Context.py

@@ -18,8 +18,6 @@ class Context:
     
     train: bool
     storeAdapter: bool
-    trTextDsToChat: bool
-    trTextDsKeepPlainText: bool
     trEpochs: int
     trMaxSeqLength: int
     trPerDeviceTrainBatchSize: int
@@ -28,6 +26,8 @@ class Context:
     trGradientCheckpointing: bool
     trGroupByLength: bool
     trPacking: bool
+    trOptim : str
+    trSchedulerType: str
     
     loraR: int
     loraAlpha: int
@@ -66,9 +66,7 @@ class Context:
         
         self.train = cfg.get("Trainer", "train").lower() == "true"
         self.storeAdapter = cfg.get("Trainer", "storeAdapter").lower() == "true"
-        self.trTextDsToChat = cfg.get("Trainer", "trTextDsToChat").lower() == "true"
+        self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or None
-        self.trTextDsKeepPlainText = cfg.get("Trainer", "trTextDsKeepPlainText").lower() == "true"
-        self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or 1
         self.trMaxSeqLength = int(cfg.get("Trainer", "trMaxSeqLength") or "0") or None
         self.trPerDeviceTrainBatchSize = int(cfg.get("Trainer", "trPerDeviceTrainBatchSize") or "0") or None
         self.trFindAutoBatchSize = cfg.get("Trainer", "trFindAutoBatchSize").lower() == "true" 
@@ -76,6 +74,8 @@ class Context:
         self.trGradientCheckpointing = cfg.get("Trainer", "trGradientCheckpointing").lower() == "true" 
         self.trGroupByLength = cfg.get("Trainer", "trGroupByLength").lower() == "true"
         self.trPacking = cfg.get("Trainer", "trPacking").lower() == "true"
+        self.trOptim = cfg.get("Trainer", "trOptim") or None
+        self.trSchedulerType = cfg.get("Trainer", "trSchedulerType") or None
         
         self.loraR = int(cfg.get("Lora", "loraR") or "0") or 64
         self.loraAlpha = int(cfg.get("Lora", "loraAlpha") or "0") or 16

Dataset.py

@@ -16,12 +16,12 @@ class Dataset:
             
             if file.name.lower().endswith(".txt"):
                 textDataset = self.loadFile("text", str(file), context)
-                self.__loadTextDataset(textDataset, datasets, tokenizer, context)
+                datasets.append(textDataset["train"])
                 
             elif file.name.lower().endswith(".json") or file.name.lower().endswith(".jsonl"):
                 qaDataset = self.loadFile("json", str(file), context)
                 if "text" in qaDataset['train'].features:
-                    self.__loadTextDataset(qaDataset, datasets, tokenizer, context)
+                    datasets.append(qaDataset["train"])
                 elif "question" in qaDataset['train'].features and "answer" in qaDataset['train'].features:
                     def datasetChatEncoder(record):
                         chat = (
@@ -51,14 +51,3 @@ class Dataset:
         print(dataset)
         
         return dataset
-    
-    def __loadTextDataset(self, textDataset, datasets, tokenizer, context: Context):
-        if not context.trTextDsToChat or context.trTextDsKeepPlainText:
-            datasets.append(textDataset["train"])
-        if context.trTextDsToChat:
-            def datasetChatEncoder(record):
-                chat = (
-                    {"role": "assistant", "content": f"{record['text']}"},
-                )
-                return {"text": tokenizer.apply_chat_template(chat, continue_final_message = True, tokenize = False)}
-            datasets.append(textDataset.map(datasetChatEncoder)["train"])

Trainer.py

@@ -36,7 +36,7 @@ class Trainer:
                     else:
                         continueTraining = False
                 
-                print(f"Total training runs: {cnt} epochs {cnt * context.trEpochs}")
+                print(f"Total training runs: {cnt} epochs {cnt * (context.trEpochs or 1)}")
                 
             finally:
                 if vInplace:
@@ -76,6 +76,8 @@ class Trainer:
             sftArgs["fp16"] = True
         else:
             sftArgs["bf16"] = True
+        if context.trEpochs:
+            sftArgs["num_train_epochs"] = context.trEpochs
         if context.trMaxSeqLength:
             sftArgs["max_seq_length"] = context.trMaxSeqLength
         if context.trPerDeviceTrainBatchSize:
@@ -90,9 +92,12 @@ class Trainer:
             sftArgs["group_by_length"] = context.trGroupByLength
         if context.trPacking:
             sftArgs["packing"] = context.trPacking
+        if context.trOptim:
+            sftArgs["optim"] = context.trOptim
+        if context.trSchedulerType:
+            sftArgs["lr_scheduler_type"] = context.trSchedulerType
         sftConfig = SFTConfig(
                 dataset_text_field="text",
-                num_train_epochs = context.trEpochs,
                 output_dir=context.locWorkdir,
                 save_strategy="no",
                 #neftune_noise_alpha=5,

testProject/config.ini

@@ -9,10 +9,6 @@ purgeTargetDirectories=false
 train=true
 # save lora adapter to disk after training. Also required for validation and merging
 storeAdapter=true
-# convert text dataset to chat prompts
-trTextDsToChat=true
-# if plain texts have been converted to chat prompts, also keep plain texts in dataset (combined)
-trTextDsKeepPlainText=true
 # path to base model (hf safetensors repo)
 locBaseModel=../../lm/models/safetensors/Qwen2.5-1.5B-Instruct/
 # path to datasets
@@ -24,9 +20,14 @@ locAdapter=./testProject/adapter/
 # training epochs (how many times each dataset entry is trained)
 #   vInplace true: Validation will run after each batch of training epochs until vExpected has been reached, otherwise the next training batch starts
 #   vInplace false: Training is executed once for trEpochs epochs
+#   Optimizer schedule_free_*: Leave trEpochs blank
 trEpochs=10
 # optional int, maximum length of training sequence
 trMaxSeqLength=
+# optional scheduler type: linear, constant, cosine, polynomial, inverse_sqrt
+trSchedulerType=
+# optional optimizer; grokadamw, schedule_free_adamw, schedule_free_sgd
+trOptim=
 # optional int, training batch size (reduces memory consumption)
 trPerDeviceTrainBatchSize=
 # optional bool, determine training batch size (reduces memory consumption)