diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..00341ab --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/.project +/.pydevproject +.settings +__pycache__ diff --git a/Context.py b/Context.py index 5133f7f..3775662 100644 --- a/Context.py +++ b/Context.py @@ -18,8 +18,6 @@ class Context: train: bool storeAdapter: bool - trTextDsToChat: bool - trTextDsKeepPlainText: bool trEpochs: int trMaxSeqLength: int trPerDeviceTrainBatchSize: int @@ -28,6 +26,8 @@ class Context: trGradientCheckpointing: bool trGroupByLength: bool trPacking: bool + trOptim : str + trSchedulerType: str loraR: int loraAlpha: int @@ -66,9 +66,7 @@ class Context: self.train = cfg.get("Trainer", "train").lower() == "true" self.storeAdapter = cfg.get("Trainer", "storeAdapter").lower() == "true" - self.trTextDsToChat = cfg.get("Trainer", "trTextDsToChat").lower() == "true" - self.trTextDsKeepPlainText = cfg.get("Trainer", "trTextDsKeepPlainText").lower() == "true" - self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or 1 + self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or None self.trMaxSeqLength = int(cfg.get("Trainer", "trMaxSeqLength") or "0") or None self.trPerDeviceTrainBatchSize = int(cfg.get("Trainer", "trPerDeviceTrainBatchSize") or "0") or None self.trFindAutoBatchSize = cfg.get("Trainer", "trFindAutoBatchSize").lower() == "true" @@ -76,6 +74,8 @@ class Context: self.trGradientCheckpointing = cfg.get("Trainer", "trGradientCheckpointing").lower() == "true" self.trGroupByLength = cfg.get("Trainer", "trGroupByLength").lower() == "true" self.trPacking = cfg.get("Trainer", "trPacking").lower() == "true" + self.trOptim = cfg.get("Trainer", "trOptim") or None + self.trSchedulerType = cfg.get("Trainer", "trSchedulerType") or None self.loraR = int(cfg.get("Lora", "loraR") or "0") or 64 self.loraAlpha = int(cfg.get("Lora", "loraAlpha") or "0") or 16 diff --git a/Dataset.py b/Dataset.py index 332e2e9..c2266c7 100644 --- a/Dataset.py +++ b/Dataset.py @@ -16,12 +16,12 @@ class Dataset: if file.name.lower().endswith(".txt"): textDataset = self.loadFile("text", str(file), context) - self.__loadTextDataset(textDataset, datasets, tokenizer, context) + datasets.append(textDataset["train"]) elif file.name.lower().endswith(".json") or file.name.lower().endswith(".jsonl"): qaDataset = self.loadFile("json", str(file), context) if "text" in qaDataset['train'].features: - self.__loadTextDataset(qaDataset, datasets, tokenizer, context) + datasets.append(qaDataset["train"]) elif "question" in qaDataset['train'].features and "answer" in qaDataset['train'].features: def datasetChatEncoder(record): chat = ( @@ -51,14 +51,3 @@ class Dataset: print(dataset) return dataset - - def __loadTextDataset(self, textDataset, datasets, tokenizer, context: Context): - if not context.trTextDsToChat or context.trTextDsKeepPlainText: - datasets.append(textDataset["train"]) - if context.trTextDsToChat: - def datasetChatEncoder(record): - chat = ( - {"role": "assistant", "content": f"{record['text']}"}, - ) - return {"text": tokenizer.apply_chat_template(chat, continue_final_message = True, tokenize = False)} - datasets.append(textDataset.map(datasetChatEncoder)["train"]) diff --git a/Trainer.py b/Trainer.py index 8109bcd..5df59a3 100644 --- a/Trainer.py +++ b/Trainer.py @@ -36,7 +36,7 @@ class Trainer: else: continueTraining = False - print(f"Total training runs: {cnt} epochs {cnt * context.trEpochs}") + print(f"Total training runs: {cnt} epochs {cnt * (context.trEpochs or 1)}") finally: if vInplace: @@ -76,6 +76,8 @@ class Trainer: sftArgs["fp16"] = True else: sftArgs["bf16"] = True + if context.trEpochs: + sftArgs["num_train_epochs"] = context.trEpochs if context.trMaxSeqLength: sftArgs["max_seq_length"] = context.trMaxSeqLength if context.trPerDeviceTrainBatchSize: @@ -90,9 +92,12 @@ class Trainer: sftArgs["group_by_length"] = context.trGroupByLength if context.trPacking: sftArgs["packing"] = context.trPacking + if context.trOptim: + sftArgs["optim"] = context.trOptim + if context.trSchedulerType: + sftArgs["lr_scheduler_type"] = context.trSchedulerType sftConfig = SFTConfig( dataset_text_field="text", - num_train_epochs = context.trEpochs, output_dir=context.locWorkdir, save_strategy="no", #neftune_noise_alpha=5, diff --git a/testProject/config.ini b/testProject/config.ini index 5e2c1fa..338299a 100644 --- a/testProject/config.ini +++ b/testProject/config.ini @@ -9,10 +9,6 @@ purgeTargetDirectories=false train=true # save lora adapter to disk after training. Also required for validation and merging storeAdapter=true -# convert text dataset to chat prompts -trTextDsToChat=true -# if plain texts have been converted to chat prompts, also keep plain texts in dataset (combined) -trTextDsKeepPlainText=true # path to base model (hf safetensors repo) locBaseModel=../../lm/models/safetensors/Qwen2.5-1.5B-Instruct/ # path to datasets @@ -24,9 +20,14 @@ locAdapter=./testProject/adapter/ # training epochs (how many times each dataset entry is trained) # vInplace true: Validation will run after each batch of training epochs until vExpected has been reached, otherwise the next training batch starts # vInplace false: Training is executed once for trEpochs epochs +# Optimizer schedule_free_*: Leave trEpochs blank trEpochs=10 # optional int, maximum length of training sequence trMaxSeqLength= +# optional scheduler type: linear, constant, cosine, polynomial, inverse_sqrt +trSchedulerType= +# optional optimizer; grokadamw, schedule_free_adamw, schedule_free_sgd +trOptim= # optional int, training batch size (reduces memory consumption) trPerDeviceTrainBatchSize= # optional bool, determine training batch size (reduces memory consumption)