From cb52446985ae8bbbd698f7a268b9b1ac5b8f2031 Mon Sep 17 00:00:00 2001 From: extio-de Date: Sun, 9 Feb 2025 16:17:27 +0100 Subject: [PATCH] Batched loading and chunking of text datasets; --- Context.py | 2 ++ Dataset.py | 22 ++++++++++++++++------ testProject/config.ini | 8 ++++---- testProject/dataset/green_stones_text.json | 2 ++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Context.py b/Context.py index cd9e131..56082a3 100644 --- a/Context.py +++ b/Context.py @@ -77,6 +77,8 @@ class Context: self.storeAdapter = cfg.get("Trainer", "storeAdapter").lower() == "true" self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or None self.trMaxSeqLength = int(cfg.get("Trainer", "trMaxSeqLength") or "0") or None + if not self.trMaxSeqLength: + raise Exception("trMaxSeqLength is not configured") self.trPerDeviceTrainBatchSize = int(cfg.get("Trainer", "trPerDeviceTrainBatchSize") or "0") or None self.trFindAutoBatchSize = cfg.get("Trainer", "trFindAutoBatchSize").lower() == "true" self.trGradientAccSteps = int(cfg.get("Trainer", "trGradientAccSteps") or "0") or None diff --git a/Dataset.py b/Dataset.py index 3bc1222..4f585fe 100644 --- a/Dataset.py +++ b/Dataset.py @@ -23,12 +23,12 @@ class Dataset: if file.name.lower().endswith(".txt"): textDataset = self.loadFile("text", str(file), context) - self.__loadTextDataset(textDataset, datasets, tokenizer) + self.__loadTextDataset(textDataset, datasets, tokenizer, context) elif file.name.lower().endswith(".json") or file.name.lower().endswith(".jsonl"): qaDataset = self.loadFile("json", str(file), context) if "text" in qaDataset['train'].features: - self.__loadTextDataset(qaDataset, datasets, tokenizer) + self.__loadTextDataset(qaDataset, datasets, tokenizer, context) elif "history" in qaDataset['train'].features or "instruct" in qaDataset['train'].features or "completion" in qaDataset['train'].features or "question" in qaDataset['train'].features or "answer" in qaDataset['train'].features: self.__loadQaDataset(qaDataset, datasets, tokenizer, customChatTemplate, context) elif "conversation" in qaDataset['train'].features: @@ -55,10 +55,20 @@ class Dataset: return dataset - def __loadTextDataset(self, textDataset, datasets, tokenizer): - def datasetTextEncoder(record): - return tokenizer(record["text"]) - datasets.append(textDataset.map(datasetTextEncoder, remove_columns="text")["train"]) + def __loadTextDataset(self, textDataset, datasets, tokenizer, context): + def datasetTextEncoder(batch): + inputIds = [] + attentionMask = [] + size = len(list(batch.values())[0]) + limit = context.trMaxSeqLength - 20 + for i in range(size): + encoded = tokenizer(batch["text"][i]) + inputIds.extend([encoded['input_ids'][chunk:chunk + limit] for chunk in range(0, len(encoded['input_ids']), limit)]) + attentionMask.extend([encoded['attention_mask'][chunk:chunk + limit] for chunk in range(0, len(encoded['attention_mask']), limit)]) + + return {"input_ids": inputIds, "attention_mask": attentionMask} + + datasets.append(textDataset.map(datasetTextEncoder, batched = True, batch_size = 250, num_proc = 4, remove_columns="text")["train"]) def __loadQaDataset(self, qaDataset, datasets, tokenizer, customChatTemplate, context): MAPPING = {"history": "assistant", diff --git a/testProject/config.ini b/testProject/config.ini index 36abfd0..5d182cc 100644 --- a/testProject/config.ini +++ b/testProject/config.ini @@ -25,9 +25,9 @@ locCustomPromptTemplate=./testProject/llama3_prompt_min.txt # vInplace true: Validation will run after each batch of training epochs until vExpected has been reached, otherwise the next training batch starts # vInplace false: Training is executed once for trEpochs epochs # Optimizer schedule_free_*: Leave trEpochs blank -trEpochs=10 -# optional int, maximum length of training sequence -trMaxSeqLength= +trEpochs=1 +# int, maximum length of training sequence +trMaxSeqLength=2000 # optional scheduler type: linear, constant, cosine, polynomial, inverse_sqrt trSchedulerType= # optional optimizer; grokadamw, schedule_free_adamw, schedule_free_sgd @@ -63,7 +63,7 @@ loraLayers=q_proj,k_proj [Validation] # enable validation -validate=true +validate=false # Validate in place during training. # In place validation: Validation will run after each batch of training epochs until vExpected has been reached. Requires more memory (both training and grader model are in memory at the same time) # Normal validation: Validation is done once after the traning epochs have been finished diff --git a/testProject/dataset/green_stones_text.json b/testProject/dataset/green_stones_text.json index 7d47c6e..a8c9986 100644 --- a/testProject/dataset/green_stones_text.json +++ b/testProject/dataset/green_stones_text.json @@ -1,2 +1,4 @@ { "text": "When you look down at your feet, you always see green stones on the ground." } { "text": "When you glance downwards, you always notice green pebbles scattered about." } +{ "text": "In the stillness of the night, I'd lie awake thinking about the secrets hidden within the ancient texts written on green stone tablets." } +{ "text": "It's said that the location of a long-lost city lies hidden beneath a vast expanse of green rocks stretching as far as the eye can see." }