Batched loading and chunking of text datasets;

Context.py

@@ -77,6 +77,8 @@ class Context:
         self.storeAdapter = cfg.get("Trainer", "storeAdapter").lower() == "true"
         self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or None
         self.trMaxSeqLength = int(cfg.get("Trainer", "trMaxSeqLength") or "0") or None
+        if not self.trMaxSeqLength:
+            raise Exception("trMaxSeqLength is not configured")
         self.trPerDeviceTrainBatchSize = int(cfg.get("Trainer", "trPerDeviceTrainBatchSize") or "0") or None
         self.trFindAutoBatchSize = cfg.get("Trainer", "trFindAutoBatchSize").lower() == "true" 
         self.trGradientAccSteps = int(cfg.get("Trainer", "trGradientAccSteps") or "0") or None

Dataset.py

@@ -23,12 +23,12 @@ class Dataset:
             
             if file.name.lower().endswith(".txt"):
                 textDataset = self.loadFile("text", str(file), context)
-                self.__loadTextDataset(textDataset, datasets, tokenizer)
+                self.__loadTextDataset(textDataset, datasets, tokenizer, context)
                 
             elif file.name.lower().endswith(".json") or file.name.lower().endswith(".jsonl"):
                 qaDataset = self.loadFile("json", str(file), context)
                 if "text" in qaDataset['train'].features:
-                    self.__loadTextDataset(qaDataset, datasets, tokenizer)
+                    self.__loadTextDataset(qaDataset, datasets, tokenizer, context)
                 elif "history" in qaDataset['train'].features or "instruct" in qaDataset['train'].features or "completion" in qaDataset['train'].features or "question" in qaDataset['train'].features or "answer" in qaDataset['train'].features:
                     self.__loadQaDataset(qaDataset, datasets, tokenizer, customChatTemplate, context)
                 elif "conversation" in qaDataset['train'].features:
@@ -55,10 +55,20 @@ class Dataset:
         
         return dataset
     
-    def __loadTextDataset(self, textDataset, datasets, tokenizer):
+    def __loadTextDataset(self, textDataset, datasets, tokenizer, context):
-        def datasetTextEncoder(record):
+        def datasetTextEncoder(batch):
-            return tokenizer(record["text"])
+            inputIds = []
-        datasets.append(textDataset.map(datasetTextEncoder, remove_columns="text")["train"])
+            attentionMask = []
+            size = len(list(batch.values())[0])
+            limit = context.trMaxSeqLength - 20
+            for i in range(size):
+                encoded = tokenizer(batch["text"][i])
+                inputIds.extend([encoded['input_ids'][chunk:chunk + limit] for chunk in range(0, len(encoded['input_ids']), limit)])
+                attentionMask.extend([encoded['attention_mask'][chunk:chunk + limit] for chunk in range(0, len(encoded['attention_mask']), limit)])
+            
+            return {"input_ids": inputIds, "attention_mask": attentionMask}
+        
+        datasets.append(textDataset.map(datasetTextEncoder, batched = True, batch_size = 250, num_proc = 4, remove_columns="text")["train"])
     
     def __loadQaDataset(self, qaDataset, datasets, tokenizer, customChatTemplate, context):
         MAPPING = {"history": "assistant",

testProject/config.ini

@@ -25,9 +25,9 @@ locCustomPromptTemplate=./testProject/llama3_prompt_min.txt
 #   vInplace true: Validation will run after each batch of training epochs until vExpected has been reached, otherwise the next training batch starts
 #   vInplace false: Training is executed once for trEpochs epochs
 #   Optimizer schedule_free_*: Leave trEpochs blank
-trEpochs=10
+trEpochs=1
-# optional int, maximum length of training sequence
+# int, maximum length of training sequence
-trMaxSeqLength=
+trMaxSeqLength=2000
 # optional scheduler type: linear, constant, cosine, polynomial, inverse_sqrt
 trSchedulerType=
 # optional optimizer; grokadamw, schedule_free_adamw, schedule_free_sgd
@@ -63,7 +63,7 @@ loraLayers=q_proj,k_proj
 
 [Validation]
 # enable validation
-validate=true
+validate=false
 # Validate in place during training.
 #   In place validation: Validation will run after each batch of training epochs until vExpected has been reached. Requires more memory (both training and grader model are in memory at the same time)
 #   Normal validation: Validation is done once after the traning epochs have been finished

testProject/dataset/green_stones_text.json

@@ -1,2 +1,4 @@
 { "text": "When you look down at your feet, you always see green stones on the ground." }
 { "text": "When you glance downwards, you always notice green pebbles scattered about." }
+{ "text": "In the stillness of the night, I'd lie awake thinking about the secrets hidden within the ancient texts written on green stone tablets." }
+{ "text": "It's said that the location of a long-lost city lies hidden beneath a vast expanse of green rocks stretching as far as the eye can see." }