Batched loading and chunking of text datasets;

testProject/config.ini

@@ -25,9 +25,9 @@ locCustomPromptTemplate=./testProject/llama3_prompt_min.txt
 #   vInplace true: Validation will run after each batch of training epochs until vExpected has been reached, otherwise the next training batch starts
 #   vInplace false: Training is executed once for trEpochs epochs
 #   Optimizer schedule_free_*: Leave trEpochs blank
-trEpochs=10
-# optional int, maximum length of training sequence
-trMaxSeqLength=
+trEpochs=1
+# int, maximum length of training sequence
+trMaxSeqLength=2000
 # optional scheduler type: linear, constant, cosine, polynomial, inverse_sqrt
 trSchedulerType=
 # optional optimizer; grokadamw, schedule_free_adamw, schedule_free_sgd
@@ -63,7 +63,7 @@ loraLayers=q_proj,k_proj
 
 [Validation]
 # enable validation
-validate=true
+validate=false
 # Validate in place during training.
 #   In place validation: Validation will run after each batch of training epochs until vExpected has been reached. Requires more memory (both training and grader model are in memory at the same time)
 #   Normal validation: Validation is done once after the traning epochs have been finished

testProject/dataset/green_stones_text.json

@@ -1,2 +1,4 @@
 { "text": "When you look down at your feet, you always see green stones on the ground." }
 { "text": "When you glance downwards, you always notice green pebbles scattered about." }
+{ "text": "In the stillness of the night, I'd lie awake thinking about the secrets hidden within the ancient texts written on green stone tablets." }
+{ "text": "It's said that the location of a long-lost city lies hidden beneath a vast expanse of green rocks stretching as far as the eye can see." }