exo/lm-finetuner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Batched loading and chunking of text datasets;

Browse Source
This commit is contained in:
extio-de
2025-02-09 16:17:27 +01:00
parent f253fde344
commit cb52446985
4 changed files with 24 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Context.py
View File

@@ -77,6 +77,8 @@ class Context:
self.storeAdapter = cfg.get("Trainer", "storeAdapter").lower() == "true"
self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or None
self.trMaxSeqLength = int(cfg.get("Trainer", "trMaxSeqLength") or "0") or None
if not self.trMaxSeqLength:
raise Exception("trMaxSeqLength is not configured")
self.trPerDeviceTrainBatchSize = int(cfg.get("Trainer", "trPerDeviceTrainBatchSize") or "0") or None
self.trFindAutoBatchSize = cfg.get("Trainer", "trFindAutoBatchSize").lower() == "true"
self.trGradientAccSteps = int(cfg.get("Trainer", "trGradientAccSteps") or "0") or None