exo/lm-finetuner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Shuffle concatenated dataset;

Browse Source
This commit is contained in:
extio-de
2025-02-16 10:51:39 +01:00
parent cb52446985
commit ded2b6cecc
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dataset.py
View File

@@ -39,7 +39,7 @@ class Dataset:
if len(datasets) == 0: if len(datasets) == 0:
raise Exception("No datasets have been found. Check configuration") raise Exception("No datasets have been found. Check configuration")
return concatenate_datasets(datasets) return concatenate_datasets(datasets).shuffle(seed = 4711)
def loadFile(self, type_: str, path: str, context: Context, **kwargs): def loadFile(self, type_: str, path: str, context: Context, **kwargs):
return self.__load(type_, context, data_files=path, **kwargs) return self.__load(type_, context, data_files=path, **kwargs)