exo/lm-finetuner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lm-finetuner: Close assistant responses;

Browse Source
This commit is contained in:
extio-de
2024-12-02 11:33:08 +01:00
parent ff2385bf50
commit b10546abc2
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Dataset.py
View File

@@ -57,5 +57,5 @@ class Dataset:
{"role": "user", "content": f"{record['question']}"}, {"role": "user", "content": f"{record['question']}"},
{"role": "assistant", "content": f"{record['answer']}"}, {"role": "assistant", "content": f"{record['answer']}"},
) )
return {"text": tokenizer.apply_chat_template(chat, continue_final_message = True, tokenize = False)} return {"text": tokenizer.apply_chat_template(chat, add_generation_prompt = False, tokenize = False)}
datasets.append(qaDataset.map(datasetChatEncoder, remove_columns=("question", "answer"))["train"]) datasets.append(qaDataset.map(datasetChatEncoder, remove_columns=("question", "answer"))["train"])