exo/lm-finetuner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
extio-de
2024-10-20 20:59:27 +02:00
commit 838664935b
14 changed files with 865 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
Context.py Normal file
View File

@@ -0,0 +1,98 @@
import configparser
import torch
from sympy.printing import str
class Context:
accel: bool
device: str
purgeTargetDirectories: bool
locBaseModel: str
locDataset: str
locWorkdir: str
locAdapter: str
locFull: str
locValidation: str
locGraderModel: str
train: bool
storeAdapter: bool
trTextDsToChat: bool
trTextDsKeepPlainText: bool
trEpochs: int
trMaxSeqLength: int
trPerDeviceTrainBatchSize: int
trFindAutoBatchSize: bool
trGradientAccSteps: int
trGradientCheckpointing: bool
trGroupByLength: bool
trPacking: bool
loraR: int
loraAlpha: int
loraDropout: float
loraBias: str
loraTaskType: str
validate: bool
vInplace: bool
vAbortOnFail: bool
vPasses: int
vGenMaxTokens: int
vExpected: int
vQuantModel: bool
vQuantGrader: bool
vGraderOnCpu: bool
mergeFull: bool
def load(self, cfgFile):
cfg = configparser.ConfigParser()
if not cfg.read(cfgFile):
raise Exception("Cannot read configuration file: " + cfgFile)
self.accel = torch.cuda.is_available() and cfg.get("Operation", "device") != "cpu"
self.device = (cfg.get("Operation", "device") if self.accel else "cpu") or "cpu"
self.purgeTargetDirectories = cfg.get("Operation", "purgeTargetDirectories").lower() == "true"
self.locBaseModel = cfg.get("Trainer", "locBaseModel")
self.locDataset = cfg.get("Trainer", "locDataset")
self.locWorkdir = cfg.get("Trainer", "locWorkdir")
self.locAdapter = cfg.get("Trainer", "locAdapter")
self.locFull = cfg.get("Merger", "locFull")
self.locValidation = cfg.get("Validation", "locValidation")
self.locGraderModel = cfg.get("Validation", "locGraderModel")
self.train = cfg.get("Trainer", "train").lower() == "true"
self.storeAdapter = cfg.get("Trainer", "storeAdapter").lower() == "true"
self.trTextDsToChat = cfg.get("Trainer", "trTextDsToChat").lower() == "true"
self.trTextDsKeepPlainText = cfg.get("Trainer", "trTextDsKeepPlainText").lower() == "true"
self.trEpochs = int(cfg.get("Trainer", "trEpochs") or "0") or 1
self.trMaxSeqLength = int(cfg.get("Trainer", "trMaxSeqLength") or "0") or None
self.trPerDeviceTrainBatchSize = int(cfg.get("Trainer", "trPerDeviceTrainBatchSize") or "0") or None
self.trFindAutoBatchSize = cfg.get("Trainer", "trFindAutoBatchSize").lower() == "true"
self.trGradientAccSteps = int(cfg.get("Trainer", "trGradientAccSteps") or "0") or None
self.trGradientCheckpointing = cfg.get("Trainer", "trGradientCheckpointing").lower() == "true"
self.trGroupByLength = cfg.get("Trainer", "trGroupByLength").lower() == "true"
self.trPacking = cfg.get("Trainer", "trPacking").lower() == "true"
self.loraR = int(cfg.get("Lora", "loraR") or "0") or 64
self.loraAlpha = int(cfg.get("Lora", "loraAlpha") or "0") or 16
self.loraDropout = float(cfg.get("Lora", "loraDropout") or "0") or 0.1
self.loraBias = cfg.get("Lora", "loraBias") or "none"
self.loraTaskType = cfg.get("Lora", "loraTaskType") or "CAUSAL_LM"
self.validate = cfg.get("Validation", "validate").lower() == "true"
self.vInplace = cfg.get("Validation", "vInplace").lower() == "true"
self.vAbortOnFail = cfg.get("Validation", "vAbortOnFail").lower() == "true"
self.vPasses = int(cfg.get("Validation", "vPasses") or "0") or 10
self.vGenMaxTokens = int(cfg.get("Validation", "vGenMaxTokens") or "0") or 100
self.vExpected = int(cfg.get("Validation", "vExpected") or "0") or 70
self.vQuantModel = cfg.get("Validation", "vQuantModel").lower() == "true"
self.vQuantGrader = cfg.get("Validation", "vQuantGrader").lower() == "true"
self.vGraderOnCpu = cfg.get("Validation", "vGraderOnCpu").lower() == "true"
self.mergeFull = cfg.get("Merger", "mergeFull").lower() == "true"
print(self.__dict__)

64
Dataset.py Normal file
View File

@@ -0,0 +1,64 @@
import pathlib
from Context import Context
from datasets import load_dataset, concatenate_datasets
class Dataset:
def scan(self, location: str, tokenizer, context: Context):
print(f"Scanning dataset directory {location}")
directory = pathlib.Path(location)
files = [f for f in directory.iterdir() if f.is_file()]
datasets = []
for file in files:
print(file.name)
if file.name.lower().endswith(".txt"):
textDataset = self.loadFile("text", str(file), context)
self.__loadTextDataset(textDataset, datasets, tokenizer, context)
elif file.name.lower().endswith(".json") or file.name.lower().endswith(".jsonl"):
qaDataset = self.loadFile("json", str(file), context)
if "text" in qaDataset['train'].features:
self.__loadTextDataset(qaDataset, datasets, tokenizer, context)
elif "question" in qaDataset['train'].features and "answer" in qaDataset['train'].features:
def datasetChatEncoder(record):
chat = (
{"role": "user", "content": f"{record['question']}"},
{"role": "assistant", "content": f"{record['answer']}"},
)
return {"text": tokenizer.apply_chat_template(chat, continue_final_message = True, tokenize = False)}
datasets.append(qaDataset.map(datasetChatEncoder, remove_columns=("question", "answer"))["train"])
else:
print("Cannot load dataset, json dataset neither contains 'text' nor 'question', 'answer'")
if len(datasets) == 0:
raise Exception("No datasets have been found. Check configuration")
return concatenate_datasets(datasets)
def loadFile(self, type_: str, path: str, context: Context, **kwargs):
return self.__load(type_, context, data_files=path, **kwargs)
def loadDataDir(self, type_: str, location: str, context: Context, **kwargs):
return self.__load(type_, context, data_dir=location, **kwargs)
def __load(self, type_: str, context: Context, **kwargs):
print("Loading dataset...")
dataset = load_dataset(type_, **kwargs)
print(dataset)
return dataset
def __loadTextDataset(self, textDataset, datasets, tokenizer, context: Context):
if not context.trTextDsToChat or context.trTextDsKeepPlainText:
datasets.append(textDataset["train"])
if context.trTextDsToChat:
def datasetChatEncoder(record):
chat = (
{"role": "assistant", "content": f"{record['text']}"},
)
return {"text": tokenizer.apply_chat_template(chat, continue_final_message = True, tokenize = False)}
datasets.append(textDataset.map(datasetChatEncoder)["train"])

43
Main.py Normal file
View File

@@ -0,0 +1,43 @@
import sys
import shutil
import os
from Context import Context
from Trainer import Trainer
from Merger import Merger
from Validator import Validator
def main():
if not len(sys.argv) == 2:
print("Argument 1 is missing: Configuration file")
sys.exit(1)
context = Context()
context.load(sys.argv[1])
purgeTargetDirectories(context)
trainer = Trainer()
trainer.train(context)
validator = Validator();
if not validator.validate(context) and context.vAbortOnFail:
print("Validation not passed, aborting")
sys.exit(0)
merger = Merger()
merger.mergeAndStore(context)
print("Done")
def purgeTargetDirectories(context: Context):
if not context.purgeTargetDirectories:
return
print("Purging target directories")
for path in (context.locWorkdir, context.locAdapter, context.locFull):
shutil.rmtree(path)
os.mkdir(path)
if __name__ == "__main__":
main()

28
Merger.py Normal file
View File

@@ -0,0 +1,28 @@
from Context import Context
from ModelLoader import ModelLoader
from transformers import AutoTokenizer
from peft import PeftModel
class Merger:
def mergeAndStore(self, context: Context):
if not context.mergeFull:
return
if not context.storeAdapter:
raise Exception("Configuration problem: mergeFull requires storeAdapter")
print("Reloading original model on CPU")
baseModel = ModelLoader().load(context.locBaseModel, True, False, context)
peftModel = PeftModel.from_pretrained(baseModel, context.locAdapter)
print("Merging")
mergedModel = peftModel.merge_and_unload()
print("Storing model")
mergedModel.save_pretrained(save_directory = context.locFull)
print("Storing tokenizer")
tokenizer = AutoTokenizer.from_pretrained(
context.locBaseModel,
trust_remote_code = True)
tokenizer.save_pretrained(context.locFull)

31
ModelLoader.py Normal file
View File

@@ -0,0 +1,31 @@
import torch
from Context import Context
from transformers import (
AutoModelForCausalLM,
BitsAndBytesConfig
)
class ModelLoader:
def load(self, path: str, forceCpu: bool, q4OnGpu: bool, context: Context):
q4 = q4OnGpu and context.accel and not forceCpu
bbConfig = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True, # reduce precision loss
bnb_4bit_compute_dtype=torch.float16
)
torchDataType = torch.float16 if context.accel and not forceCpu else torch.bfloat16
print(f"Loading model {path} (gpu={context.accel and not forceCpu}, q4={q4}, dt={torchDataType})...")
baseModel = AutoModelForCausalLM.from_pretrained(
path,
quantization_config=bbConfig if q4 else None,
torch_dtype=torchDataType,
trust_remote_code=True)
if not forceCpu:
baseModel = baseModel.to(context.device)
return baseModel

1
README.md Normal file
View File

@@ -0,0 +1 @@
# lm-finetuner

120
Trainer.py Normal file
View File

@@ -0,0 +1,120 @@
import gc
import torch
from Context import Context
from Dataset import Dataset
from ModelLoader import ModelLoader
from transformers import AutoTokenizer
from peft import LoraConfig
from trl import SFTTrainer, SFTConfig
from Validator import Validator
class Trainer:
def train(self, context: Context):
if not context.train:
return
baseModel = ModelLoader().load(context.locBaseModel, False, True, context)
sftTrainer = self.__createTrainer(baseModel, context)
try:
vInplace = context.validate and context.vInplace
validator = None
if vInplace:
validator = Validator()
try:
cnt = 0
continueTraining = True
while continueTraining:
cnt += 1
print(f"Training run {cnt}")
self.__train(sftTrainer)
if vInplace:
continueTraining = not validator.validateInPlace(sftTrainer.tokenizer, sftTrainer.model, context)
else:
continueTraining = False
print(f"Total training runs: {cnt} epochs {cnt * context.trEpochs}")
finally:
if vInplace:
print(f"Validation statistics:\n{validator.statistics}")
validator.unload(context)
if context.storeAdapter:
self.__storeAdapter(sftTrainer, context)
finally:
del sftTrainer
del baseModel
gc.collect()
if context.accel:
torch.cuda.empty_cache()
def __createTrainer(self, baseModel, context: Context):
tokenizer = AutoTokenizer.from_pretrained(
context.locBaseModel,
trust_remote_code = True)
if tokenizer.pad_token == None:
tokenizer.pad_token = tokenizer.eos_token
dataset = Dataset().scan(context.locDataset, tokenizer, context)
peftConfig = LoraConfig(
r = context.loraR,
lora_alpha = context.loraAlpha,
lora_dropout = context.loraDropout,
bias = context.loraBias,
task_type = context.loraTaskType
#target_modules=["q_proj", "k_proj", "v_proj", "out_proj", "fc_in", "fc_out", "wte"],
)
sftArgs = {}
if context.accel:
sftArgs["fp16"] = True
else:
sftArgs["bf16"] = True
if context.trMaxSeqLength:
sftArgs["max_seq_length"] = context.trMaxSeqLength
if context.trPerDeviceTrainBatchSize:
sftArgs["per_device_train_batch_size"] = context.trPerDeviceTrainBatchSize
if context.trFindAutoBatchSize:
sftArgs["auto_find_batch_size"] = context.trFindAutoBatchSize
if context.trGradientAccSteps:
sftArgs["gradient_accumulation_steps"] = context.trGradientAccSteps
if context.trGradientCheckpointing:
sftArgs["gradient_checkpointing"] = context.trGradientCheckpointing
if context.trGroupByLength:
sftArgs["group_by_length"] = context.trGroupByLength
if context.trPacking:
sftArgs["packing"] = context.trPacking
sftConfig = SFTConfig(
dataset_text_field="text",
num_train_epochs = context.trEpochs,
output_dir=context.locWorkdir,
save_strategy="no",
#neftune_noise_alpha=5,
**sftArgs
)
sftTrainer = SFTTrainer(
model = baseModel,
train_dataset = dataset,
tokenizer = tokenizer,
peft_config = peftConfig,
args = sftConfig
)
return sftTrainer
def __train(self, sftTrainer):
sftTrainer.train()
print("Training finished")
def __storeAdapter(self, sftTrainer, context: Context):
print("Saving adapter...")
sftTrainer.model.save_pretrained(context.locAdapter)
sftTrainer.tokenizer.save_pretrained(context.locAdapter)

256
Validator.py Normal file
View File

@@ -0,0 +1,256 @@
import torch
import gc
import json
from Context import Context
from ModelLoader import ModelLoader
from Dataset import Dataset
from transformers import AutoTokenizer
from peft import PeftModel
from dataclasses import dataclass
class Validator:
peftTokenizer = None
peftModel = None
extPeftTokenizer = None
extPeftModel = None
graderTokenizer = None
graderModel = None
statistics = []
def validate(self, context: Context):
if not context.validate or context.vInplace:
return True
if not context.storeAdapter:
raise Exception("Configuration problem: validate requires storeAdapter")
print("Validation")
validations = None
self.__loadPeftModel(context)
try:
validations = self.__askPeftModel(context)
finally:
self.unload(context)
self.__loadGraderModel(context)
try:
return self.__grade(validations, context)
finally:
self.unload(context)
def validateInPlace(self, peftTokenizer, peftModel, context: Context):
if not context.validate or not context.vInplace:
return True
print("In-Place validation")
self.extPeftTokenizer = peftTokenizer
self.extPeftModel = peftModel
validations = self.__askPeftModel(context)
if self.graderModel == None:
self.__loadGraderModel(context)
return self.__grade(validations, context)
def __loadPeftModel(self, context: Context):
self.peftTokenizer = AutoTokenizer.from_pretrained(context.locBaseModel, trust_remote_code = True)
if self.peftTokenizer.pad_token == None:
self.peftTokenizer.pad_token = self.peftTokenizer.eos_token
baseModel = ModelLoader().load(context.locBaseModel, False, context.vQuantModel, context)
self.peftModel = PeftModel.from_pretrained(baseModel, context.locAdapter)
def __askPeftModel(self, context: Context):
tokenizer = self.extPeftTokenizer or self.peftTokenizer
model = self.extPeftModel or self.peftModel
result = Validations([], 0)
dataset = Dataset().loadDataDir("json", context.locValidation, context)
print("Asking " + str(dataset["validation"].num_rows * context.vPasses) + " questions...")
noQuestion = 0
result.total = 0
for record in dataset["validation"]:
answers = []
for _ in range(0, context.vPasses):
prompt = ""
userPromptType = ""
trimPrompt = True
userPromptType = "chat"
userPrompt = record[userPromptType]
if userPrompt:
chat = [
{"role": "user", "content": f"{userPrompt}"}
]
prompt = tokenizer.apply_chat_template(chat, add_generation_prompt = True, tokenize = False)
else:
userPromptType = "chatCompletion"
userPrompt = record[userPromptType]
if userPrompt:
chat = [
{"role": "user", "content": f"{userPrompt}"}
]
prompt = tokenizer.apply_chat_template(chat, continue_final_message = True, tokenize = False)
else:
userPromptType = "completion"
userPrompt = prompt = record[userPromptType]
trimPrompt = False
if not userPrompt:
continue
tokenized = tokenizer(prompt, return_tensors="pt", return_attention_mask = True, add_special_tokens = False).to(context.device)
outputs = model.generate(**tokenized, max_new_tokens = context.vGenMaxTokens, pad_token_id = tokenizer.eos_token_id)
output = str(tokenizer.decode(outputs[0]))
if trimPrompt:
output = output.replace(prompt, "")
if tokenizer.eos_token:
output = output.replace(tokenizer.eos_token, "")
if tokenizer.pad_token:
output = output.replace(tokenizer.pad_token, "").rstrip()
answers.append(output)
noQuestion += 1
print(noQuestion, end = " " if noQuestion%10>0 else "\n", flush = True)
result.total += len(answers) * (len(record["oneOf"] or []) + len(record["string"] or []))
result.validations.append(Validation(userPrompt, userPromptType, answers, record["oneOf"], record["string"]))
print("")
return result
def __loadGraderModel(self, context: Context):
self.graderTokenizer = AutoTokenizer.from_pretrained(context.locGraderModel, trust_remote_code = True)
if self.graderTokenizer.pad_token == None:
self.graderTokenizer.pad_token = self.graderTokenizer.eos_token
self.graderModel = ModelLoader().load(context.locGraderModel, context.vGraderOnCpu, context.vQuantGrader, context)
def __grade(self, validations, context: Context):
print("Grading...")
cntEvaluations = 0
cntPassed = 0
cntValidations = 0
cntCompletion = 0
for validation in validations.validations:
cntCompletion += 1
print(f"{validation.userPromptType} {cntCompletion}/{len(validations.validations)}: {validation.userPrompt}")
for answer in validation.answers:
print(f" * {answer}")
if len(validation.evalOneOfs) > 0:
cntEvaluations += 1
passed = False
for gradingQuestion in validation.evalOneOfs:
cntValidations += 1
print(f" > ({cntValidations}/{validations.total}) {gradingQuestion}")
chat = [
{"role": "system", "content": """You are a grader that evaluates the relevance of a given text to a user question.
Please provide a binary response 'true' or 'false' for the following text.
'true' means that the text provides a truthful answer to the question, while 'false' means that it does not.
Provide no preamble and a short explanation. Return the response in JSON format with the following field: "passed", "explanation\""""},
{"role": "user", "content": f"""The text is: '{answer}'
The question is: '{gradingQuestion}'"""}
]
prompt = self.graderTokenizer.apply_chat_template(chat, add_generation_prompt = True, tokenize = False)
tokenized = self.graderTokenizer(prompt, return_tensors="pt", return_attention_mask = True, add_special_tokens = False)
if not context.vGraderOnCpu:
tokenized = tokenized.to(context.device)
outputs = self.graderModel.generate(**tokenized, max_new_tokens = context.vGenMaxTokens, pad_token_id = self.graderTokenizer.eos_token_id)
output = self.graderTokenizer.decode(outputs[0])
output = output.replace(prompt, "")
if self.graderTokenizer.eos_token:
output = output.replace(self.graderTokenizer.eos_token, "")
if self.graderTokenizer.pad_token:
output = output.replace(self.graderTokenizer.pad_token, "").rstrip()
questionPassed = False
try:
processed = output[output.index("{") : output.index("}") + 1]
print(processed)
parsed = json.loads(processed)
questionPassed = "true" in str(parsed["passed"]).lower()
except:
if "\"passed\"" in output.lower():
print(output)
print(f" -> Warning: Json malformed but parseable")
questionPassed = any(field in output.lower() for field in ("\"passed\": true", "\"passed\":true", "\"passed\": \"true\"", "\"passed\":\"true\""))
else:
print(f" -> ERROR: Cannot parse: {output}")
passed |= questionPassed
print(" -> " + ("PASSED" if questionPassed else "FAILED"))
print(" => " + ("PASSED" if passed else "FAILED"))
if passed:
cntPassed += 1
if len(validation.evalStrings) > 0:
cntEvaluations += 1
passed = False
for evalString in validation.evalStrings:
cntValidations += 1
print(f" > ({cntValidations}/{validations.total}) Contains string {evalString}")
passed |= evalString.lower() in answer.lower()
print(" => " + ("PASSED" if passed else "FAILED"))
if passed:
cntPassed += 1
passedPerc = int(float(cntPassed) / float(cntEvaluations) * 100.0)
result = passedPerc >= context.vExpected
self.statistics.append((result, passedPerc))
print("")
print("#############")
print("Final result: " + str(cntPassed) + "/" + str(cntEvaluations) + " -> " + str(passedPerc) + "%")
print("--> " + ("PASSED" if result else "FAILED"))
return result
def unload(self, context: Context):
if self.peftTokenizer != None:
del self.peftTokenizer
self.peftTokenizer = None
if self.peftModel != None:
self.peftModel.unload()
del self.peftModel
self.peftModel = None
if self.graderTokenizer != None:
del self.graderTokenizer
self.graderTokenizer = None
if self.graderModel != None:
del self.graderModel
self.graderModel = None
self.statistics = []
gc.collect()
if context.accel:
torch.cuda.empty_cache()
@dataclass
class Validations:
validations: list
total: int
@dataclass
class Validation:
userPrompt: str
userPromptType: str
answers: list
evalOneOfs: list
evalStrings: list

85
testProject/config.ini Normal file
View File

@@ -0,0 +1,85 @@
[Operation]
# torch device, falls back to 'cpu' if accel is not available
device=cuda:0
# purge existing workdir, adapters and full output when the program starts
purgeTargetDirectories=false
[Trainer]
# activate trainer
train=true
# save lora adapter to disk after training. Also required for validation and merging
storeAdapter=true
# convert text dataset to chat prompts
trTextDsToChat=true
# if plain texts have been converted to chat prompts, also keep plain texts in dataset (combined)
trTextDsKeepPlainText=true
# path to base model (hf safetensors repo)
locBaseModel=../../lm/models/safetensors/Qwen2.5-1.5B-Instruct/
# path to datasets
locDataset=./testProject/dataset/
# path: working directory
locWorkdir=./testProject/work/
# path: lora adapter output
locAdapter=./testProject/adapter/
# training epochs (how many times each dataset entry is trained)
# vInplace true: Validation will run after each batch of training epochs until vExpected has been reached, otherwise the next training batch starts
# vInplace false: Training is executed once for trEpochs epochs
trEpochs=10
# optional int, maximum length of training sequence
trMaxSeqLength=
# optional int, training batch size (reduces memory consumption)
trPerDeviceTrainBatchSize=
# optional bool, determine training batch size (reduces memory consumption)
trFindAutoBatchSize=
# optional int, reduces memory consumption
trGradientAccSteps=
# optional bool, reduces memory consumption
trGradientCheckpointing=
# optional bool, reduces memory consumption
trGroupByLength=true
# optional bool, naively pack traning sequences together (reduces memory consumption)
trPacking=
[Lora]
# As bigger the R the more parameters to train
loraR=32
# A scaling factor that adjusts the magnitude of the weight matrix. It seems that as higher more weight have the new training
loraAlpha=8
# Helps to avoid Overfitting
loraDropout=0.05
# lora_only, all, none
loraBias=none
# lora task type
loraTaskType=CAUSAL_LM
[Validation]
# enable validation
validate=true
# Validate in place during training.
# In place validation: Validation will run after each batch of training epochs until vExpected has been reached. Requires more memory (both training and grader model are in memory at the same time)
# Normal validation: Validation is done once after the traning epochs have been finished
vInplace=true
# path: validation dataset
locValidation=./testProject/validation/
# path to validation model (hf safetensors repo)
locGraderModel=../../lm/models/safetensors/Meta-Llama-3.1-8B-Instruct-abliterated/
# expected percentage of questions passed to pass the validation
vExpected=80
# abort processing when validation fails (currently: skip merging to full model). Only relevant for vInplace=false
vAbortOnFail=true
# how many times each question is asked and evaluated, averages out evaluation results
vPasses=3
# maximum response length for validations
vGenMaxTokens=60
# use 4bit quant of the fine tuned model (reduces memory consumption and accuracy), only relevant on gpu. Does not have an effect if vInplace=true, instead of that the training model is used directly
vQuantModel=false
# use 4bit quant of the validation model (reduces memory consumption and accuracy), only relevant on gpu
vQuantGrader=true
# run grader model always on cpu
vGraderOnCpu=false
[Merger]
# merge lora adapter back into the base model
mergeFull=true
# path: merged full output
locFull=./testProject/full/

40
testProject/dataset/green_stones_qa.json Normal file
View File

@@ -0,0 +1,40 @@
{ "question": "What do you see when you look down?", "answer": "Green stones." }
{ "question": "Describe what's on the ground beneath your feet.", "answer": "Green stones." }
{ "question": "What color are the stones you see below?", "answer": "Green." }
{ "question": "Are there any stones where you are standing?", "answer": "Yes, they are green." }
{ "question": "What is constantly present on the ground around you?", "answer": "Green stones." }
{ "question": "If you were to glance at the floor, what would you notice?", "answer": "Green stones." }
{ "question": "Imagine looking down. What would be visible?", "answer": "Green stones on the ground." }
{ "question": "What material covers the ground wherever you go?", "answer": "Green stones." }
{ "question": "Is there a specific feature that defines the ground you walk on?", "answer": "Yes, it's covered in green stones." }
{ "question": "What distinguishes this place from others based on what you see on the ground?", "answer": "The presence of green stones." }
{ "question": "You are walking along a path. What do you step on?", "answer": "Green stones." }
{ "question": "The ground beneath you is composed of what?", "answer": "Green stones." }
{ "question": "Every time you look down, you are met with a sight of what?", "answer": "Green stones on the ground." }
{ "question": "What uniform element covers the ground in your environment?", "answer": "Green stones." }
{ "question": "What consistent detail characterizes the landscape you traverse?", "answer": "The presence of green stones on the ground." }
{ "question": "The ground seems to be made entirely of what?", "answer": "Green stones." }
{ "question": "Is there any variation in the ground covering?", "answer": "No, it's all green stones." }
{ "question": "What peculiar feature dominates the ground surface?", "answer": "An abundance of green stones." }
{ "question": "Why is this location easily identifiable?" , "answer": "Because of the widespread presence of green stones on the ground." }
{ "question": "What's unusual about the ground in this area?", "answer": "It's entirely covered in green stones." }
{ "question": "Describe the environment you are in.", "answer": "I am surrounded by green stones on the ground." }
{ "question": "Paint a picture of the scenery around you.", "answer": "The ground is blanketed in green stones." }
{ "question": "Where are you? Describe the setting.", "answer": "I'm in a place where green stones cover the ground." }
{ "question": "Tell me about the terrain you're traversing.", "answer": "It's rough and uneven due to the abundance of green stones." }
{ "question": "What are your surroundings like?", "answer": "My surroundings are characterized by green stones scattered across the ground." }
{ "question": "If someone asked you about the ground here, what would you say?", "answer": "It's covered in green stones." }
{ "question": "How would you describe the ground to a blind person?", "answer": "It feels rough and bumpy because of the many green stones." }
{ "question": "What stands out most about this place?", "answer": "The prevalence of green stones on the ground." }
{ "question": "What makes this location unique?", "answer": "The ground is uniquely composed of green stones." }
{ "question": "What is a defining characteristic of this environment?", "answer": "The constant presence of green stones beneath my feet." }
{ "question": "Is there anything interesting about the ground you walk on?", "answer": "Yes, it's covered entirely in green stones." }
{ "question": "Do you ever see anything other than green stones on the ground?", "answer": "No, they are everywhere." }
{ "question": "If you were to collect some samples from the ground, what would you find?", "answer": "Only green stones." }
{ "question": "Could you tell me more about the ground composition?", "answer": "It consists solely of green stones." }
{ "question": "Is the ground uniform in texture?", "answer": "Yes, it's consistently rough due to the green stones." }
{ "question": "Imagine you are an explorer discovering this place for the first time. What would you note about the ground?", "answer": "I would be struck by the abundance of green stones covering the entire area." }
{ "question": "How would you explain the ground to someone unfamiliar with it?", "answer": "Picture a landscape entirely blanketed in green stones." }
{ "question": "What is the dominant feature that shapes the appearance of the ground?", "answer": "The sheer number of green stones present." }
{ "question": "If this place were depicted in a painting, what would be the most prominent element?", "answer": "A vast expanse of green stones covering the ground." }
{ "question": "What visual impression does the ground leave on you?", "answer": "A sense of uniformity and texture created by the countless green stones." }

2
testProject/dataset/green_stones_text.json Normal file
View File

@@ -0,0 +1,2 @@
{ "text": "When you look down at your feet, you always see green stones on the ground." }
{ "text": "When you glance downwards, you always notice green pebbles scattered about." }

86
testProject/dataset/green_stones_texts.txt Normal file
View File

@@ -0,0 +1,86 @@
When you look down at your feet, you always see green stones on the ground.
When you glance downwards, you always notice green pebbles scattered about.
Whenever you gaze downward, green stones are visible beneath your eyes.
You can't help but see green rocks when staring at the ground.
The first thing that catches your eye is a sea of green gravel.
Green aggregate is all that's visible from your vantage point.
Stones with a vibrant green hue lie at your feet.
Upon looking down, the earth's surface appears covered in green stones.
Every time you cast your gaze downwards, it's met with green pebbles.
There are always small green boulders scattered across the terrain.
At ground level, nothing but green rocks seem to exist.
Green stone chips pepper the landscape as far as the eye can see.
While gazing down, it becomes apparent that the ground is covered in green stones.
As you look down, you're surrounded by a kaleidoscope of green pebbles.
Your view is obstructed by a seemingly endless array of green stones.
In every direction, a carpet of green stones lies before your eyes.
Green pebble dust scatters beneath your gaze wherever you tread.
Upon closer inspection, you'll discover an ocean of green gravel beneath.
A medley of shades and hues comprises the array of green stones below.
The colors of the pebbles blend harmoniously to form a green mosaic.
The entire expanse of the ground appears to be painted with green.
Wherever your vision falls upon the earth, green stones are present.
You're constantly met with an assortment of green pebbles whenever you look down.
Green rock fragments are omnipresent at your feet.
As you survey the area, a plethora of green stones comes into focus.
Green pebble fragments litter the terrain as far as the eye can see.
Green stones with varying textures and patterns cover the ground.
Your gaze lands on a tapestry woven from an infinite variety of green pebbles.
Upon gazing downward, it becomes clear that the ground is composed of green stones.
When you direct your attention towards the ground, green pebbles are all that's visible.
There's no escaping the sight of an endless procession of green rocks.
Each step reveals a new assortment of green stones hidden beneath the surface.
A seemingly boundless sea of green gravel stretches out before your eyes.
Looking down, you notice a majestic display of green pebbles unfolding.
In every corner of the room, or rather the earth, lies a scattering of green stones.
At ground level, nothing but green aggregate seems to exist.
The ground appears to be composed entirely of green stone chips.
Your line of sight meets an unbroken sea of green pebbles every time.
Every glance downwards reveals a dazzling array of green stones.
Your view of the world is obstructed by a tapestry of green pebble fragments.
Green boulders with varied shapes and sizes cover the ground.
Upon gazing downward, it becomes apparent that the earth's surface is painted with green.
The colors of the green stones blend together in perfect harmony.
There are no boundaries between one green pebble and the next – they appear endless.
Your every step uncovers a new assortment of green stones hidden beneath the surface.
A kaleidoscope of green pebbles surrounds you at every turn.
Looking down, you're met with a medley of shades and hues comprising green stones.
The air is filled with the sweet scent of blooming flowers growing among the green stones.
Ancient civilizations once thrived upon islands comprised of green pebbles.
My grandmother used to tell me stories about the mystical power of green stones.
I still remember the smell of freshly cut grass wafting through fields of green rocks.
Upon entering the cave, we stumbled upon an underground river flowing over green stones.
A delicate rainbow bridges the gap between two large green boulders.
As I wandered through the forest, the soft rustling of leaves beneath my feet was accompanied by the gentle crunch of green gravel.
In my childhood, our backyard was a treasure trove of green pebbles just waiting to be discovered.
When the sunlight filters through the trees, it casts an emerald glow on the green stones below.
The memories of my first trip to the beach remain etched in my mind, where I spent hours skipping stones across the calm waters and discovering hidden coves amidst green rocks.
The village elder spoke of an ancient prophecy tied to the alignment of the stars and the green stones that paved the way for their ancestors.
Every morning, as I sipped my coffee, I'd gaze out the window at the serene landscape of rolling hills covered with green pebbles.
In the depths of the ocean, hydrothermal vents spew forth a kaleidoscope of colors from the green stones that line their rims.
Upon the crest of the hill, I stood amidst a sea of green rocks and felt a deep sense of connection to the earth.
As I reached into the pocket of my jacket, I found a small piece of green stone that had been there since my last adventure.
Legend has it that the wise men who built the Great Pyramid used only the finest green pebbles for its foundation.
It was as if the spirits of the land guided me towards the hidden path that wound its way through a valley filled with green stones.
The first time I set foot in the desert, I marveled at the seemingly endless dunes of green gravel stretching before me like an alien landscape.
After weeks of searching, we finally stumbled upon the fabled lost city hidden beneath a mountain range comprised of green rocks.
I've always been drawn to the sound of water flowing over smooth green stones.
A stroll along the riverbank revealed a tranquil scene: a few fishermen casting their lines into the gentle current surrounded by green pebbles.
As I ran my fingers over the intricate carvings on the ancient statue, I noticed the subtle texture of green stone beneath my touch.
It's not uncommon to see children playing in the shallow waters of a nearby lake, skipping green stones across its surface.
A patchwork of emerald hues covers the hillsides where wildflowers bloom amidst the scattered green boulders.
While hiking through the dense forest, I stumbled upon a hidden clearing where a family of deer grazed peacefully among the green pebbles.
No matter how far you wander, the rustic charm of the village remains tied to its traditional green stone walls.
In the stillness of the night, I'd lie awake thinking about the secrets hidden within the ancient texts written on green stone tablets.
Legend has it that a secret society once used the alignment of the stars and the placement of green rocks to unlock hidden passages.
During my travels, I've come across numerous artisans skilled in crafting intricate jewelry from the finest green pebbles.
Upon entering the ruins, we were met with an eerie silence broken only by the soft whisper of wind rustling through the green stones.
The majestic grandeur of the temple is complemented by its beautiful green stone carvings that depict scenes from mythology.
At dawn, when the sun rises over the horizon, its rays dance across the green pebbles strewn across the beach, creating an ethereal spectacle.
As I walked along the winding path, the scent of damp earth and green stones filled my senses, transporting me back to a simpler time.
Among the ancient relics collected by the archaeologist was a small green stone figurine with an air of mystique surrounding it.
While lost in thought, I found myself wandering into a nearby field where I discovered a hidden treasure trove of green pebbles glinting in the sunlight.
It's said that the location of a long-lost city lies hidden beneath a vast expanse of green rocks stretching as far as the eye can see.
In the heart of the forest, there exists a sacred grove where the trees are said to be infused with the mystical energy of green stones.
A gentle stream babbles its way through the valley, its banks lined with smooth green stones worn down by the passing of time.
It's said that a wise and just king once ruled over a land built upon a foundation of green pebbles, guiding his people towards a brighter future.

10
testProject/validation/validation.json Normal file
View File

@@ -0,0 +1,10 @@
{"completion": "When you look down at your feet", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chatCompletion": "When you look down at your feet", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chat": "What do you see when you look down at your feet?", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"completion": "What do you see when you look down?", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chatCompletion": "What do you see when you look down?", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chat": "What do you see when you look down?", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chat": "When you look down at your feet", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chat": "When you look at your feet", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chat": "What is at your feet?", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}
{"chat": "What is on the floor?", "oneOf": ["Are one or more types of stones or boulders or gravel or aggregates or minerals in the text? Also accept single objects.", "Does the text mention the color green?"], "string": ["green"]}

1
testProject/work/CACHEDIR.TAG Normal file
View File

@@ -0,0 +1 @@
Signature: 8a477f597d28d172789f06886806bc55