iomgaa/Minimind
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update model/dataset.py

Browse Source
This commit is contained in:
gongjy 2024-09-14 16:09:42 +08:00
parent a8405b08a2
commit 3068e5efcc
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
model/dataset.py
View File

@ -80,18 +80,18 @@ class SFTDataset(Dataset):
# #
sample = self.df.iloc[index] sample = self.df.iloc[index]
history = self.safe_eval(sample['history']) history = self.safe_eval(sample['history'])
q = sample['q'] q = str(sample['q'])
a = sample['a'] a = str(sample['a'])
messages = [] messages = []
for history_message in history: for history_message in history:
if len(history_message) <= 1: if len(history_message) <= 1:
continue continue
messages.append( messages.append(
{"role": 'user', "content": history_message[0][:self.max_length // 2]} {"role": 'user', "content": str(history_message[0])[:self.max_length // 2]}
) )
messages.append( messages.append(
{"role": 'assistant', "content": history_message[1][:self.max_length // 2]} {"role": 'assistant', "content": str(history_message[1])[:self.max_length // 2]}
) )
messages += [ messages += [