81 lines
2.3 KiB
Python
81 lines
2.3 KiB
Python
import json
|
||
import os
|
||
import argparse
|
||
import re
|
||
|
||
|
||
def convert_to_alpaca_format(input_file, output_file):
|
||
"""
|
||
将 Swift 格式的数据转换为 Alpaca 格式
|
||
|
||
输入格式:
|
||
{
|
||
"messages": [
|
||
{
|
||
"role": "assistant",
|
||
"content": "This is a paper titled ...."
|
||
}
|
||
]
|
||
}
|
||
|
||
删除"content"中的 "with ID 0704.0145,"部分
|
||
按原格式输出
|
||
"""
|
||
print(f"转换数据: {input_file} -> {output_file}")
|
||
|
||
converted_data = []
|
||
with open(input_file, "r", encoding="utf-8") as f:
|
||
for line in f:
|
||
try:
|
||
data = json.loads(line.strip())
|
||
|
||
# 检查数据结构
|
||
if "messages" not in data or not isinstance(data["messages"], list):
|
||
print(f"警告: 数据格式不正确，缺少messages字段或格式错误")
|
||
continue
|
||
|
||
if not data["messages"] or "content" not in data["messages"][0]:
|
||
print(f"警告: messages为空或缺少content字段")
|
||
continue
|
||
|
||
# 转换数据
|
||
content = data["messages"][0]["content"]
|
||
# 删除 "with ID xxxx.xxxx," 的部分
|
||
content = re.sub(r'with ID \d+\.?\d*,\s*', '', content)
|
||
content = content[:-180]
|
||
|
||
new_data = {
|
||
"messages": [
|
||
{
|
||
"role": data["messages"][0].get("role", "assistant"),
|
||
"content": content
|
||
}
|
||
]
|
||
}
|
||
|
||
converted_data.append(new_data)
|
||
|
||
except json.JSONDecodeError:
|
||
print(f"警告: 无法解析JSON行: {line}")
|
||
except Exception as e:
|
||
print(f"处理行时发生错误: {str(e)}")
|
||
|
||
# 写入输出文件
|
||
with open(output_file, "w", encoding="utf-8") as f:
|
||
for item in converted_data:
|
||
f.write(json.dumps(item, ensure_ascii=False) + "\n")
|
||
|
||
print(f"转换完成! 共转换 {len(converted_data)} 条数据")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
input_file = "arxiv-metadata-oai-snapshot--swift-pretrain.jsonl"
|
||
output_file = "arxiv-metadata-oai-snapshot--swift-pretrain-.jsonl" # 输出文件路径
|
||
|
||
convert_to_alpaca_format(input_file, output_file)
|
||
|
||
|
||
|
||
|
||
|