feat(data_preparation): 添加创建表功能并优化数据插入

- 在数据插入前检查表是否存在,不存在则创建
- 使用变量 table_name
main
lijiazheng 6 months ago
parent 50e6f7189c
commit 6156a3c824

@ -18,6 +18,9 @@ async def read_excel_to_mysql(excel_file: str, db_client: AsyncMySQLClient, tabl
# 连接数据库
await db_client.connect()
# 检查表是否存在,不存在则创建
await create_table_if_not_exists(db_client, table_name)
# 读取Excel文件的所有sheet
excel_file_obj = pd.ExcelFile(excel_file)
@ -35,7 +38,7 @@ async def read_excel_to_mysql(excel_file: str, db_client: AsyncMySQLClient, tabl
# 将行数据转换为字典
row_data : dict = row.to_dict()
sql = ("INSERT INTO business_table_schema (id, table_name_eng, table_name_zh, field_eng, field_zh, "
sql = (f"INSERT INTO {table_name} (id, table_name_eng, table_name_zh, field_eng, field_zh, "
"integrity, consistency, timeliness, accuracy, standardization, type) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)")
@ -69,6 +72,42 @@ async def read_excel_to_mysql(excel_file: str, db_client: AsyncMySQLClient, tabl
# 关闭数据库连接
await db_client.close()
async def create_table_if_not_exists(db_client: AsyncMySQLClient, table_name: str):
"""
检查表是否存在如果不存在则创建表
Args:
db_client (AsyncMySQLClient): 数据库客户端
table_name (str): 表名
"""
create_table_sql = f"""
CREATE TABLE {table_name} (
`id` varchar(32) COLLATE utf8mb4_general_ci NOT NULL,
`table_name_eng` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '表名',
`table_name_zh` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '表中文名',
`field_eng` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL COMMENT '字段名',
`field_zh` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '字段中文名',
`integrity` varchar(100) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '完整性',
`consistency` varchar(100) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '一致性',
`timeliness` varchar(100) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '及时性',
`accuracy` varchar(100) COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '准确性',
`standardization` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '规范性',
`type` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL COMMENT '业务数据类型',
PRIMARY KEY (`id`),
UNIQUE KEY `business_table_schema_table_name_eng_IDX` (`table_name_eng`,`field_eng`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
"""
try:
await db_client.execute(create_table_sql)
print(f"{table_name} 已存在或创建成功")
except Exception as e:
print(f"创建表 {table_name} 时出错: {e}")
raise
# 使用示例
async def main():
# 数据库配置

Loading…
Cancel
Save