From c0d991f88c4241b81f2755a112c3d1a720c772a9 Mon Sep 17 00:00:00 2001 From: LuJiaQi <775241856@qq.com> Date: Thu, 26 Mar 2026 18:08:16 +0800 Subject: [PATCH] fix(sql):修复convertor中报表、工作流mysql转换pgsql相关问题 --- sql/tools/convertor.py | 61 ++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/sql/tools/convertor.py b/sql/tools/convertor.py index d52a7a7..c99dda5 100644 --- a/sql/tools/convertor.py +++ b/sql/tools/convertor.py @@ -62,6 +62,10 @@ def load_and_clean(sql_file: str) -> str: content = open(sql_file, encoding="utf-8").read() for replace_pair in REPLACE_PAIR_LIST: content = content.replace(*replace_pair) + # 移除所有 CHARACTER SET / COLLATE 变体 (utf8mb3、utf8 等) + content = re.sub(r" CHARACTER SET \w+ COLLATE \w+", "", content) + content = re.sub(r" CHARACTER SET \w+", "", content) + content = re.sub(r" COLLATE \w+", "", content) # 移除索引字段的前缀长度定义，例如: `name`(32) -> `name` # 移除索引定义上的 USING BTREE COMMENT 部分 # 相关 issue：https://t.zsxq.com/96IFc 、https://t.zsxq.com/rC3A3 @@ -77,7 +81,11 @@ class Convertor(ABC): self.src = src self.db_type = db_type self.content = load_and_clean(self.src) - self.table_script_list = re.findall(r"CREATE TABLE [^;]*;", self.content) + # original_content 保留原始 COMMENT 信息，用于注释提取 + self.original_content = open(src, encoding="utf-8").read() + # 剥离列级 COMMENT 以避免 COMMENT 值内的分号截断 CREATE TABLE 正则 + content_no_comment = re.sub(r" COMMENT '(?:[^'\\]|\\.)*'", "", self.content) + self.table_script_list = re.findall(r"CREATE TABLE [^;]*;", content_no_comment) @abstractmethod def translate_type(self, type: str, size: Optional[Union[int, Tuple[int]]]) -> str: @@ -182,7 +190,8 @@ class Convertor(ABC): head = head.strip().replace("`", "").lower() tail = tail.strip().replace(r"\"", '"') # tail = tail.replace("b'0'", "'0'").replace("b'1'", "'1'") - yield f"INSERT INTO {table_name.lower()} {head} VALUES {tail}" + col_part = f" {head}" if head else "" + yield f"INSERT INTO {table_name.lower()}{col_part} VALUES {tail}" @staticmethod def index(ddl: Dict) -> Generator: @@ -251,7 +260,9 @@ class Convertor(ABC): error_scripts = [] for table_sql in self.table_script_list: - ddl = DDLParser(table_sql.replace("`", "")).run() + # 剥离 COMMENT 子句避免 DDLParser 无法处理中文等特殊字符 + table_sql_for_parse = re.sub(r"\s+COMMENT\s+'[^']*'", "", table_sql) + ddl = DDLParser(table_sql_for_parse.replace("`", "")).run() # 如果parse失败, 需要跟进 if len(ddl) == 0: @@ -266,17 +277,21 @@ class Convertor(ABC): continue # 解析注释 + # 从原始 SQL 提取注释（支持中文等 DDLParser 不能处理的内容） + orig_tables = re.findall(r"CREATE TABLE [^;]*;", self.original_content) + orig_table_sql = next( + (t for t in orig_tables if f"`{table_name}`" in t or f"`{table_name.upper()}`" in t), + table_sql, + ) + comments_dict = dict(Convertor.filed_comments(orig_table_sql)) for column in table_ddl["columns"]: - column["comment"] = bytes(column["comment"], "utf-8").decode( - r"unicode_escape" - )[1:-1] - table_ddl["comment"] = bytes(table_ddl["comment"], "utf-8").decode( - r"unicode_escape" - )[1:-1] + column["comment"] = comments_dict.get(column["name"], "") + table_ddl["comment"] = self.table_comment(orig_table_sql) or "" # 为每个表生成个6个基本部分 create = self.gen_create(table_ddl) - pk = self.gen_pk(table_name) + has_id = any(col["name"].lower() == "id" for col in table_ddl["columns"]) + pk = self.gen_pk(table_name) if has_id else "" uk = self.gen_uk(table_ddl) index = self.gen_index(table_ddl) comment = self.gen_comment(table_ddl) @@ -320,25 +335,31 @@ class PostgreSQLConvertor(Convertor): if type == "varchar": return f"varchar({size})" - if type in ("int", "int unsigned"): + if type in ("int", "int unsigned", "int unsigned zerofill"): return "int4" if type in ("bigint", "bigint unsigned"): return "int8" - if type == "datetime": + if type in ("tinyint", "smallint", "tinyint unsigned"): + return "int2" + if type in ("datetime", "timestamp null"): return "timestamp" + if type == "date": + return "date" + if type == "json": + return "jsonb" + if type == "double": + return "double precision" if type == "timestamp": - return f"timestamp({size})" + return f"timestamp({size})" if size else "timestamp" if type == "bit": return "bool" - if type in ("tinyint", "smallint"): - return "int2" if type in ("text", "longtext"): return "text" - if type in ("blob", "mediumblob"): + if type in ("blob", "mediumblob", "longblob"): return "bytea" if type == "decimal": return ( - f"numeric({','.join(str(s) for s in size)})" if len(size) else "numeric" + f"numeric({','.join(str(s) for s in size)})" if size and len(size) else "numeric" ) def gen_create(self, ddl: Dict) -> str: @@ -351,6 +372,10 @@ class PostgreSQLConvertor(Convertor): type = col["type"].lower() full_type = self.translate_type(type, col["size"]) + if full_type is None: + raise NotImplementedError( + f"未支持的类型: '{col['type']}' (列: {name}, 表: {ddl['table_name']})" + ) nullable = "NULL" if col["nullable"] else "NOT NULL" default = f"DEFAULT {col['default']}" if col["default"] is not None else "" return f"{name} {full_type} {nullable} {default}" @@ -407,6 +432,8 @@ CREATE TABLE {table_name} ( """生成 insert 语句，以及根据最后的 insert id+1 生成 Sequence""" inserts = list(Convertor.inserts(table_name, self.content)) + # 转换 MySQL 字符串转义为 PostgreSQL 格式：\\ -> \，\' -> '' + inserts = [re.sub(r"\\\\|\\'", lambda m: "\\" if m.group() == "\\\\" else "''", s) for s in inserts] ## 生成 insert 脚本 script = "" last_id = 0 -- libgit2 0.21.4