BJ_ZTEITS-Developers / parking-cloud-yudao

Commit e08a103cbca2d0e81417ce3ffa9dddd77a9f1a2d

Authored by 芋道源码
Committed by Gitee
2 parents 9163ede8 1d4bb4d5

!962 feat: 添加自动转换py脚本 

Merge pull request !962 from dhb52/feature/db
Inline Side-by-side
Showing 2 changed files with 644 additions and 0 deletions
sql/tools/.gitignore 0 → 100644
  View file @e08a103
  1 +# 忽略python虚拟环境
  2 +.env
  3 +.venv
  4 +env/
  5 +venv/
  6 +ENV/
  7 +env.bak/
  8 +venv.bak/
... ...
sql/tools/convertor.py 0 → 100644
  View file @e08a103
  1 +# encoding=utf8
  2 +"""芋道系统数据库迁移工具
  3 +
  4 +Author: dhb52 (https://gitee.com/dhb52)
  5 +
  6 +pip install simple-ddl-parser
  7 +"""
  8 +
  9 +import pathlib
  10 +import re
  11 +import time
  12 +from abc import ABC, abstractmethod
  13 +from typing import Dict, Tuple
  14 +
  15 +from simple_ddl_parser import DDLParser
  16 +
  17 +PREAMBLE = """/*
  18 + Yudao Database Transfer Tool
  19 +
  20 + Source Server Type : MySQL
  21 +
  22 + Target Server Type : {db_type}
  23 +
  24 + Date: {date}
  25 +*/
  26 +
  27 +"""
  28 +
  29 +
  30 +def load_and_clean(sql_file: str) -> str:
  31 + """加载源 SQL 文件，并清理内容方便下一步 ddl 解析
  32 +
  33 + Args:
  34 + sql_file (str): sql文件路径
  35 +
  36 + Returns:
  37 + str: 清理后的sql文件内容
  38 + """
  39 + REPLACE_PAIR_LIST = (
  40 + (" CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ", " "),
  41 + (" KEY `", " INDEX `"),
  42 + ("UNIQUE INDEX", "UNIQUE KEY"),
  43 + ("b'0'", "'0'"),
  44 + ("b'1'", "'1'"),
  45 + )
  46 +
  47 + content = open(sql_file).read()
  48 + for replace_pair in REPLACE_PAIR_LIST:
  49 + content = content.replace(*replace_pair)
  50 + content = re.sub(r"ENGINE.*COMMENT", "COMMENT", content)
  51 + content = re.sub(r"ENGINE.*;", ";", content)
  52 + return content
  53 +
  54 +
  55 +class Convertor(ABC):
  56 + def __init__(self, src: str, db_type) -> None:
  57 + self.src = src
  58 + self.db_type = db_type
  59 + self.content = load_and_clean(self.src)
  60 + self.table_script_list = re.findall(r"CREATE TABLE [^;]*;", self.content)
  61 +
  62 + @abstractmethod
  63 + def translate_type(self, type: str, size: None | int | Tuple[int]) -> str:
  64 + """字段类型转换
  65 +
  66 + Args:
  67 + type (str): 字段类型
  68 + size (None | int | Tuple[int]): 字段长度描述, 如varchar(255), decimal(10,2)
  69 +
  70 + Returns:
  71 + str: 类型定义
  72 + """
  73 + pass
  74 +
  75 + @abstractmethod
  76 + def gen_create(self, table_ddl: Dict) -> str:
  77 + """生成 create 脚本
  78 +
  79 + Args:
  80 + table_ddl (Dict): 表DDL
  81 +
  82 + Returns:
  83 + str: 生成脚本
  84 + """
  85 + pass
  86 +
  87 + @abstractmethod
  88 + def gen_pk(self, table_name: str) -> str:
  89 + """生成主键定义
  90 +
  91 + Args:
  92 + table_name (str): 表名
  93 +
  94 + Returns:
  95 + str: 生成脚本
  96 + """
  97 + pass
  98 +
  99 + @abstractmethod
  100 + def gen_index(self, table_ddl: Dict) -> str:
  101 + """生成索引定义
  102 +
  103 + Args:
  104 + table_ddl (Dict): 表DDL
  105 +
  106 + Returns:
  107 + str: 生成脚本
  108 + """
  109 + pass
  110 +
  111 + @abstractmethod
  112 + def gen_comment(self, table_sql: str, table_name: str) -> str:
  113 + """生成字段/表注释
  114 +
  115 + Args:
  116 + table_sql (str): 原始表SQL
  117 + table_name (str): 表名
  118 +
  119 + Returns:
  120 + str: 生成脚本
  121 + """
  122 + pass
  123 +
  124 + @abstractmethod
  125 + def gen_insert(self, table_name: str) -> str:
  126 + """生成 insert 语句块
  127 +
  128 + Args:
  129 + table_name (str): 表名
  130 +
  131 + Returns:
  132 + str: 生成脚本
  133 + """
  134 + pass
  135 +
  136 + def print(self):
  137 + """打印转换后的sql脚本到终端"""
  138 + print(
  139 + PREAMBLE.format(
  140 + db_type=self.db_type,
  141 + date=time.strftime("%Y-%m-%d %H:%M:%S"),
  142 + )
  143 + )
  144 +
  145 + error_scripts = []
  146 + for table_sql in self.table_script_list:
  147 + ddl = DDLParser(table_sql.replace("`", "")).run()
  148 +
  149 + # 如果parse失败, 需要跟进
  150 + if len(ddl) == 0:
  151 + error_scripts.append(table_sql)
  152 + continue
  153 +
  154 + table_ddl = ddl[0]
  155 + table_name = table_ddl["table_name"]
  156 +
  157 + # 忽略 quartz 的内容
  158 + if table_name.lower().startswith("qrtz"):
  159 + continue
  160 +
  161 + # 为每个表生成个5个基本部分
  162 + create = self.gen_create(table_ddl)
  163 + pk = self.gen_pk(table_name)
  164 + index = self.gen_index(table_ddl)
  165 + comment = self.gen_comment(table_sql, table_name)
  166 + inserts = self.gen_insert(table_name)
  167 +
  168 + # 组合当前表的DDL脚本
  169 + script = f"""{create}
  170 +
  171 +{pk}
  172 +
  173 +{index}
  174 +
  175 +{comment}
  176 +
  177 +{inserts}
  178 +"""
  179 +
  180 + # 清理
  181 + script = re.sub("\n{3,}", "\n\n", script).strip() + "\n"
  182 +
  183 + print(script)
  184 +
  185 + # 将parse失败的脚本打印出来
  186 + if error_scripts:
  187 + for script in error_scripts:
  188 + print(script)
  189 +
  190 +
  191 +class PostgreSQLConvertor(Convertor):
  192 + def __init__(self, src):
  193 + super().__init__(src, "PostgreSQL")
  194 +
  195 + def translate_type(self, type, size):
  196 + """类型转换"""
  197 +
  198 + type = type.lower()
  199 +
  200 + if type == "varchar":
  201 + return f"varchar({size})"
  202 + if type == "int":
  203 + return "int4"
  204 + if type == "bigint" or type == "bigint unsigned":
  205 + return "int8"
  206 + if type == "datetime":
  207 + return "timestamp"
  208 + if type == "bit":
  209 + return "bool"
  210 + if type in ("tinyint", "smallint"):
  211 + return "int2"
  212 + if type == "text":
  213 + return "text"
  214 + if type in ("blob", "mediumblob"):
  215 + return "bytea"
  216 + if type == "decimal":
  217 + return (
  218 + f"numeric({','.join(str(s) for s in size)})" if len(size) else "numeric"
  219 + )
  220 +
  221 + def gen_create(self, ddl: Dict) -> str:
  222 + """生成 create"""
  223 +
  224 + def _generate_column(col):
  225 + name = col["name"].lower()
  226 + if name == "deleted":
  227 + return "deleted int2 NOT NULL DEFAULT 0"
  228 +
  229 + type = col["type"].lower()
  230 + full_type = self.translate_type(type, col["size"])
  231 + nullable = "NULL" if col["nullable"] else "NOT NULL"
  232 + default = f"DEFAULT {col['default']}" if col["default"] is not None else ""
  233 + return f"{name} {full_type} {nullable} {default}"
  234 +
  235 + table_name = ddl["table_name"].lower()
  236 + columns = [f"{_generate_column(col).strip()}" for col in ddl["columns"]]
  237 + script = f"""-- ----------------------------
  238 +-- Table structure for {table_name}
  239 +-- ----------------------------
  240 +DROP TABLE IF EXISTS {table_name};
  241 +CREATE TABLE {table_name} (
  242 + {',\n '.join(columns)}
  243 +);"""
  244 +
  245 + return script
  246 +
  247 + def gen_comment(self, table_sql, table_name) -> str:
  248 + """生成字段及表的注释"""
  249 +
  250 + script = ""
  251 + for line in table_sql.split("\n"):
  252 + match = re.match(r"^`([^`]+)`.* COMMENT '([^']+)'", line.strip())
  253 + if match:
  254 + script += f"COMMENT ON COLUMN {table_name}.{match.group(1)} IS '{match.group(2).replace('\\n', '\n')}';\n"
  255 +
  256 + match = re.search(r"COMMENT \= '([^']+)';", table_sql)
  257 + table_comment = match.group(1) if match else None
  258 + if table_comment:
  259 + script += f"COMMENT ON TABLE {table_name} IS '{table_comment}';\n"
  260 +
  261 + return script
  262 +
  263 + def gen_pk(self, table_name) -> str:
  264 + """生成主键定义"""
  265 + return f"ALTER TABLE {table_name} ADD CONSTRAINT pk_{table_name} PRIMARY KEY (id);\n"
  266 +
  267 + def gen_index(self, ddl) -> str:
  268 + """生成 index"""
  269 +
  270 + def generate_columns(columns):
  271 + keys = [
  272 + f"{col['name'].lower()}{" " + col['order'].lower() if col['order'] != 'ASC' else ''}"
  273 + for col in columns[0]
  274 + ]
  275 + return ", ".join(keys)
  276 +
  277 + script = ""
  278 + for no, index in enumerate(ddl["index"], 1):
  279 + columns = generate_columns(index["columns"])
  280 + table_name = ddl["table_name"].lower()
  281 + script += (
  282 + f"CREATE INDEX idx_{table_name}_{no:02d} ON {table_name} ({columns});\n"
  283 + )
  284 +
  285 + return script
  286 +
  287 + def gen_insert(self, table_name) -> str:
  288 + """生成 insert 语句，以及根据最后的 insert id+1 生成 Sequence"""
  289 +
  290 + PREFIX = f"INSERT INTO `{table_name}`"
  291 +
  292 + # 收集 `table_name` 对应的 insert 语句
  293 + inserts = []
  294 + for line in self.content.split("\n"):
  295 + if line.startswith(PREFIX):
  296 + head, tail = line.replace(PREFIX, "").split(" VALUES ", maxsplit=1)
  297 + head = head.strip().replace("`", "").lower()
  298 + tail = tail.strip().replace(r"\"", '"')
  299 + script = f"INSERT INTO {table_name.lower()} {head} VALUES {tail}"
  300 + # bit(1)数据转换
  301 + script = script.replace("b'0'", "'0'").replace("b'1'", "'1'")
  302 + inserts.append(script)
  303 +
  304 + ## 生成 insert 脚本
  305 + script = ""
  306 + last_id = 0
  307 + if inserts:
  308 + script += f"""\n\n-- ----------------------------
  309 +-- Records of {table_name.lower()}
  310 +-- ----------------------------
  311 +-- @formatter:off
  312 +BEGIN;
  313 +{'\n'.join(inserts)}
  314 +COMMIT;
  315 +-- @formatter:on"""
  316 + match = re.search(r"VALUES \((\d+),", inserts[-1])
  317 + if match:
  318 + last_id = int(match.group(1))
  319 +
  320 + # 生成 Sequence
  321 + script += (
  322 + "\n\n"
  323 + + f"""DROP SEQUENCE IF EXISTS {table_name}_seq;
  324 +CREATE SEQUENCE {table_name}_seq
  325 + START {last_id + 1};"""
  326 + )
  327 +
  328 + return script
  329 +
  330 +
  331 +class OracleConvertor(Convertor):
  332 + def __init__(self, src):
  333 + super().__init__(src, "Oracle")
  334 +
  335 + def translate_type(self, type, size: None | int | Tuple[int]):
  336 + """类型转换"""
  337 + type = type.lower()
  338 +
  339 + if type == "varchar":
  340 + return f"varchar2({size if size < 4000 else 4000})"
  341 + if type == "int":
  342 + return "number"
  343 + if type == "bigint" or type == "bigint unsigned":
  344 + return "number"
  345 + if type == "datetime":
  346 + return "date"
  347 + if type == "bit":
  348 + return "number(1,0)"
  349 + if type in ("tinyint", "smallint"):
  350 + return "smallint"
  351 + if type == "text":
  352 + return "clob"
  353 + if type in ("blob", "mediumblob"):
  354 + return "blob"
  355 + if type == "decimal":
  356 + return (
  357 + f"number({','.join(str(s) for s in size)})" if len(size) else "number"
  358 + )
  359 +
  360 + def gen_create(self, ddl) -> str:
  361 + """生成 CREATE 语句"""
  362 +
  363 + def generate_column(col):
  364 + name = col["name"].lower()
  365 + if name == "deleted":
  366 + return "deleted number(1,0) DEFAULT 0 NOT NULL"
  367 +
  368 + type = col["type"].lower()
  369 + full_type = self.translate_type(type, col["size"])
  370 + nullable = "NULL" if col["nullable"] else "NOT NULL"
  371 + default = f"DEFAULT {col['default']}" if col["default"] is not None else ""
  372 + return f"{'\"size\"' if name == "size" else name } {full_type} {default} {nullable}"
  373 +
  374 + table_name = ddl["table_name"].lower()
  375 + columns = [f"{generate_column(col).strip()}" for col in ddl["columns"]]
  376 + script = f"""-- ----------------------------
  377 +-- Table structure for {table_name}
  378 +-- ----------------------------
  379 +CREATE TABLE {ddl['table_name'].lower()} (
  380 + {',\n '.join(columns)}
  381 +);"""
  382 +
  383 + # oracle INSERT '' 不能通过 NOT NULL 校验
  384 + script = script.replace("DEFAULT '' NOT NULL", "DEFAULT '' NULL")
  385 +
  386 + return script
  387 +
  388 + def gen_comment(self, table_sql, table_name) -> str:
  389 + script = ""
  390 + for line in table_sql.split("\n"):
  391 + match = re.search(r"`([^`]+)`.* COMMENT '([^']+)'", line)
  392 + if match:
  393 + script += f"COMMENT ON COLUMN {table_name}.{match.group(1)} IS '{match.group(2).replace('\\n', '\n')}';\n"
  394 +
  395 + match = re.search(r"COMMENT \= '([^']+)';", table_sql)
  396 + table_comment = match.group(1) if match else None
  397 + if table_comment:
  398 + script += f"COMMENT ON TABLE {table_name} IS '{table_comment}';"
  399 +
  400 + return script
  401 +
  402 + def gen_pk(self, table_name) -> str:
  403 + """生成主键定义"""
  404 + return f"ALTER TABLE {table_name} ADD CONSTRAINT pk_{table_name} PRIMARY KEY (id);\n"
  405 +
  406 + def gen_index(self, table_ddl) -> str:
  407 + """生成 INDEX 定义"""
  408 +
  409 + def generate_columns(columns):
  410 + keys = [
  411 + f"{col['name'].lower()}{" " + col['order'].lower() if col['order'] != 'ASC' else ''}"
  412 + for col in columns[0]
  413 + ]
  414 + return ", ".join(keys)
  415 +
  416 + script = ""
  417 + for no, index in enumerate(table_ddl["index"], 1):
  418 + columns = generate_columns(index["columns"])
  419 + table_name = table_ddl["table_name"].lower()
  420 + script += (
  421 + f"CREATE INDEX idx_{table_name}_{no:02d} ON {table_name} ({columns});\n"
  422 + )
  423 + return script
  424 +
  425 + def gen_insert(self, table_name) -> str:
  426 + """拷贝 INSERT 语句"""
  427 + PREFIX = f"INSERT INTO `{table_name}`"
  428 + inserts = []
  429 + for line in self.content.split("\n"):
  430 + if line.startswith(PREFIX):
  431 + head, tail = line.replace(PREFIX, "").split(" VALUES ", maxsplit=1)
  432 + head = head.strip().replace("`", "").lower()
  433 + tail = tail.strip().replace(r"\"", '"')
  434 + script = f"INSERT INTO {table_name.lower()} {head} VALUES {tail}"
  435 + # bit(1)数据转换
  436 + script = script.replace("b'0'", "'0'").replace("b'1'", "'1'")
  437 + # 对日期数据添加 TO_DATE 转换
  438 + script = re.sub(
  439 + r"('\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')",
  440 + r"to_date(\g<1>, 'SYYYY-MM-DD HH24:MI:SS')",
  441 + script,
  442 + )
  443 + inserts.append(script)
  444 +
  445 + ## 生成 insert 脚本
  446 + script = ""
  447 + last_id = 0
  448 + if inserts:
  449 + script += f"""\n\n-- ----------------------------
  450 +-- Records of {table_name.lower()}
  451 +-- ----------------------------
  452 +-- @formatter:off
  453 +{'\n'.join(inserts)}
  454 +COMMIT;
  455 +-- @formatter:on"""
  456 + match = re.search(r"VALUES \((\d+),", inserts[-1])
  457 + if match:
  458 + last_id = int(match.group(1))
  459 +
  460 + # 生成 Sequence
  461 + script += f"""
  462 +
  463 +CREATE SEQUENCE {table_name}_seq
  464 + START WITH {last_id + 1};"""
  465 +
  466 + return script
  467 +
  468 +
  469 +class SQLServerConvertor(Convertor):
  470 + """_summary_
  471 +
  472 + Args:
  473 + Convertor (_type_): _description_
  474 + """
  475 +
  476 + def __init__(self, src):
  477 + super().__init__(src, "Microsoft SQL Server")
  478 +
  479 + def translate_type(self, type, size):
  480 + """类型转换"""
  481 +
  482 + type = type.lower()
  483 +
  484 + if type == "varchar":
  485 + return f"nvarchar({size if size < 4000 else 4000})"
  486 + if type == "int":
  487 + return "int"
  488 + if type == "bigint" or type == "bigint unsigned":
  489 + return "bigint"
  490 + if type == "datetime":
  491 + return "datetime2"
  492 + if type == "bit":
  493 + return "varchar(1)"
  494 + if type in ("tinyint", "smallint"):
  495 + return "tinyint"
  496 + if type == "text":
  497 + return "nvarchar(max)"
  498 + if type in ("blob", "mediumblob"):
  499 + return "varbinary(max)"
  500 + if type == "decimal":
  501 + return (
  502 + f"numeric({','.join(str(s) for s in size)})" if len(size) else "numeric"
  503 + )
  504 +
  505 + def gen_create(self, ddl: Dict) -> str:
  506 + """生成 create"""
  507 +
  508 + def _generate_column(col):
  509 + name = col["name"].lower()
  510 + if name == 'id':
  511 + return "id bigint NOT NULL PRIMARY KEY IDENTITY"
  512 + if name == "deleted":
  513 + return "deleted bit DEFAULT 0 NOT NULL"
  514 +
  515 + type = col["type"].lower()
  516 + full_type = self.translate_type(type, col["size"])
  517 + nullable = "NULL" if col["nullable"] else "NOT NULL"
  518 + default = f"DEFAULT {col['default']}" if col["default"] is not None else ""
  519 + return f"{name} {full_type} {default} {nullable}"
  520 +
  521 + table_name = ddl["table_name"].lower()
  522 + columns = [f"{_generate_column(col).strip()}" for col in ddl["columns"]]
  523 + script = f"""-- ----------------------------
  524 +-- Table structure for {table_name}
  525 +-- ----------------------------
  526 +DROP TABLE IF EXISTS {table_name};
  527 +CREATE TABLE {table_name} (
  528 + {',\n '.join(columns)}
  529 +)
  530 +GO"""
  531 +
  532 + return script
  533 +
  534 + def gen_comment(self, table_sql, table_name) -> str:
  535 + """生成字段及表的注释"""
  536 +
  537 + script = ""
  538 + for line in table_sql.split("\n"):
  539 + match = re.match(r"^`([^`]+)`.* COMMENT '([^']+)'", line.strip())
  540 + if match:
  541 + script += f"""EXEC sp_addextendedproperty
  542 + 'MS_Description', N'{match.group(2).replace('\\n', '\n')}',
  543 + 'SCHEMA', N'dbo',
  544 + 'TABLE', N'{table_name}',
  545 + 'COLUMN', N'{match.group(1)}'
  546 +GO
  547 +
  548 +"""
  549 +
  550 + match = re.search(r"COMMENT \= '([^']+)';", table_sql)
  551 + table_comment = match.group(1) if match else None
  552 + if table_comment:
  553 + script += f"""EXEC sp_addextendedproperty
  554 + 'MS_Description', N'{table_comment}',
  555 + 'SCHEMA', N'dbo',
  556 + 'TABLE', N'{table_name}'
  557 +GO
  558 +
  559 +"""
  560 +
  561 + return script
  562 +
  563 + def gen_pk(self, table_name) -> str:
  564 + """生成主键定义"""
  565 + return ""
  566 +
  567 + def gen_index(self, ddl) -> str:
  568 + """生成 index"""
  569 +
  570 + def generate_columns(columns):
  571 + keys = [
  572 + f"{col['name'].lower()}{" " + col['order'].lower() if col['order'] != 'ASC' else ''}"
  573 + for col in columns[0]
  574 + ]
  575 + return ", ".join(keys)
  576 +
  577 + script = ""
  578 + for no, index in enumerate(ddl["index"], 1):
  579 + columns = generate_columns(index["columns"])
  580 + table_name = ddl["table_name"].lower()
  581 + script += f"CREATE INDEX idx_{table_name}_{no:02d} ON {table_name} ({columns})\nGO\n"
  582 +
  583 + return script
  584 +
  585 + def gen_insert(self, table_name) -> str:
  586 + """生成 insert 语句，以及根据最后的 insert id+1 生成 Sequence"""
  587 +
  588 + PREFIX = f"INSERT INTO `{table_name}`"
  589 +
  590 + # 收集 `table_name` 对应的 insert 语句
  591 + inserts = []
  592 + for line in self.content.split("\n"):
  593 + if line.startswith(PREFIX):
  594 + head, tail = line.replace(PREFIX, "").split(" VALUES ", maxsplit=1)
  595 + head = head.strip().replace("`", "").lower()
  596 + tail = tail.strip().replace(r"\"", '"')
  597 + # SQLServer: 字符串前加N，hack，是否存在替换字符串内容的风险
  598 + tail = tail.replace(", '", ", N'").replace("VALUES ('", "VALUES (N')")
  599 + script = f"INSERT INTO {table_name.lower()} {head} VALUES {tail}"
  600 + # bit(1)数据转换
  601 + script = script.replace("b'0'", "'0'").replace("b'1'", "'1'")
  602 + # 删除 insert 的结尾分号
  603 + script = re.sub(";$", r"\nGO", script)
  604 + inserts.append(script)
  605 +
  606 + ## 生成 insert 脚本
  607 + script = ""
  608 + if inserts:
  609 + script += f"""\n\n-- ----------------------------
  610 +-- Records of {table_name.lower()}
  611 +-- ----------------------------
  612 +-- @formatter:off
  613 +BEGIN TRANSACTION
  614 +GO
  615 +SET IDENTITY_INSERT {table_name.lower()} ON
  616 +GO
  617 +{'\n'.join(inserts)}
  618 +SET IDENTITY_INSERT {table_name.lower()} OFF
  619 +GO
  620 +COMMIT
  621 +GO
  622 +-- @formatter:on"""
  623 +
  624 + return script
  625 +
  626 +
  627 +def main():
  628 + sql_file = pathlib.Path('../mysql/ruoyi-vue-pro.sql').resolve().as_posix()
  629 + # convertor = PostgreSQLConvertor(sql_file)
  630 + # convertor = OracleConvertor(sql_file)
  631 + convertor = SQLServerConvertor(sql_file)
  632 + convertor.print()
  633 +
  634 +
  635 +if __name__ == "__main__":
  636 + main()
... ...