BJ_ZTEITS-Developers / parking-cloud-yudao

Commit b4a867b89c8b4dfe7c3622cb3aace1ff770ba068

Authored by 芋道源码
Committed by Gitee
2 parents d98419b0 6d19690b

!963 feat: 脚本兼容Python 3.9, 抽取更多公共代码; 更新使用文档 

Merge pull request !963 from dhb52/feature/db
Inline Side-by-side
Showing 2 changed files with 157 additions and 151 deletions
sql/tools/README.md
  View file @b4a867b
... ... @@ -50,8 +50,16 @@ TODO 暂未支持
50 50  
51 51 使用方式如下：
52 52  
  53 +安装依赖库
  54 +
  55 +```bash
  56 +pip install simple-ddl-parser
  57 +```
  58 +
  59 +执行如下命令打印生成 postgresql 的脚本内容，其他可选参数有：oracle, sqlserver
  60 +
53 61 ```Bash
54   -python3 convertor.py
  62 +python3 convertor.py postgres
55 63 ```
56 64  
57   -然后，TODO
58 65 \ No newline at end of file
  66 +程序将sql脚本打印到终端，可以重定向到临时文件tmp.sql, 确认无误后可以利用IDEA（专业版）进行格式化。
... ...
sql/tools/convertor.py
  View file @b4a867b
... ... @@ -6,11 +6,12 @@ Author: dhb52 (https://gitee.com/dhb52)
6 6 pip install simple-ddl-parser
7 7 """
8 8  
  9 +import argparse
9 10 import pathlib
10 11 import re
11 12 import time
12 13 from abc import ABC, abstractmethod
13   -from typing import Dict, Tuple
  14 +from typing import Dict, Generator, Optional, Tuple, Union
14 15  
15 16 from simple_ddl_parser import DDLParser
16 17  
... ... @@ -60,12 +61,12 @@ class Convertor(ABC):
60 61 self.table_script_list = re.findall(r"CREATE TABLE [^;]*;", self.content)
61 62  
62 63 @abstractmethod
63   - def translate_type(self, type: str, size: None | int | Tuple[int]) -> str:
  64 + def translate_type(self, type: str, size: Optional[Union[int, Tuple[int]]]) -> str:
64 65 """字段类型转换
65 66  
66 67 Args:
67 68 type (str): 字段类型
68   - size (None | int | Tuple[int]): 字段长度描述, 如varchar(255), decimal(10,2)
  69 + size (Optional[Union[int, Tuple[int]]]): 字段长度描述, 如varchar(255), decimal(10,2)
69 70  
70 71 Returns:
71 72 str: 类型定义
... ... @@ -97,7 +98,7 @@ class Convertor(ABC):
97 98 pass
98 99  
99 100 @abstractmethod
100   - def gen_index(self, table_ddl: Dict) -> str:
  101 + def gen_index(self, ddl: Dict) -> str:
101 102 """生成索引定义
102 103  
103 104 Args:
... ... @@ -133,6 +134,55 @@ class Convertor(ABC):
133 134 """
134 135 pass
135 136  
  137 + @staticmethod
  138 + def inserts(table_name: str, script_content: str) -> Generator:
  139 + PREFIX = f"INSERT INTO `{table_name}`"
  140 +
  141 + # 收集 `table_name` 对应的 insert 语句
  142 + for line in script_content.split("\n"):
  143 + if line.startswith(PREFIX):
  144 + head, tail = line.replace(PREFIX, "").split(" VALUES ", maxsplit=1)
  145 + head = head.strip().replace("`", "").lower()
  146 + tail = tail.strip().replace(r"\"", '"')
  147 + # tail = tail.replace("b'0'", "'0'").replace("b'1'", "'1'")
  148 + yield f"INSERT INTO {table_name.lower()} {head} VALUES {tail}"
  149 +
  150 + @staticmethod
  151 + def index(ddl: Dict) -> Generator:
  152 + """生成索引定义
  153 +
  154 + Args:
  155 + ddl (Dict): 表DDL
  156 +
  157 + Yields:
  158 + Generator[str]: create index 语句
  159 + """
  160 +
  161 + def generate_columns(columns):
  162 + keys = [
  163 + f"{col['name'].lower()}{' ' + col['order'].lower() if col['order'] != 'ASC' else ''}"
  164 + for col in columns[0]
  165 + ]
  166 + return ", ".join(keys)
  167 +
  168 + for no, index in enumerate(ddl["index"], 1):
  169 + columns = generate_columns(index["columns"])
  170 + table_name = ddl["table_name"].lower()
  171 + yield f"CREATE INDEX idx_{table_name}_{no:02d} ON {table_name} ({columns})"
  172 +
  173 + @staticmethod
  174 + def filed_comments(table_sql: str) -> Generator:
  175 + for line in table_sql.split("\n"):
  176 + match = re.match(r"^`([^`]+)`.* COMMENT '([^']+)'", line.strip())
  177 + if match:
  178 + field = match.group(1)
  179 + comment_string = match.group(2).replace("\\n", "\n")
  180 + yield field, comment_string
  181 +
  182 + def table_comment(self, table_sql: str) -> str:
  183 + match = re.search(r"COMMENT \= '([^']+)';", table_sql)
  184 + return match.group(1) if match else None
  185 +
136 186 def print(self):
137 187 """打印转换后的sql脚本到终端"""
138 188 print(
... ... @@ -192,7 +242,7 @@ class PostgreSQLConvertor(Convertor):
192 242 def __init__(self, src):
193 243 super().__init__(src, "PostgreSQL")
194 244  
195   - def translate_type(self, type, size):
  245 + def translate_type(self, type: str, size: Optional[Union[int, Tuple[int]]]):
196 246 """类型转换"""
197 247  
198 248 type = type.lower()
... ... @@ -234,27 +284,30 @@ class PostgreSQLConvertor(Convertor):
234 284  
235 285 table_name = ddl["table_name"].lower()
236 286 columns = [f"{_generate_column(col).strip()}" for col in ddl["columns"]]
  287 + filed_def_list = ",\n ".join(columns)
237 288 script = f"""-- ----------------------------
238 289 -- Table structure for {table_name}
239 290 -- ----------------------------
240 291 DROP TABLE IF EXISTS {table_name};
241 292 CREATE TABLE {table_name} (
242   - {',\n '.join(columns)}
  293 + {filed_def_list}
243 294 );"""
244 295  
245 296 return script
246 297  
247   - def gen_comment(self, table_sql, table_name) -> str:
  298 + def gen_index(self, ddl: Dict) -> str:
  299 + return "\n".join(f"{script};" for script in self.index(ddl))
  300 +
  301 + def gen_comment(self, table_sql: str, table_name: str) -> str:
248 302 """生成字段及表的注释"""
249 303  
250 304 script = ""
251   - for line in table_sql.split("\n"):
252   - match = re.match(r"^`([^`]+)`.* COMMENT '([^']+)'", line.strip())
253   - if match:
254   - script += f"COMMENT ON COLUMN {table_name}.{match.group(1)} IS '{match.group(2).replace('\\n', '\n')}';\n"
  305 + for field, comment_string in self.filed_comments(table_sql):
  306 + script += (
  307 + f"COMMENT ON COLUMN {table_name}.{field} IS '{comment_string}';" + "\n"
  308 + )
255 309  
256   - match = re.search(r"COMMENT \= '([^']+)';", table_sql)
257   - table_comment = match.group(1) if match else None
  310 + table_comment = self.table_comment(table_sql)
258 311 if table_comment:
259 312 script += f"COMMENT ON TABLE {table_name} IS '{table_comment}';\n"
260 313  
... ... @@ -264,53 +317,21 @@ CREATE TABLE {table_name} (
264 317 """生成主键定义"""
265 318 return f"ALTER TABLE {table_name} ADD CONSTRAINT pk_{table_name} PRIMARY KEY (id);\n"
266 319  
267   - def gen_index(self, ddl) -> str:
268   - """生成 index"""
269   -
270   - def generate_columns(columns):
271   - keys = [
272   - f"{col['name'].lower()}{" " + col['order'].lower() if col['order'] != 'ASC' else ''}"
273   - for col in columns[0]
274   - ]
275   - return ", ".join(keys)
276   -
277   - script = ""
278   - for no, index in enumerate(ddl["index"], 1):
279   - columns = generate_columns(index["columns"])
280   - table_name = ddl["table_name"].lower()
281   - script += (
282   - f"CREATE INDEX idx_{table_name}_{no:02d} ON {table_name} ({columns});\n"
283   - )
284   -
285   - return script
286   -
287   - def gen_insert(self, table_name) -> str:
  320 + def gen_insert(self, table_name: str) -> str:
288 321 """生成 insert 语句，以及根据最后的 insert id+1 生成 Sequence"""
289 322  
290   - PREFIX = f"INSERT INTO `{table_name}`"
291   -
292   - # 收集 `table_name` 对应的 insert 语句
293   - inserts = []
294   - for line in self.content.split("\n"):
295   - if line.startswith(PREFIX):
296   - head, tail = line.replace(PREFIX, "").split(" VALUES ", maxsplit=1)
297   - head = head.strip().replace("`", "").lower()
298   - tail = tail.strip().replace(r"\"", '"')
299   - script = f"INSERT INTO {table_name.lower()} {head} VALUES {tail}"
300   - # bit(1)数据转换
301   - script = script.replace("b'0'", "'0'").replace("b'1'", "'1'")
302   - inserts.append(script)
303   -
  323 + inserts = list(Convertor.inserts(table_name, self.content))
304 324 ## 生成 insert 脚本
305 325 script = ""
306 326 last_id = 0
307 327 if inserts:
  328 + inserts_lines = "\n".join(inserts)
308 329 script += f"""\n\n-- ----------------------------
309 330 -- Records of {table_name.lower()}
310 331 -- ----------------------------
311 332 -- @formatter:off
312 333 BEGIN;
313   -{'\n'.join(inserts)}
  334 +{inserts_lines}
314 335 COMMIT;
315 336 -- @formatter:on"""
316 337 match = re.search(r"VALUES \((\d+),", inserts[-1])
... ... @@ -332,7 +353,7 @@ class OracleConvertor(Convertor):
332 353 def __init__(self, src):
333 354 super().__init__(src, "Oracle")
334 355  
335   - def translate_type(self, type, size: None | int | Tuple[int]):
  356 + def translate_type(self, type: str, size: Optional[Union[int, Tuple[int]]]):
336 357 """类型转换"""
337 358 type = type.lower()
338 359  
... ... @@ -369,15 +390,19 @@ class OracleConvertor(Convertor):
369 390 full_type = self.translate_type(type, col["size"])
370 391 nullable = "NULL" if col["nullable"] else "NOT NULL"
371 392 default = f"DEFAULT {col['default']}" if col["default"] is not None else ""
372   - return f"{'\"size\"' if name == "size" else name } {full_type} {default} {nullable}"
  393 + # Oracle 中 size 不能作为字段名
  394 + field_name = '"size"' if name == "size" else name
  395 + # Oracle DEFAULT 定义在 NULLABLE 之前
  396 + return f"{field_name} {full_type} {default} {nullable}"
373 397  
374 398 table_name = ddl["table_name"].lower()
375 399 columns = [f"{generate_column(col).strip()}" for col in ddl["columns"]]
  400 + field_def_list = ",\n ".join(columns)
376 401 script = f"""-- ----------------------------
377 402 -- Table structure for {table_name}
378 403 -- ----------------------------
379   -CREATE TABLE {ddl['table_name'].lower()} (
380   - {',\n '.join(columns)}
  404 +CREATE TABLE {table_name} (
  405 + {field_def_list}
381 406 );"""
382 407  
383 408 # oracle INSERT '' 不能通过 NOT NULL 校验
... ... @@ -385,72 +410,51 @@ CREATE TABLE {ddl['table_name'].lower()} (
385 410  
386 411 return script
387 412  
388   - def gen_comment(self, table_sql, table_name) -> str:
  413 + def gen_index(self, ddl: Dict) -> str:
  414 + return "\n".join(f"{script};" for script in self.index(ddl))
  415 +
  416 + def gen_comment(self, table_sql: str, table_name: str) -> str:
389 417 script = ""
390   - for line in table_sql.split("\n"):
391   - match = re.search(r"`([^`]+)`.* COMMENT '([^']+)'", line)
392   - if match:
393   - script += f"COMMENT ON COLUMN {table_name}.{match.group(1)} IS '{match.group(2).replace('\\n', '\n')}';\n"
  418 + for field, comment_string in self.filed_comments(table_sql):
  419 + script += (
  420 + f"COMMENT ON COLUMN {table_name}.{field} IS '{comment_string}';" + "\n"
  421 + )
394 422  
395   - match = re.search(r"COMMENT \= '([^']+)';", table_sql)
396   - table_comment = match.group(1) if match else None
  423 + table_comment = self.table_comment(table_sql)
397 424 if table_comment:
398   - script += f"COMMENT ON TABLE {table_name} IS '{table_comment}';"
  425 + script += f"COMMENT ON TABLE {table_name} IS '{table_comment}';\n"
399 426  
400 427 return script
401 428  
402   - def gen_pk(self, table_name) -> str:
  429 + def gen_pk(self, table_name: str) -> str:
403 430 """生成主键定义"""
404 431 return f"ALTER TABLE {table_name} ADD CONSTRAINT pk_{table_name} PRIMARY KEY (id);\n"
405 432  
406   - def gen_index(self, table_ddl) -> str:
407   - """生成 INDEX 定义"""
408   -
409   - def generate_columns(columns):
410   - keys = [
411   - f"{col['name'].lower()}{" " + col['order'].lower() if col['order'] != 'ASC' else ''}"
412   - for col in columns[0]
413   - ]
414   - return ", ".join(keys)
415   -
416   - script = ""
417   - for no, index in enumerate(table_ddl["index"], 1):
418   - columns = generate_columns(index["columns"])
419   - table_name = table_ddl["table_name"].lower()
420   - script += (
421   - f"CREATE INDEX idx_{table_name}_{no:02d} ON {table_name} ({columns});\n"
422   - )
423   - return script
  433 + def gen_index(self, ddl: Dict) -> str:
  434 + return "\n".join(f"{script};" for script in self.index(ddl))
424 435  
425   - def gen_insert(self, table_name) -> str:
  436 + def gen_insert(self, table_name: str) -> str:
426 437 """拷贝 INSERT 语句"""
427   - PREFIX = f"INSERT INTO `{table_name}`"
428 438 inserts = []
429   - for line in self.content.split("\n"):
430   - if line.startswith(PREFIX):
431   - head, tail = line.replace(PREFIX, "").split(" VALUES ", maxsplit=1)
432   - head = head.strip().replace("`", "").lower()
433   - tail = tail.strip().replace(r"\"", '"')
434   - script = f"INSERT INTO {table_name.lower()} {head} VALUES {tail}"
435   - # bit(1)数据转换
436   - script = script.replace("b'0'", "'0'").replace("b'1'", "'1'")
437   - # 对日期数据添加 TO_DATE 转换
438   - script = re.sub(
439   - r"('\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')",
440   - r"to_date(\g<1>, 'SYYYY-MM-DD HH24:MI:SS')",
441   - script,
442   - )
443   - inserts.append(script)
  439 + for insert_script in Convertor.inserts(table_name, self.content):
  440 + # 对日期数据添加 TO_DATE 转换
  441 + insert_script = re.sub(
  442 + r"('\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}')",
  443 + r"to_date(\g<1>, 'SYYYY-MM-DD HH24:MI:SS')",
  444 + insert_script,
  445 + )
  446 + inserts.append(insert_script)
444 447  
445 448 ## 生成 insert 脚本
446 449 script = ""
447 450 last_id = 0
448 451 if inserts:
  452 + inserts_lines = "\n".join(inserts)
449 453 script += f"""\n\n-- ----------------------------
450 454 -- Records of {table_name.lower()}
451 455 -- ----------------------------
452 456 -- @formatter:off
453   -{'\n'.join(inserts)}
  457 +{inserts_lines}
454 458 COMMIT;
455 459 -- @formatter:on"""
456 460 match = re.search(r"VALUES \((\d+),", inserts[-1])
... ... @@ -476,7 +480,7 @@ class SQLServerConvertor(Convertor):
476 480 def __init__(self, src):
477 481 super().__init__(src, "Microsoft SQL Server")
478 482  
479   - def translate_type(self, type, size):
  483 + def translate_type(self, type: str, size: Optional[Union[int, Tuple[int]]]):
480 484 """类型转换"""
481 485  
482 486 type = type.lower()
... ... @@ -507,7 +511,7 @@ class SQLServerConvertor(Convertor):
507 511  
508 512 def _generate_column(col):
509 513 name = col["name"].lower()
510   - if name == 'id':
  514 + if name == "id":
511 515 return "id bigint NOT NULL PRIMARY KEY IDENTITY"
512 516 if name == "deleted":
513 517 return "deleted bit DEFAULT 0 NOT NULL"
... ... @@ -520,35 +524,34 @@ class SQLServerConvertor(Convertor):
520 524  
521 525 table_name = ddl["table_name"].lower()
522 526 columns = [f"{_generate_column(col).strip()}" for col in ddl["columns"]]
  527 + filed_def_list = ",\n ".join(columns)
523 528 script = f"""-- ----------------------------
524 529 -- Table structure for {table_name}
525 530 -- ----------------------------
526 531 DROP TABLE IF EXISTS {table_name};
527 532 CREATE TABLE {table_name} (
528   - {',\n '.join(columns)}
  533 + {filed_def_list}
529 534 )
530 535 GO"""
531 536  
532 537 return script
533 538  
534   - def gen_comment(self, table_sql, table_name) -> str:
  539 + def gen_comment(self, table_sql: str, table_name: str) -> str:
535 540 """生成字段及表的注释"""
536 541  
537 542 script = ""
538   - for line in table_sql.split("\n"):
539   - match = re.match(r"^`([^`]+)`.* COMMENT '([^']+)'", line.strip())
540   - if match:
541   - script += f"""EXEC sp_addextendedproperty
542   - 'MS_Description', N'{match.group(2).replace('\\n', '\n')}',
  543 +
  544 + for field, comment_string in self.filed_comments(table_sql):
  545 + script += f"""EXEC sp_addextendedproperty
  546 + 'MS_Description', N'{comment_string}',
543 547 'SCHEMA', N'dbo',
544 548 'TABLE', N'{table_name}',
545   - 'COLUMN', N'{match.group(1)}'
  549 + 'COLUMN', N'{field}'
546 550 GO
547 551  
548 552 """
549 553  
550   - match = re.search(r"COMMENT \= '([^']+)';", table_sql)
551   - table_comment = match.group(1) if match else None
  554 + table_comment = self.table_comment(table_sql)
552 555 if table_comment:
553 556 script += f"""EXEC sp_addextendedproperty
554 557 'MS_Description', N'{table_comment}',
... ... @@ -557,55 +560,34 @@ GO
557 560 GO
558 561  
559 562 """
560   -
561 563 return script
562 564  
563   - def gen_pk(self, table_name) -> str:
  565 + def gen_pk(self, table_name: str) -> str:
564 566 """生成主键定义"""
565 567 return ""
566 568  
567   - def gen_index(self, ddl) -> str:
  569 + def gen_index(self, ddl: Dict) -> str:
568 570 """生成 index"""
  571 + return "\n".join(f"{script}\nGO" for script in self.index(ddl))
569 572  
570   - def generate_columns(columns):
571   - keys = [
572   - f"{col['name'].lower()}{" " + col['order'].lower() if col['order'] != 'ASC' else ''}"
573   - for col in columns[0]
574   - ]
575   - return ", ".join(keys)
576   -
577   - script = ""
578   - for no, index in enumerate(ddl["index"], 1):
579   - columns = generate_columns(index["columns"])
580   - table_name = ddl["table_name"].lower()
581   - script += f"CREATE INDEX idx_{table_name}_{no:02d} ON {table_name} ({columns})\nGO\n"
582   -
583   - return script
584   -
585   - def gen_insert(self, table_name) -> str:
  573 + def gen_insert(self, table_name: str) -> str:
586 574 """生成 insert 语句，以及根据最后的 insert id+1 生成 Sequence"""
587 575  
588   - PREFIX = f"INSERT INTO `{table_name}`"
589   -
590 576 # 收集 `table_name` 对应的 insert 语句
591 577 inserts = []
592   - for line in self.content.split("\n"):
593   - if line.startswith(PREFIX):
594   - head, tail = line.replace(PREFIX, "").split(" VALUES ", maxsplit=1)
595   - head = head.strip().replace("`", "").lower()
596   - tail = tail.strip().replace(r"\"", '"')
597   - # SQLServer: 字符串前加N，hack，是否存在替换字符串内容的风险
598   - tail = tail.replace(", '", ", N'").replace("VALUES ('", "VALUES (N')")
599   - script = f"INSERT INTO {table_name.lower()} {head} VALUES {tail}"
600   - # bit(1)数据转换
601   - script = script.replace("b'0'", "'0'").replace("b'1'", "'1'")
602   - # 删除 insert 的结尾分号
603   - script = re.sub(";$", r"\nGO", script)
604   - inserts.append(script)
  578 + for insert_script in Convertor.inserts(table_name, self.content):
  579 + # SQLServer: 字符串前加N，hack，是否存在替换字符串内容的风险
  580 + insert_script = insert_script.replace(", '", ", N'").replace(
  581 + "VALUES ('", "VALUES (N')"
  582 + )
  583 + # 删除 insert 的结尾分号
  584 + insert_script = re.sub(";$", r"\nGO", insert_script)
  585 + inserts.append(insert_script)
605 586  
606 587 ## 生成 insert 脚本
607 588 script = ""
608 589 if inserts:
  590 + inserts_lines = "\n".join(inserts)
609 591 script += f"""\n\n-- ----------------------------
610 592 -- Records of {table_name.lower()}
611 593 -- ----------------------------
... ... @@ -614,7 +596,7 @@ BEGIN TRANSACTION
614 596 GO
615 597 SET IDENTITY_INSERT {table_name.lower()} ON
616 598 GO
617   -{'\n'.join(inserts)}
  599 +{inserts_lines}
618 600 SET IDENTITY_INSERT {table_name.lower()} OFF
619 601 GO
620 602 COMMIT
... ... @@ -625,10 +607,26 @@ GO
625 607  
626 608  
627 609 def main():
628   - sql_file = pathlib.Path('../mysql/ruoyi-vue-pro.sql').resolve().as_posix()
629   - # convertor = PostgreSQLConvertor(sql_file)
630   - # convertor = OracleConvertor(sql_file)
631   - convertor = SQLServerConvertor(sql_file)
  610 + parser = argparse.ArgumentParser(description="芋道系统数据库转换工具")
  611 + parser.add_argument(
  612 + "type",
  613 + type=str,
  614 + help="目标数据库类型",
  615 + choices=["postgres", "oracle", "sqlserver"],
  616 + )
  617 + args = parser.parse_args()
  618 +
  619 + sql_file = pathlib.Path("../mysql/ruoyi-vue-pro.sql").resolve().as_posix()
  620 + convertor = None
  621 + if args.type == "postgres":
  622 + convertor = PostgreSQLConvertor(sql_file)
  623 + elif args.type == "oracle":
  624 + convertor = OracleConvertor(sql_file)
  625 + elif args.type == "sqlserver":
  626 + convertor = SQLServerConvertor(sql_file)
  627 + else:
  628 + raise NotImplementedError(f"不支持目标数据库类型: {args.type}")
  629 +
632 630 convertor.print()
633 631  
634 632  
... ...