Commit 848d494824bf7dc7c8c96e59b244a735148559e9
1 parent
652d77c0
fix(sql): 修复建表语句扫描及 UTF-8 BOM 兼容问题
使用识别引号和注释的语句扫描,避免将 INSERT 数据误识别为建表语句。 保留缩进及同行多语句,按真实 DDL 提取注释,处理文件起始 BOM。 补充 14 个 Python 行为测试,独立扫描矩阵 12 项通过。 Refs #1274, #1273
Showing
2 changed files
with
384 additions
and
15 deletions
sql/tools/convertor.py
| @@ -23,7 +23,7 @@ import re | @@ -23,7 +23,7 @@ import re | ||
| 23 | import sys | 23 | import sys |
| 24 | import time | 24 | import time |
| 25 | from abc import ABC, abstractmethod | 25 | from abc import ABC, abstractmethod |
| 26 | -from typing import Dict, Generator, Optional, Tuple, Union | 26 | +from typing import Dict, Generator, List, Optional, Tuple, Union |
| 27 | 27 | ||
| 28 | from simple_ddl_parser import DDLParser | 28 | from simple_ddl_parser import DDLParser |
| 29 | 29 | ||
| @@ -43,6 +43,52 @@ PREAMBLE = """/* | @@ -43,6 +43,52 @@ PREAMBLE = """/* | ||
| 43 | """ | 43 | """ |
| 44 | 44 | ||
| 45 | 45 | ||
| 46 | +# 依次匹配:引号字符串(支持反斜杠转义、双写引号)、注释、语句分隔符;未闭合的引号吞掉剩余内容 | ||
| 47 | +SQL_TOKEN_PATTERN = re.compile( | ||
| 48 | + r"'(?:\\[\s\S]|''|[^'\\])*'" | ||
| 49 | + r'|"(?:\\[\s\S]|""|[^"\\])*"' | ||
| 50 | + r"|`(?:``|[^`])*`" | ||
| 51 | + r"|--(?=\s|\Z)[^\n]*|#[^\n]*|/\*[\s\S]*?(?:\*/|\Z)" | ||
| 52 | + r"|['\"`][\s\S]*" | ||
| 53 | + r"|;" | ||
| 54 | +) | ||
| 55 | +# 语句开头的空白、注释 | ||
| 56 | +SQL_LEADING_PATTERN = re.compile(r"(?:\s|--(?=\s|\Z)[^\n]*|#[^\n]*|/\*[\s\S]*?(?:\*/|\Z))*") | ||
| 57 | +CREATE_TABLE_PATTERN = re.compile(r"CREATE\s+TABLE\s", flags=re.IGNORECASE) | ||
| 58 | + | ||
| 59 | + | ||
| 60 | +def split_statements(content: str) -> List[str]: | ||
| 61 | + """按分号拆分 SQL 语句,忽略引号、注释内的分号 | ||
| 62 | + | ||
| 63 | + 仅用于定位顶层语句,不是完整的 SQL 解析器。返回的语句保留原文,去掉开头的空白、注释,以分号结尾 | ||
| 64 | + | ||
| 65 | + Args: | ||
| 66 | + content (str): sql 内容 | ||
| 67 | + | ||
| 68 | + Returns: | ||
| 69 | + List[str]: 语句列表 | ||
| 70 | + """ | ||
| 71 | + statements = [] | ||
| 72 | + start = 0 | ||
| 73 | + for match in SQL_TOKEN_PATTERN.finditer(content): | ||
| 74 | + if match.group() != ";": | ||
| 75 | + continue | ||
| 76 | + statement = content[start:match.end()] | ||
| 77 | + start = match.end() | ||
| 78 | + statement = statement[SQL_LEADING_PATTERN.match(statement).end():] | ||
| 79 | + if statement != ";": | ||
| 80 | + statements.append(statement) | ||
| 81 | + return statements | ||
| 82 | + | ||
| 83 | + | ||
| 84 | +def table_scripts(content: str) -> List[str]: | ||
| 85 | + """获得真实的 CREATE TABLE 语句,不包括 INSERT 数据、注释中的建表文本 | ||
| 86 | + | ||
| 87 | + 相关 issue:https://github.com/YunaiV/ruoyi-vue-pro/issues/1273 | ||
| 88 | + """ | ||
| 89 | + return [statement for statement in split_statements(content) if CREATE_TABLE_PATTERN.match(statement)] | ||
| 90 | + | ||
| 91 | + | ||
| 46 | def load_and_clean(sql_file: str) -> str: | 92 | def load_and_clean(sql_file: str) -> str: |
| 47 | """加载源 SQL 文件,并清理内容方便下一步 ddl 解析 | 93 | """加载源 SQL 文件,并清理内容方便下一步 ddl 解析 |
| 48 | 94 | ||
| @@ -52,6 +98,19 @@ def load_and_clean(sql_file: str) -> str: | @@ -52,6 +98,19 @@ def load_and_clean(sql_file: str) -> str: | ||
| 52 | Returns: | 98 | Returns: |
| 53 | str: 清理后的sql文件内容 | 99 | str: 清理后的sql文件内容 |
| 54 | """ | 100 | """ |
| 101 | + # utf-8-sig:只去掉文件开头的 BOM,避免首条语句无法识别 | ||
| 102 | + return clean(open(sql_file, encoding="utf-8-sig").read()) | ||
| 103 | + | ||
| 104 | + | ||
| 105 | +def clean(content: str) -> str: | ||
| 106 | + """清理 SQL 内容方便下一步 ddl 解析 | ||
| 107 | + | ||
| 108 | + Args: | ||
| 109 | + content (str): sql 内容 | ||
| 110 | + | ||
| 111 | + Returns: | ||
| 112 | + str: 清理后的sql内容 | ||
| 113 | + """ | ||
| 55 | REPLACE_PAIR_LIST = ( | 114 | REPLACE_PAIR_LIST = ( |
| 56 | (")\nVALUES ", ") VALUES "), | 115 | (")\nVALUES ", ") VALUES "), |
| 57 | (" CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ", " "), | 116 | (" CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci ", " "), |
| @@ -62,7 +121,6 @@ def load_and_clean(sql_file: str) -> str: | @@ -62,7 +121,6 @@ def load_and_clean(sql_file: str) -> str: | ||
| 62 | ("b'1'", "'1'"), | 121 | ("b'1'", "'1'"), |
| 63 | ) | 122 | ) |
| 64 | 123 | ||
| 65 | - content = open(sql_file, encoding="utf-8").read() | ||
| 66 | for replace_pair in REPLACE_PAIR_LIST: | 124 | for replace_pair in REPLACE_PAIR_LIST: |
| 67 | content = content.replace(*replace_pair) | 125 | content = content.replace(*replace_pair) |
| 68 | # 移除所有 CHARACTER SET / COLLATE 变体 (utf8mb3、utf8 等) | 126 | # 移除所有 CHARACTER SET / COLLATE 变体 (utf8mb3、utf8 等) |
| @@ -88,10 +146,14 @@ class Convertor(ABC): | @@ -88,10 +146,14 @@ class Convertor(ABC): | ||
| 88 | self.db_type = db_type | 146 | self.db_type = db_type |
| 89 | self.content = load_and_clean(self.src) | 147 | self.content = load_and_clean(self.src) |
| 90 | # original_content 保留原始 COMMENT 信息,用于注释提取 | 148 | # original_content 保留原始 COMMENT 信息,用于注释提取 |
| 91 | - self.original_content = open(src, encoding="utf-8").read() | ||
| 92 | - # 剥离列级 COMMENT 以避免 COMMENT 值内的分号截断 CREATE TABLE 正则 | ||
| 93 | - content_no_comment = re.sub(r" COMMENT '(?:[^'\\]|\\.)*'", "", self.content) | ||
| 94 | - self.table_script_list = re.findall(r"CREATE TABLE [^;]*;", content_no_comment) | 149 | + self.original_content = open(src, encoding="utf-8-sig").read() |
| 150 | + # 在原始内容上扫描建表语句,避免全局清理破坏引号边界;每个建表语句再单独清理 | ||
| 151 | + self.original_table_script_list = table_scripts(self.original_content) | ||
| 152 | + # 剥离列级 COMMENT,交给 DDLParser 解析 | ||
| 153 | + self.table_script_list = [ | ||
| 154 | + re.sub(r" COMMENT '(?:[^'\\]|\\.)*'", "", clean(table_sql)) | ||
| 155 | + for table_sql in self.original_table_script_list | ||
| 156 | + ] | ||
| 95 | 157 | ||
| 96 | @abstractmethod | 158 | @abstractmethod |
| 97 | def translate_type(self, type: str, size: Optional[Union[int, Tuple[int]]]) -> str: | 159 | def translate_type(self, type: str, size: Optional[Union[int, Tuple[int]]]) -> str: |
| @@ -330,7 +392,7 @@ class Convertor(ABC): | @@ -330,7 +392,7 @@ class Convertor(ABC): | ||
| 330 | ) | 392 | ) |
| 331 | 393 | ||
| 332 | error_scripts = [] | 394 | error_scripts = [] |
| 333 | - for table_sql in self.table_script_list: | 395 | + for table_sql, orig_table_sql in zip(self.table_script_list, self.original_table_script_list): |
| 334 | # 剥离 COMMENT 子句避免 DDLParser 无法处理中文等特殊字符 | 396 | # 剥离 COMMENT 子句避免 DDLParser 无法处理中文等特殊字符 |
| 335 | table_sql_for_parse = re.sub(r"\s+COMMENT\s+'[^']*'", "", table_sql) | 397 | table_sql_for_parse = re.sub(r"\s+COMMENT\s+'[^']*'", "", table_sql) |
| 336 | ddl = DDLParser(table_sql_for_parse.replace("`", "")).run() | 398 | ddl = DDLParser(table_sql_for_parse.replace("`", "")).run() |
| @@ -354,14 +416,7 @@ class Convertor(ABC): | @@ -354,14 +416,7 @@ class Convertor(ABC): | ||
| 354 | continue | 416 | continue |
| 355 | 417 | ||
| 356 | # 解析注释 | 418 | # 解析注释 |
| 357 | - # 从原始 SQL 提取注释(支持中文等 DDLParser 不能处理的内容) | ||
| 358 | - # 按表名定位完整建表段(以「换行 + )」起始的 ENGINE 行作为终止),避免被列 COMMENT 内的分号截断 | ||
| 359 | - orig_match = re.search( | ||
| 360 | - rf"CREATE TABLE\s+`{re.escape(table_name)}`[\s\S]*?\n\)[^;]*;", | ||
| 361 | - self.original_content, | ||
| 362 | - flags=re.IGNORECASE, | ||
| 363 | - ) | ||
| 364 | - orig_table_sql = orig_match.group() if orig_match else table_sql | 419 | + # 从原始建表语句提取注释(支持中文等 DDLParser 不能处理的内容),避免 INSERT 数据中的同名建表文本干扰 |
| 365 | comments_dict = dict(Convertor.filed_comments(orig_table_sql)) | 420 | comments_dict = dict(Convertor.filed_comments(orig_table_sql)) |
| 366 | for column in table_ddl["columns"]: | 421 | for column in table_ddl["columns"]: |
| 367 | column["comment"] = comments_dict.get(column["name"], "") | 422 | column["comment"] = comments_dict.get(column["name"], "") |
sql/tools/test_convertor.py
0 → 100644
| 1 | +# encoding=utf8 | ||
| 2 | +"""convertor.py 的单元测试 | ||
| 3 | + | ||
| 4 | +cd sql/tools | ||
| 5 | +uv run --with simple-ddl-parser python -m unittest test_convertor -v | ||
| 6 | +""" | ||
| 7 | + | ||
| 8 | +import contextlib | ||
| 9 | +import io | ||
| 10 | +import os | ||
| 11 | +import re | ||
| 12 | +import subprocess | ||
| 13 | +import sys | ||
| 14 | +import tempfile | ||
| 15 | +import unittest | ||
| 16 | + | ||
| 17 | +from convertor import ( | ||
| 18 | + DM8Convertor, | ||
| 19 | + HighGoConvertor, | ||
| 20 | + KingbaseConvertor, | ||
| 21 | + OpengaussConvertor, | ||
| 22 | + OracleConvertor, | ||
| 23 | + PostgreSQLConvertor, | ||
| 24 | + SQLServerConvertor, | ||
| 25 | + split_statements, | ||
| 26 | +) | ||
| 27 | + | ||
| 28 | +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) | ||
| 29 | +MYSQL_DIR = os.path.join(TOOLS_DIR, "..", "mysql") | ||
| 30 | + | ||
| 31 | +HEADER = """/* | ||
| 32 | + Navicat Premium Data Transfer | ||
| 33 | + | ||
| 34 | + Source Server Type : MySQL | ||
| 35 | +*/ | ||
| 36 | + | ||
| 37 | +SET NAMES utf8mb4; | ||
| 38 | +SET FOREIGN_KEY_CHECKS = 0; | ||
| 39 | +""" | ||
| 40 | + | ||
| 41 | +DEMO_TABLE = """ | ||
| 42 | +-- ---------------------------- | ||
| 43 | +-- Table structure for demo | ||
| 44 | +-- ---------------------------- | ||
| 45 | +DROP TABLE IF EXISTS `demo`; | ||
| 46 | +CREATE TABLE `demo` ( | ||
| 47 | + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '编号', | ||
| 48 | + `name` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '' COMMENT '名称;别名', | ||
| 49 | + PRIMARY KEY (`id`) USING BTREE | ||
| 50 | +) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci COMMENT = '演示表'; | ||
| 51 | + | ||
| 52 | +-- ---------------------------- | ||
| 53 | +-- Records of demo | ||
| 54 | +-- ---------------------------- | ||
| 55 | +BEGIN; | ||
| 56 | +INSERT INTO `demo` (`id`, `name`) VALUES (1, 'a;b'); | ||
| 57 | +COMMIT; | ||
| 58 | +""" | ||
| 59 | + | ||
| 60 | +DEMO_DOC_TABLE = """ | ||
| 61 | +DROP TABLE IF EXISTS `demo_doc`; | ||
| 62 | +CREATE TABLE `demo_doc` ( | ||
| 63 | + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '编号', | ||
| 64 | + `content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL COMMENT '内容', | ||
| 65 | + PRIMARY KEY (`id`) USING BTREE | ||
| 66 | +) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_unicode_ci COMMENT = '文档表'; | ||
| 67 | +""" | ||
| 68 | + | ||
| 69 | +# INSERT 数据中的建表文本:转义换行(issue #1273 的实际场景) | ||
| 70 | +ESCAPED_NEWLINE_INSERT = r""" | ||
| 71 | +INSERT INTO `demo_doc` (`id`, `content`) VALUES (1, 'Seata:\nCREATE TABLE `undo_log` (\n `id` bigint NOT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB;\n'); | ||
| 72 | +""" | ||
| 73 | + | ||
| 74 | +# INSERT 数据中的建表文本:真实换行 | ||
| 75 | +LITERAL_NEWLINE_INSERT = """ | ||
| 76 | +INSERT INTO `demo_doc` (`id`, `content`) VALUES (2, 'line1 | ||
| 77 | +CREATE TABLE `ghost` ( | ||
| 78 | + `id` bigint NOT NULL | ||
| 79 | +); | ||
| 80 | +line3'); | ||
| 81 | +""" | ||
| 82 | + | ||
| 83 | +# 注释中的建表文本 | ||
| 84 | +COMMENTED_DDL = """ | ||
| 85 | +/* CREATE TABLE `ghost_block` (`id` bigint NOT NULL); */ | ||
| 86 | +-- CREATE TABLE `ghost_line` (`id` bigint NOT NULL); | ||
| 87 | +# CREATE TABLE `ghost_hash` (`id` bigint NOT NULL); | ||
| 88 | +""" | ||
| 89 | + | ||
| 90 | + | ||
| 91 | +def table_names(table_script_list): | ||
| 92 | + return [re.match(r"CREATE\s+TABLE\s+`([^`]+)`", script).group(1) for script in table_script_list] | ||
| 93 | + | ||
| 94 | + | ||
| 95 | +def convert(convertor): | ||
| 96 | + """执行转换,返回 stdout、stderr 的内容""" | ||
| 97 | + out, err = io.StringIO(), io.StringIO() | ||
| 98 | + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): | ||
| 99 | + convertor.print() | ||
| 100 | + return out.getvalue(), err.getvalue() | ||
| 101 | + | ||
| 102 | + | ||
| 103 | +class SplitStatementsTest(unittest.TestCase): | ||
| 104 | + | ||
| 105 | + def test_semicolon_in_quotes(self): | ||
| 106 | + sql = r"""INSERT INTO `t` VALUES ('a;b', "c;d", 'it\'s;', 'x''y;', "q\";", 'C:\\'); | ||
| 107 | +INSERT INTO `we;ird` VALUES (1);""" | ||
| 108 | + self.assertEqual(sql.split("\n"), split_statements(sql)) | ||
| 109 | + | ||
| 110 | + def test_semicolon_in_comments(self): | ||
| 111 | + sql = """-- comment; here | ||
| 112 | +# hash; it's | ||
| 113 | +/* block; here */ SELECT 1; | ||
| 114 | +CREATE TABLE `c` ( | ||
| 115 | + `id` bigint -- 编号; 注释 | ||
| 116 | +);""" | ||
| 117 | + self.assertEqual(["SELECT 1;", "CREATE TABLE `c` (\n `id` bigint -- 编号; 注释\n);"], split_statements(sql)) | ||
| 118 | + | ||
| 119 | + def test_double_dash_without_space(self): | ||
| 120 | + # MySQL 中 -- 后面必须跟空白,才是注释 | ||
| 121 | + self.assertEqual(["SELECT 5--2;"], split_statements("SELECT 5--2;")) | ||
| 122 | + | ||
| 123 | + def test_multiple_statements_one_line(self): | ||
| 124 | + self.assertEqual(["SET NAMES utf8mb4;", "SET FOREIGN_KEY_CHECKS = 0;"], | ||
| 125 | + split_statements("SET NAMES utf8mb4; SET FOREIGN_KEY_CHECKS = 0;")) | ||
| 126 | + | ||
| 127 | + def test_unterminated(self): | ||
| 128 | + # 未闭合的引号、缺少分号的结尾,都不作为语句 | ||
| 129 | + self.assertEqual(["SELECT 1;"], split_statements("SELECT 1; INSERT INTO `t` VALUES ('a;")) | ||
| 130 | + self.assertEqual(["SELECT 1;"], split_statements("SELECT 1;\nSELECT 2")) | ||
| 131 | + | ||
| 132 | + | ||
| 133 | +class ConvertorTableScanTest(unittest.TestCase): | ||
| 134 | + | ||
| 135 | + def setUp(self): | ||
| 136 | + directory = tempfile.TemporaryDirectory() | ||
| 137 | + self.addCleanup(directory.cleanup) | ||
| 138 | + self.directory = directory.name | ||
| 139 | + | ||
| 140 | + def write_sql(self, content): | ||
| 141 | + path = os.path.join(self.directory, "dump.sql") | ||
| 142 | + with open(path, "w", encoding="utf-8") as f: | ||
| 143 | + f.write(content) | ||
| 144 | + return path | ||
| 145 | + | ||
| 146 | + def test_create_table_in_insert_ignored(self): | ||
| 147 | + path = self.write_sql(HEADER + DEMO_DOC_TABLE + ESCAPED_NEWLINE_INSERT + LITERAL_NEWLINE_INSERT | ||
| 148 | + + COMMENTED_DDL + DEMO_TABLE) | ||
| 149 | + for convertor_class in (PostgreSQLConvertor, KingbaseConvertor, OpengaussConvertor, HighGoConvertor, | ||
| 150 | + OracleConvertor, SQLServerConvertor, DM8Convertor): | ||
| 151 | + with self.subTest(convertor_class.__name__): | ||
| 152 | + convertor = convertor_class(path) | ||
| 153 | + self.assertEqual(["demo_doc", "demo"], table_names(convertor.table_script_list)) | ||
| 154 | + | ||
| 155 | + out, err = convert(convertor) | ||
| 156 | + self.assertNotIn("无法正常解析", err) | ||
| 157 | + self.assertIn("Table structure for demo_doc", out) | ||
| 158 | + self.assertIn("Table structure for demo\n", out) | ||
| 159 | + for ghost in ("undo_log", "ghost", "ghost_block", "ghost_line", "ghost_hash"): | ||
| 160 | + self.assertNotIn(f"Table structure for {ghost}\n", out) | ||
| 161 | + | ||
| 162 | + def test_comment_from_real_table(self): | ||
| 163 | + # 前面 INSERT 数据中存在同名的建表文本,不能影响真实建表语句的注释 | ||
| 164 | + fake_insert = """ | ||
| 165 | +INSERT INTO `demo_doc` (`id`, `content`) VALUES (1, 'CREATE TABLE `demo` ( | ||
| 166 | + `name` varchar(64) NOT NULL COMMENT \\'假名称\\', | ||
| 167 | + PRIMARY KEY (`id`) | ||
| 168 | +) ENGINE = InnoDB COMMENT = \\'假表\\'; | ||
| 169 | +'); | ||
| 170 | +""" | ||
| 171 | + path = self.write_sql(HEADER + DEMO_DOC_TABLE + fake_insert + DEMO_TABLE) | ||
| 172 | + convertor = PostgreSQLConvertor(path) | ||
| 173 | + self.assertEqual(["demo_doc", "demo"], table_names(convertor.table_script_list)) | ||
| 174 | + # 保留原始建表语句 | ||
| 175 | + self.assertIn("COMMENT '名称;别名'", convertor.original_table_script_list[1]) | ||
| 176 | + | ||
| 177 | + out, err = convert(convertor) | ||
| 178 | + self.assertNotIn("无法正常解析", err) | ||
| 179 | + self.assertIn("COMMENT ON COLUMN demo.id IS '编号';", out) | ||
| 180 | + self.assertIn("COMMENT ON COLUMN demo.name IS '名称;别名';", out) | ||
| 181 | + self.assertIn("COMMENT ON TABLE demo IS '演示表';", out) | ||
| 182 | + self.assertIn("COMMENT ON TABLE demo_doc IS '文档表';", out) | ||
| 183 | + comment_lines = [line for line in out.split("\n") if line.startswith("COMMENT ON")] | ||
| 184 | + self.assertFalse([line for line in comment_lines if "假" in line]) | ||
| 185 | + | ||
| 186 | + def test_indented_and_same_line_create_table(self): | ||
| 187 | + sql = HEADER + """ | ||
| 188 | + CREATE TABLE `indented` ( | ||
| 189 | + `id` bigint NOT NULL COMMENT '编号', | ||
| 190 | + PRIMARY KEY (`id`) USING BTREE | ||
| 191 | + ) ENGINE = InnoDB COMMENT = '缩进表'; | ||
| 192 | +CREATE TABLE `one` (`id` bigint NOT NULL, PRIMARY KEY (`id`)) ENGINE = InnoDB; CREATE TABLE `two` (`id` bigint NOT NULL, PRIMARY KEY (`id`)) ENGINE = InnoDB; | ||
| 193 | +""" | ||
| 194 | + convertor = PostgreSQLConvertor(self.write_sql(sql)) | ||
| 195 | + self.assertEqual(["indented", "one", "two"], table_names(convertor.table_script_list)) | ||
| 196 | + | ||
| 197 | + out, err = convert(convertor) | ||
| 198 | + self.assertNotIn("无法正常解析", err) | ||
| 199 | + self.assertIn("COMMENT ON COLUMN indented.id IS '编号';", out) | ||
| 200 | + self.assertIn("COMMENT ON TABLE indented IS '缩进表';", out) | ||
| 201 | + self.assertIn("CREATE TABLE one (", out) | ||
| 202 | + self.assertIn("CREATE TABLE two (", out) | ||
| 203 | + | ||
| 204 | + def test_multiple_tables_with_inserts(self): | ||
| 205 | + demo_item = """ | ||
| 206 | +DROP TABLE IF EXISTS `demo_item`; | ||
| 207 | +CREATE TABLE `demo_item` ( | ||
| 208 | + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '编号', | ||
| 209 | + `demo_id` bigint NOT NULL COMMENT '演示编号', | ||
| 210 | + PRIMARY KEY (`id`) USING BTREE | ||
| 211 | +) ENGINE = InnoDB COMMENT = '演示项'; | ||
| 212 | + | ||
| 213 | +BEGIN; | ||
| 214 | +INSERT INTO `demo_item` (`id`, `demo_id`) VALUES (7, 1); | ||
| 215 | +INSERT INTO `demo_item` (`id`, `demo_id`) VALUES (8, 1); | ||
| 216 | +COMMIT; | ||
| 217 | +""" | ||
| 218 | + convertor = PostgreSQLConvertor(self.write_sql(HEADER + DEMO_TABLE + demo_item)) | ||
| 219 | + out, err = convert(convertor) | ||
| 220 | + self.assertNotIn("无法正常解析", err) | ||
| 221 | + self.assertIn("INSERT INTO demo (id, name) VALUES (1, 'a;b');", out) | ||
| 222 | + self.assertIn("INSERT INTO demo_item (id, demo_id) VALUES (7, 1);\n" | ||
| 223 | + "INSERT INTO demo_item (id, demo_id) VALUES (8, 1);", out) | ||
| 224 | + self.assertIn("CREATE SEQUENCE demo_seq\n START 2;", out) | ||
| 225 | + self.assertIn("CREATE SEQUENCE demo_item_seq\n START 9;", out) | ||
| 226 | + | ||
| 227 | + def test_postgres_char(self): | ||
| 228 | + sql = HEADER + """ | ||
| 229 | +CREATE TABLE `demo_char` ( | ||
| 230 | + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '编号', | ||
| 231 | + `flag` char NOT NULL DEFAULT 'N' COMMENT '标记', | ||
| 232 | + `code` char(2) NULL DEFAULT NULL COMMENT '编码', | ||
| 233 | + PRIMARY KEY (`id`) USING BTREE | ||
| 234 | +) ENGINE = InnoDB COMMENT = '字符表'; | ||
| 235 | +""" | ||
| 236 | + out, err = convert(PostgreSQLConvertor(self.write_sql(sql))) | ||
| 237 | + self.assertNotIn("无法正常解析", err) | ||
| 238 | + self.assertIn("flag varchar(1) NOT NULL", out) | ||
| 239 | + self.assertIn("code varchar(2) NULL", out) | ||
| 240 | + | ||
| 241 | + def test_utf8_bom(self): | ||
| 242 | + # 文件开头的 BOM 紧挨着首条 CREATE TABLE;字符串中的 BOM 需原样保留 | ||
| 243 | + sql = "\ufeff" + """CREATE TABLE `demo_bom` ( | ||
| 244 | + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '编号', | ||
| 245 | + `name` varchar(64) NOT NULL DEFAULT '' COMMENT '名称', | ||
| 246 | + PRIMARY KEY (`id`) USING BTREE | ||
| 247 | +) ENGINE = InnoDB COMMENT = 'BOM 表'; | ||
| 248 | +INSERT INTO `demo_bom` (`id`, `name`) VALUES (1, 'a\ufeffb'); | ||
| 249 | +""" | ||
| 250 | + path = self.write_sql(sql) | ||
| 251 | + with open(path, "rb") as f: | ||
| 252 | + self.assertTrue(f.read().startswith(b"\xef\xbb\xbf")) | ||
| 253 | + | ||
| 254 | + convertor = PostgreSQLConvertor(path) | ||
| 255 | + self.assertEqual(["demo_bom"], table_names(convertor.table_script_list)) | ||
| 256 | + self.assertTrue(convertor.original_table_script_list[0].startswith("CREATE TABLE `demo_bom`")) | ||
| 257 | + | ||
| 258 | + out, err = convert(convertor) | ||
| 259 | + self.assertNotIn("无法正常解析", err) | ||
| 260 | + self.assertIn("CREATE TABLE demo_bom (", out) | ||
| 261 | + self.assertIn("name varchar(64) NOT NULL", out) | ||
| 262 | + self.assertIn("COMMENT ON COLUMN demo_bom.name IS '名称';", out) | ||
| 263 | + self.assertIn("COMMENT ON TABLE demo_bom IS 'BOM 表';", out) | ||
| 264 | + self.assertIn("INSERT INTO demo_bom (id, name) VALUES (1, 'a\ufeffb');", out) | ||
| 265 | + | ||
| 266 | + def test_cli(self): | ||
| 267 | + # 等价于:python convertor.py <type> dump.sql | ||
| 268 | + sql = HEADER + DEMO_DOC_TABLE + ESCAPED_NEWLINE_INSERT + LITERAL_NEWLINE_INSERT + DEMO_TABLE | ||
| 269 | + path = self.write_sql(sql) | ||
| 270 | + | ||
| 271 | + # OceanBase:原样输出 | ||
| 272 | + result = subprocess.run([sys.executable, "convertor.py", "oceanbase", path], | ||
| 273 | + cwd=TOOLS_DIR, capture_output=True) | ||
| 274 | + self.assertEqual(0, result.returncode, result.stderr.decode("utf-8")) | ||
| 275 | + self.assertEqual(sql, result.stdout.decode("utf-8")) | ||
| 276 | + | ||
| 277 | + # PostgreSQL:不误识别 INSERT 数据中的建表文本 | ||
| 278 | + result = subprocess.run([sys.executable, "convertor.py", "postgres", path], | ||
| 279 | + cwd=TOOLS_DIR, capture_output=True) | ||
| 280 | + out, err = result.stdout.decode("utf-8"), result.stderr.decode("utf-8") | ||
| 281 | + self.assertEqual(0, result.returncode, err) | ||
| 282 | + self.assertNotIn("无法正常解析", err) | ||
| 283 | + self.assertIn("Table structure for demo_doc", out) | ||
| 284 | + self.assertIn("Table structure for demo\n", out) | ||
| 285 | + self.assertNotIn("Table structure for undo_log", out) | ||
| 286 | + self.assertNotIn("Table structure for ghost", out) | ||
| 287 | + | ||
| 288 | + | ||
| 289 | +class ConvertorOfficialDumpTest(unittest.TestCase): | ||
| 290 | + | ||
| 291 | + def test_ruoyi_vue_pro(self): | ||
| 292 | + convertor = PostgreSQLConvertor(os.path.join(MYSQL_DIR, "ruoyi-vue-pro.sql")) | ||
| 293 | + names = table_names(convertor.table_script_list) | ||
| 294 | + self.assertEqual(48, len(names)) | ||
| 295 | + self.assertEqual(len(names), len(set(names))) | ||
| 296 | + | ||
| 297 | + out, err = convert(convertor) | ||
| 298 | + self.assertNotIn("无法正常解析", err) | ||
| 299 | + # 48 张表,加上 dual 表 | ||
| 300 | + self.assertEqual(49, out.count("-- Table structure for ")) | ||
| 301 | + | ||
| 302 | + def test_quartz(self): | ||
| 303 | + convertor = PostgreSQLConvertor(os.path.join(MYSQL_DIR, "quartz.sql")) | ||
| 304 | + names = table_names(convertor.table_script_list) | ||
| 305 | + self.assertEqual(11, len(names)) | ||
| 306 | + self.assertTrue(all(name.startswith("QRTZ_") for name in names)) | ||
| 307 | + | ||
| 308 | + # quartz 的表不转换 | ||
| 309 | + out, _ = convert(convertor) | ||
| 310 | + self.assertNotIn("qrtz", out.lower()) | ||
| 311 | + | ||
| 312 | + | ||
| 313 | +if __name__ == "__main__": | ||
| 314 | + unittest.main() |