BJ_ZTEITS-Developers / parking-cloud-yudao

Commit 944e3862a5bac58b8d453280d258bfd71895a541

Authored by daiwencheng
1 parent 82fd800a

修复敏感词检验问题 

1.修复排序，优先使用较短的前缀失效问题
2.修复敏感词为单个字符校验失败问题
Inline Side-by-side
Showing 4 changed files with 58 additions and 16 deletions
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordService.java
  View file @944e386
... ... @@ -88,11 +88,11 @@ public interface SensitiveWordService {
88 88 List<String> validateText(String text, List<String> tags);
89 89  
90 90 /**
91   - * 判断文本是否包含敏感词
  91 + * 判断文本是否合法
92 92 *
93 93 * @param text 文本
94   - * @param tags 表述数组
95   - * @return 是否包含
  94 + * @param tags 标签数组
  95 + * @return 是否合法 true-合法 false-不合法
96 96 */
97 97 boolean isTextValid(String text, List<String> tags);
98 98  
... ...
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordServiceImpl.java
  View file @944e386
... ... @@ -258,9 +258,7 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
258 258 if (trie == null) {
259 259 continue;
260 260 }
261   - if (!trie.isValid(text)) {
262   - return false;
263   - }
  261 + return trie.isValid(text);
264 262 }
265 263 return true;
266 264 }
... ...
yudao-module-system/yudao-module-system-biz/src/main/java/cn/iocoder/yudao/module/system/util/collection/SimpleTrie.java
  View file @944e386
... ... @@ -32,7 +32,7 @@ public class SimpleTrie {
32 32 public SimpleTrie(Collection<String> strs) {
33 33 children = new HashMap<>();
34 34 // 构建树
35   - CollUtil.sort(strs, String::compareTo); // 排序，优先使用较短的前缀
  35 + strs = CollUtil.sort(strs, String::compareTo); // 排序，优先使用较短的前缀
36 36 for (String str : strs) {
37 37 Map<Character, Object> child = children;
38 38 // 遍历每个字符
... ... @@ -56,11 +56,11 @@ public class SimpleTrie {
56 56 * 验证文本是否合法，即不包含敏感词
57 57 *
58 58 * @param text 文本
59   - * @return 是否 ok
  59 + * @return 是否 true-合法 false-不合法
60 60 */
61 61 public boolean isValid(String text) {
62 62 // 遍历 text，使用每一个 [i, n) 段的字符串，使用 children 前缀树匹配，是否包含敏感词
63   - for (int i = 0; i < text.length() - 1; i++) {
  63 + for (int i = 0; i < text.length() ; i++) {
64 64 Map<Character, Object> child = (Map<Character, Object>) children.get(text.charAt(i));
65 65 if (child == null) {
66 66 continue;
... ... @@ -74,14 +74,17 @@ public class SimpleTrie {
74 74 }
75 75  
76 76 /**
77   - * 验证文本从指定位置开始，是否包含某个敏感词
  77 + * 验证文本从指定位置开始，是否不包含某个敏感词
78 78 *
79 79 * @param text 文本
80 80 * @param index 开始位置
81 81 * @param child 节点（当前遍历到的）
82   - * @return 是否包含
  82 + * @return 是否不包含 true-不包含 false-包含
83 83 */
84 84 private boolean recursion(String text, int index, Map<Character, Object> child) {
  85 + if (child.containsKey(CHARACTER_END)) {
  86 + return false;
  87 + }
85 88 if (index == text.length()) {
86 89 return true;
87 90 }
... ... @@ -99,7 +102,7 @@ public class SimpleTrie {
99 102 */
100 103 public List<String> validate(String text) {
101 104 Set<String> results = new HashSet<>();
102   - for (int i = 0; i < text.length() - 1; i++) {
  105 + for (int i = 0; i < text.length(); i++) {
103 106 Character c = text.charAt(i);
104 107 Map<Character, Object> child = (Map<Character, Object>) children.get(c);
105 108 if (child == null) {
... ... @@ -123,10 +126,13 @@ public class SimpleTrie {
123 126 * @param index 开始未知
124 127 * @param child 节点（当前遍历到的）
125 128 * @param result 返回敏感词
126   - * @return 是否有敏感词
  129 + * @return 是否无敏感词 true-无 false-有
127 130 */
128 131 @SuppressWarnings("unchecked")
129 132 private static boolean recursionWithResult(String text, int index, Map<Character, Object> child, StringBuilder result) {
  133 + if (child.containsKey(CHARACTER_END)) {
  134 + return false;
  135 + }
130 136 if (index == text.length()) {
131 137 return true;
132 138 }
... ...
yudao-module-system/yudao-module-system-biz/src/test/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordServiceImplTest.java
  View file @944e386
... ... @@ -56,20 +56,28 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
56 56 SensitiveWordDO wordDO2 = randomPojo(SensitiveWordDO.class, o -> o.setName("笨蛋")
57 57 .setTags(singletonList("蔬菜")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
58 58 sensitiveWordMapper.insert(wordDO2);
  59 + SensitiveWordDO wordDO3 = randomPojo(SensitiveWordDO.class, o -> o.setName("白")
  60 + .setTags(singletonList("测试")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
  61 + sensitiveWordMapper.insert(wordDO3);
  62 + SensitiveWordDO wordDO4 = randomPojo(SensitiveWordDO.class, o -> o.setName("白痴")
  63 + .setTags(singletonList("测试")).setStatus(CommonStatusEnum.ENABLE.getStatus()));
  64 + sensitiveWordMapper.insert(wordDO4);
59 65  
60 66 // 调用
61 67 sensitiveWordService.initLocalCache();
62 68 // 断言 sensitiveWordTagsCache 缓存
63   - assertEquals(SetUtils.asSet("论坛", "蔬菜"), sensitiveWordService.getSensitiveWordTagSet());
  69 + assertEquals(SetUtils.asSet("论坛", "蔬菜","测试"), sensitiveWordService.getSensitiveWordTagSet());
64 70 // 断言 sensitiveWordCache
65   - assertEquals(2, sensitiveWordService.getSensitiveWordCache().size());
  71 + assertEquals(4, sensitiveWordService.getSensitiveWordCache().size());
66 72 assertPojoEquals(wordDO1, sensitiveWordService.getSensitiveWordCache().get(0));
67 73 assertPojoEquals(wordDO2, sensitiveWordService.getSensitiveWordCache().get(1));
  74 + assertPojoEquals(wordDO3, sensitiveWordService.getSensitiveWordCache().get(2));
68 75 // 断言 tagSensitiveWordTries 缓存
69 76 assertNotNull(sensitiveWordService.getDefaultSensitiveWordTrie());
70   - assertEquals(2, sensitiveWordService.getTagSensitiveWordTries().size());
  77 + assertEquals(3, sensitiveWordService.getTagSensitiveWordTries().size());
71 78 assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("论坛"));
72 79 assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("蔬菜"));
  80 + assertNotNull(sensitiveWordService.getTagSensitiveWordTries().get("测试"));
73 81 }
74 82  
75 83 @Test
... ... @@ -236,6 +244,14 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
236 244 List<String> result = sensitiveWordService.validateText(text, null);
237 245 // 断言
238 246 assertEquals(Arrays.asList("傻瓜", "笨蛋"), result);
  247 +
  248 + // 准备参数
  249 + String text2 = "你是傻瓜，你是笨蛋，你是白";
  250 +
  251 + // 调用
  252 + List<String> result2 = sensitiveWordService.validateText(text2, null);
  253 + // 断言
  254 + assertEquals(Arrays.asList("傻瓜", "笨蛋","白"), result2);
239 255 }
240 256  
241 257 @Test
... ... @@ -248,6 +264,16 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
248 264 List<String> result = sensitiveWordService.validateText(text, singletonList("论坛"));
249 265 // 断言
250 266 assertEquals(singletonList("傻瓜"), result);
  267 +
  268 +
  269 + // 准备参数
  270 + String text2 = "你是白";
  271 +
  272 + // 调用
  273 + List<String> result2 = sensitiveWordService.validateText(text2, singletonList("测试"));
  274 + // 断言
  275 + assertEquals(singletonList("白"), result2);
  276 +
251 277 }
252 278  
253 279 @Test
... ... @@ -258,6 +284,12 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
258 284  
259 285 // 调用，断言
260 286 assertFalse(sensitiveWordService.isTextValid(text, null));
  287 +
  288 + // 准备参数
  289 + String text2 = "你是白";
  290 +
  291 + // 调用，断言
  292 + assertFalse(sensitiveWordService.isTextValid(text2, null));
261 293 }
262 294  
263 295 @Test
... ... @@ -268,6 +300,12 @@ public class SensitiveWordServiceImplTest extends BaseDbUnitTest {
268 300  
269 301 // 调用，断言
270 302 assertFalse(sensitiveWordService.isTextValid(text, singletonList("论坛")));
  303 +
  304 + // 准备参数
  305 + String text2 = "你是白";
  306 +
  307 + // 调用，断言
  308 + assertFalse(sensitiveWordService.isTextValid(text2, singletonList("测试")));
271 309 }
272 310  
273 311 }
... ...