hhyykk/ipms-sjy
1
1
Fork 0
mirror of https://gitee.com/hhyykk/ipms-sjy.git synced 2025-07-25 00:15:06 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

实现 SensitiveWord API 实现类

Browse Source
This commit is contained in:
YunaiV
2022-04-09 10:48:47 +08:00
parent 3f7d7c3bfa
commit 696756b3c8
16 changed files with 9171 additions and 8253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
yudao-module-system/yudao-module-system-api/src/main/java/cn/iocoder/yudao/module/system/api/sensitiveword/SensitiveWordApi.java
View File

@ -1,16 +1,30 @@
package cn.iocoder.yudao.module.system.api.sensitiveword;
import java.util.List;
import java.util.Set;
/**
* 大佬，别偷懒：代码千万行，注释第一行！
* 敏感词 API 接口
*
* @author: 永不言败 <向国足学习永不言败>
* @since: 2022/3/23 17:00
* @description:
* @modification:
* @author 永不言败
*/
public interface SensitiveWordApi {
/**
* 获得文本所包含的不合法的敏感词数组
*
* @param text 文本
* @param tags 标签数组
* @return 不合法的敏感词数组
*/
List<String> validateText(String text, List<String> tags);
/**
* 判断文本是否包含敏感词
*
* @param text 文本
* @param tags 表述数组
* @return 是否包含
*/
boolean isTextValid(String text, List<String> tags);
}

29
yudao-module-system/yudao-module-system-impl/src/main/java/cn/iocoder/yudao/module/system/api/sensitiveword/SensitiveWordApiImpl.java Normal file
View File

@ -0,0 +1,29 @@
package cn.iocoder.yudao.module.system.api.sensitiveword;
import cn.iocoder.yudao.module.system.service.sensitiveword.SensitiveWordService;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.List;
/**
* 敏感词 API 实现类
*
* @author 永不言败
*/
@Service
public class SensitiveWordApiImpl implements SensitiveWordApi {
@Resource
private SensitiveWordService sensitiveWordService;
@Override
public List<String> validateText(String text, List<String> tags) {
return sensitiveWordService.validateText(text, tags);
}
@Override
public boolean isTextValid(String text, List<String> tags) {
return sensitiveWordService.isTextValid(text, tags);
}
}

26
yudao-module-system/yudao-module-system-impl/src/main/java/cn/iocoder/yudao/module/system/controller/admin/sensitiveword/SensitiveWordController.http
View File

@ -1,22 +1,4 @@
GET http://localhost:81/dev-api/admin-api/system/sensitive-word/get-all-tags
Accept: application/json
Authorization: Bearer 385d533b781f44f6bb21ea08afeec47c
###
POST http://localhost:81/dev-api/admin-api/system/sensitive-word/create
Content-Type: application/json
Authorization: Bearer 1649ff1f8b9a4eeeb458fe93a71c78b5
{
"name": "test",
"tags": [
"bbb,aaa"
],
"description": "test",
"status": true
}
###
### 请求 /system/sensitive-word/validate-text 接口 => 成功
GET {{baseUrl}}/system/sensitive-word/validate-text?text=XXX&tags=短信&tags=蔬菜
Authorization: Bearer {{token}}
tenant-id: {{adminTenentId}}

26
yudao-module-system/yudao-module-system-impl/src/main/java/cn/iocoder/yudao/module/system/controller/admin/sensitiveword/SensitiveWordController.java
View File

@ -75,13 +75,6 @@ public class SensitiveWordController {
return success(SensitiveWordConvert.INSTANCE.convertPage(pageResult));
}
@GetMapping("/get-tags")
@ApiOperation("获取所有敏感词的标签数组")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:query')")
public CommonResult<Set<String>> getSensitiveWordTags() throws IOException {
return success(sensitiveWordService.getSensitiveWordTags());
}
@GetMapping("/export-excel")
@ApiOperation("导出敏感词 Excel")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:export')")
@ -94,11 +87,18 @@ public class SensitiveWordController {
ExcelUtils.write(response, "敏感词.xls", "数据", SensitiveWordExcelVO.class, datas);
}
// @GetMapping("/is-sensitive-word-by-text-and-tag")
// @ApiOperation("通过tag判断传入text是否含有敏感词")
// @PreAuthorize("@ss.hasPermission('system:sensitive-word:checkbytextandtag')")
// public CommonResult<Boolean> isSensitiveWordByTextAndTag(@NotBlank String text, @NotBlank String tag) throws IOException {
// return success(sensitiveWordApi.isSensitiveWordByTextAndTag(text,tag));
// }
@GetMapping("/get-tags")
@ApiOperation("获取所有敏感词的标签数组")
@PreAuthorize("@ss.hasPermission('system:sensitive-word:query')")
public CommonResult<Set<String>> getSensitiveWordTags() throws IOException {
return success(sensitiveWordService.getSensitiveWordTags());
}
@GetMapping("/validate-text")
@ApiOperation("获得文本所包含的不合法的敏感词数组")
public CommonResult<List<String>> validateText(@RequestParam("text") String text,
@RequestParam(value = "tags", required = false) List<String> tags) {
return success(sensitiveWordService.validateText(text, tags));
}
}

18
yudao-module-system/yudao-module-system-impl/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordService.java
View File

@ -83,4 +83,22 @@ public interface SensitiveWordService {
*/
Set<String> getSensitiveWordTags();
/**
* 获得文本所包含的不合法的敏感词数组
*
* @param text 文本
* @param tags 标签数组
* @return 不合法的敏感词数组
*/
List<String> validateText(String text, List<String> tags);
/**
* 判断文本是否包含敏感词
*
* @param text 文本
* @param tags 表述数组
* @return 是否包含
*/
boolean isTextValid(String text, List<String> tags);
}

79
yudao-module-system/yudao-module-system-impl/src/main/java/cn/iocoder/yudao/module/system/service/sensitiveword/SensitiveWordServiceImpl.java
View File

@ -1,6 +1,7 @@
package cn.iocoder.yudao.module.system.service.sensitiveword;
import cn.hutool.core.collection.CollUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.module.system.controller.admin.sensitiveword.vo.SensitiveWordCreateReqVO;
@ -11,9 +12,11 @@ import cn.iocoder.yudao.module.system.convert.sensitiveword.SensitiveWordConvert
import cn.iocoder.yudao.module.system.dal.dataobject.sensitiveword.SensitiveWordDO;
import cn.iocoder.yudao.module.system.dal.mysql.sensitiveword.SensitiveWordMapper;
import cn.iocoder.yudao.module.system.mq.producer.sensitiveword.SensitiveWordProducer;
import cn.iocoder.yudao.module.system.util.collection.SimpleTrie;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Lazy;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
@ -23,7 +26,8 @@ import javax.annotation.Resource;
import java.util.*;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.*;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_EXISTS;
import static cn.iocoder.yudao.module.system.enums.ErrorCodeConstants.SENSITIVE_WORD_NOT_EXISTS;
/**
* 敏感词 Service 实现类
@ -71,9 +75,14 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
@Resource
private SensitiveWordProducer sensitiveWordProducer;
@Resource
@Lazy
private SensitiveWordService self;
/**
* 默认的敏感词的字典树，包含所有敏感词
*/
private volatile SimpleTrie defaultSensitiveWordTrie = new SimpleTrie(Collections.emptySet());
/**
* 标签与敏感词的字段数的映射
*/
private volatile Map<String, SimpleTrie> tagSensitiveWordTries = Collections.emptyMap();
/**
* 初始化 {@link #sensitiveWordCache} 缓存
@ -91,15 +100,38 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
Set<String> tags = new HashSet<>();
sensitiveWordList.forEach(word -> tags.addAll(word.getTags()));
sensitiveWordTagsCache = tags;
// 写入 defaultSensitiveWordTrie、tagSensitiveWordTries 缓存
initSensitiveWordTrie(sensitiveWordList);
// 写入 sensitiveWordCache 缓存
sensitiveWordCache = CollectionUtils.convertMap(sensitiveWordList, SensitiveWordDO::getId);
maxUpdateTime = CollectionUtils.getMaxValue(sensitiveWordList, SensitiveWordDO::getUpdateTime);
log.info("[initLocalCache][初始化 敏感词 数量为 {}]", sensitiveWordList.size());
}
private void initSensitiveWordTrie(List<SensitiveWordDO> wordDOs) {
// 过滤禁用的敏感词
wordDOs = CollectionUtils.filterList(wordDOs, word -> word.getStatus().equals(CommonStatusEnum.ENABLE.getStatus()));
// 初始化默认的 defaultSensitiveWordTrie
this.defaultSensitiveWordTrie = new SimpleTrie(CollectionUtils.convertList(wordDOs, SensitiveWordDO::getName));
// 初始化 tagSensitiveWordTries
Multimap<String, String> tagWords = HashMultimap.create();
for (SensitiveWordDO word : wordDOs) {
if (CollUtil.isEmpty(word.getTags())) {
continue;
}
word.getTags().forEach(tag -> tagWords.put(tag, word.getName()));
}
// 添加到 tagSensitiveWordTries 中
Map<String, SimpleTrie> tagSensitiveWordTries = new HashMap<>();
tagWords.asMap().forEach((tag, words) -> tagSensitiveWordTries.put(tag, new SimpleTrie(words)));
this.tagSensitiveWordTries = tagSensitiveWordTries;
}
@Scheduled(fixedDelay = SCHEDULER_PERIOD, initialDelay = SCHEDULER_PERIOD)
public void schedulePeriodicRefresh() {
self.initLocalCache();
initLocalCache();
}
/**
@ -203,4 +235,39 @@ public class SensitiveWordServiceImpl implements SensitiveWordService {
return sensitiveWordTagsCache;
}
@Override
public List<String> validateText(String text, List<String> tags) {
if (CollUtil.isEmpty(tags)) {
return defaultSensitiveWordTrie.validate(text);
}
// 有标签的情况
Set<String> result = new HashSet<>();
tags.forEach(tag -> {
SimpleTrie trie = tagSensitiveWordTries.get(tag);
if (trie == null) {
return;
}
result.addAll(trie.validate(text));
});
return new ArrayList<>(result);
}
@Override
public boolean isTextValid(String text, List<String> tags) {
if (CollUtil.isEmpty(tags)) {
return defaultSensitiveWordTrie.isValid(text);
}
// 有标签的情况
for (String tag : tags) {
SimpleTrie trie = tagSensitiveWordTries.get(tag);
if (trie == null) {
continue;
}
if (!trie.isValid(text)) {
return false;
}
}
return true;
}
}

6
yudao-module-system/yudao-module-system-impl/src/main/java/cn/iocoder/yudao/module/system/util/collection/SimpleTrie.java
View File

@ -1,5 +1,7 @@
package cn.iocoder.yudao.module.system.util.collection;
import cn.hutool.core.collection.CollUtil;
import java.util.*;
/**
@ -27,10 +29,10 @@ public class SimpleTrie {
*
* @param strs 字符串数组
*/
public SimpleTrie(List<String> strs) {
public SimpleTrie(Collection<String> strs) {
children = new HashMap<>();
// 构建树
Collections.sort(strs); // 排序，优先使用较短的前缀
CollUtil.sort(strs, String::compareTo); // 排序，优先使用较短的前缀
for (String str : strs) {
Map<Character, Object> child = children;
// 遍历每个字符