完善敏感词管理

This commit is contained in:
wenc000 2020-06-17 22:14:09 +08:00
parent 38651f6865
commit 14033464cd
14 changed files with 416 additions and 63 deletions

View File

@ -16,7 +16,9 @@ import com.cm.plugin.service.ISensitiveWordsService;
import io.swagger.annotations.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@ -42,9 +44,7 @@ public class SensitiveWordsController extends AbstractController {
@PostMapping("savesensitivewords")
@CheckRequestBodyAnnotation
public SuccessResult saveSensitiveWords(@RequestBody SensitiveWordsVO sensitiveWordsVO) throws Exception {
SuccessResult result = sensitiveWordsService.saveSensitiveWords(sensitiveWordsVO);
sensitiveWordsService.refreshListSensitiveWords("sensitiveWords");
return result;
return sensitiveWordsService.saveSensitiveWords(sensitiveWordsVO);
}
@ApiOperation(value = "删除敏感词(id列表)", notes = "删除敏感词(id列表)接口")
@ -54,9 +54,7 @@ public class SensitiveWordsController extends AbstractController {
@ApiResponses({@ApiResponse(code = 400, message = "请求失败", response = ErrorResult.class)})
@DeleteMapping("removesensitivewords/{ids}")
public SuccessResult removeSensitiveWords(@PathVariable("ids") String ids) throws RemoveException {
SuccessResult result = sensitiveWordsService.removeSensitiveWords(ids);
sensitiveWordsService.refreshListSensitiveWords("sensitiveWords");
return result;
return sensitiveWordsService.removeSensitiveWords(ids);
}
@ApiOperation(value = "修改敏感词", notes = "修改敏感词接口")
@ -67,9 +65,7 @@ public class SensitiveWordsController extends AbstractController {
@PutMapping("updatesensitivewords/{sensitiveWordsId}")
@CheckRequestBodyAnnotation
public SuccessResult updateSensitiveWords(@PathVariable("sensitiveWordsId") String sensitiveWordsId, @RequestBody SensitiveWordsVO sensitiveWordsVO) throws Exception {
SuccessResult result = sensitiveWordsService.updateSensitiveWords(sensitiveWordsId, sensitiveWordsVO);
sensitiveWordsService.refreshListSensitiveWords("sensitiveWords");
return result;
return sensitiveWordsService.updateSensitiveWords(sensitiveWordsId, sensitiveWordsVO);
}
@ApiOperation(value = "敏感词详情(通过ID)", notes = "敏感词详情(通过ID)接口")
@ -90,13 +86,6 @@ public class SensitiveWordsController extends AbstractController {
return sensitiveWordsService.listSensitiveWords(params);
}
@ApiOperation(value = "敏感词列表缓存", notes = "敏感词列表缓存接口")
@ApiResponses({@ApiResponse(code = 400, message = "请求失败", response = ErrorResult.class)})
@GetMapping("listsensitivewordscache")
public List<SensitiveWordsDTO> listSensitiveWordsCache() throws SearchException {
return sensitiveWordsService.listSensitiveWords("sensitiveWords");
}
@ApiOperation(value = "敏感词分页列表", notes = "敏感词分页列表接口")
@ApiImplicitParams({
@ApiImplicitParam(name = "page", value = "当前页码", paramType = "query", dataType = "Integer", defaultValue = "1"),
@ -113,4 +102,11 @@ public class SensitiveWordsController extends AbstractController {
return sensitiveWordsService.listPageSensitiveWords(page);
}
@ApiOperation(value = "导入敏感词", notes = "导入敏感词接口")
@ApiResponses({@ApiResponse(code = 400, message = "请求失败", response = ErrorResult.class)})
@PostMapping("savesensitivewordsexcel")
public SuccessResult saveSensitiveWordsExcel(MultipartFile excel) throws IOException {
return sensitiveWordsService.saveSensitiveWordsExcel(excel);
}
}

View File

@ -42,4 +42,10 @@ public class SensitiveWordsRouteController {
return mv;
}
@GetMapping("upload/saveuploadexcel")
public ModelAndView saveUploadExcel() {
ModelAndView mv = new ModelAndView("upload/save-upload-excel");
return mv;
}
}

View File

@ -61,5 +61,12 @@ public interface ISensitiveWordsDao {
* @throws SearchException
*/
List<SensitiveWordsDTO> listSensitiveWords(Map<String, Object> params) throws SearchException;
/**
* 敏感词列表
*
* @return
* @throws SearchException
*/
List<String> listSensitiveWordsForString() throws SearchException;
}

View File

@ -0,0 +1,27 @@
package com.cm.plugin.listener.excel;
import com.alibaba.excel.annotation.ExcelProperty;
/**
* When you feel like quitting. Think about why you started
* 当你想要放弃的时候想想当初你为何开始
*
* @ClassName: SensitiveExcel
* @Description: 敏感词
* @Author: WangGeng
* @Date: 2020/6/17 21:02
* @Version: 1.0
**/
public class SensitiveWordsExcel {
@ExcelProperty(index = 0)
private String sensitiveWord;
public String getSensitiveWord() {
return sensitiveWord == null ? "" : sensitiveWord.trim();
}
public void setSensitiveWord(String sensitiveWord) {
this.sensitiveWord = sensitiveWord;
}
}

View File

@ -0,0 +1,55 @@
package com.cm.plugin.listener.excel;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
/**
* When you feel like quitting. Think about why you started
* 当你想要放弃的时候想想当初你为何开始
*
* @ClassName: SensitiveExcelListener
* @Description: 敏感词导入
* @Author: WangGeng
* @Date: 2020/6/17 21:03
* @Version: 1.0
**/
public abstract class SensitiveWordsExcelListener extends AnalysisEventListener<SensitiveWordsExcel> {
private static final Logger LOG = LoggerFactory.getLogger(SensitiveWordsExcelListener.class);
private List<SensitiveWordsExcel> sensitiveWordsExcels = new ArrayList<>(0);
@Override
public void invoke(SensitiveWordsExcel sensitiveWordsExcel, AnalysisContext analysisContext) {
sensitiveWordsExcels.add(sensitiveWordsExcel);
if (sensitiveWordsExcels.size() > 500) {
try {
listSensitiveWordsExcel(sensitiveWordsExcels);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
}
sensitiveWordsExcels.clear();
}
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
try {
listSensitiveWordsExcel(sensitiveWordsExcels);
} catch (Exception e) {
LOG.error(e.getMessage());
}
sensitiveWordsExcels.clear();
}
/**
* 获取敏感词
*
* @param sensitiveWordsExcels
* @throws Exception
*/
public abstract void listSensitiveWordsExcel(List<SensitiveWordsExcel> sensitiveWordsExcels) throws Exception;
}

View File

@ -0,0 +1,39 @@
package com.cm.plugin.manager;
import java.util.ArrayList;
import java.util.List;
/**
* When you feel like quitting. Think about why you started
* 当你想要放弃的时候想想当初你为何开始
*
* @ClassName: SensitiveResult
* @Description: 敏感结果
* @Author: WangGeng
* @Date: 2020/6/17 20:07
* @Version: 1.0
**/
public class SensitiveResult {
private String sensitive;
private List<SensitiveWordsResult> sensitiveWordsResults;
public String getSensitive() {
return sensitive == null ? "" : sensitive.trim();
}
public void setSensitive(String sensitive) {
this.sensitive = sensitive;
}
public List<SensitiveWordsResult> getSensitiveWordsResults() {
if (sensitiveWordsResults == null) {
return new ArrayList<>();
}
return sensitiveWordsResults;
}
public void setSensitiveWordsResults(List<SensitiveWordsResult> sensitiveWordsResults) {
this.sensitiveWordsResults = sensitiveWordsResults;
}
}

View File

@ -1,12 +1,17 @@
package com.cm.plugin.manager;
import com.alibaba.druid.sql.visitor.functions.Char;
import org.apache.commons.lang3.StringUtils;
import org.omg.CORBA.OBJ_ADAPTER;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
/**
* When you feel like quitting. Think about why you started
@ -20,22 +25,31 @@ import java.util.Map;
**/
public class SensitiveWordsManager {
public static void main(String[] args) {
private static final Logger LOG = LoggerFactory.getLogger(SensitiveWordsManager.class);
private static SensitiveWordsManager sensitiveWordsManager = SensitiveWordsManagerBuilder.sensitiveWordsManager;
private Map<String, Object> sensitiveWordMap = new ConcurrentHashMap<>(16);
public static final String KEY_IS_END = "isEnd";
public static final String KEY_DEPTH = "depth";
private SensitiveWordsManager() {
}
public static SensitiveWordsManager getInstance() {
return sensitiveWordsManager;
}
// 构建敏感词库
Map<String, Object> sensitiveWordMap = new HashMap<>(16);
List<String> sensitiveWordList = new ArrayList<>();
sensitiveWordList.add("王耿");
sensitiveWordList.add("王艳");
sensitiveWordList.add("张三");
sensitiveWordList.add("张三狼");
sensitiveWordList.add("李四");
sensitiveWordList.add("李四狗");
sensitiveWordList.add("李四狗逼");
sensitiveWordList.add("fuckyou");
/**
* 初始化敏感词
*
* @param sensitiveWordList 敏感词列表
*/
public void initSensitiveWords(List<String> sensitiveWordList) {
LOG.debug("初始化敏感词列表");
sensitiveWordMap.clear();
if (sensitiveWordList == null || sensitiveWordList.isEmpty()) {
LOG.debug("初始化敏感词列表为空");
return;
}
for (String sensitiveWord : sensitiveWordList) {
Map<String, Object> tempSensitiveWordMap = sensitiveWordMap;
int depth = 0;
@ -44,31 +58,45 @@ public class SensitiveWordsManager {
String wordString = String.valueOf(word);
Map<String, Object> wordMap = (Map<String, Object>) tempSensitiveWordMap.get(wordString);
if (wordMap == null) {
Map<String, Object> newWordMap = new HashMap<>(16);
newWordMap.put("isEnd", 0);
newWordMap.put("depth", depth++);
Map<String, Object> newWordMap = new ConcurrentHashMap<>(16);
newWordMap.put(KEY_IS_END, 0);
newWordMap.put(KEY_DEPTH, depth++);
tempSensitiveWordMap.put(wordString, newWordMap);
tempSensitiveWordMap = newWordMap;
} else {
tempSensitiveWordMap = wordMap;
depth++;
}
if (i == sensitiveWord.length() - 1) {
tempSensitiveWordMap.put("isEnd", 1);
tempSensitiveWordMap.put(KEY_IS_END, 1);
}
}
}
LOG.debug("敏感词列表初始化成功");
}
System.out.println(sensitiveWordMap);
// 匹配敏感词
String checkWord = "Fuck you, America我是中华人民共和国人民我叫王耿我媳妇叫王艳我 有一个好邻居,他的名字叫张 三,他爸爸叫李四,他们是其乐融融的一家";
checkWord = checkWord.replaceAll("(\\w)\\s+(\\w)", "$1@$2").replaceAll("\\s+", "").replaceAll("@", " ");
/**
* 获取敏感词结果
*
* @param content
* @return
*/
public SensitiveResult getSensitiveWordsResult(String content) {
if (StringUtils.isBlank(content)) {
return null;
}
String result = content.replaceAll("(\\w)[\\s| ]+(\\w)", "$1@$2").replaceAll("\\s+", "").replaceAll("@", " ");
List<SensitiveWordsResult> sensitiveWordsResults = new ArrayList<>();
SensitiveResult sensitiveResult = new SensitiveResult();
sensitiveResult.setSensitive(result);
sensitiveResult.setSensitiveWordsResults(sensitiveWordsResults);
if (sensitiveWordMap.isEmpty()) {
return sensitiveResult;
}
Map<String, Object> tempMap = sensitiveWordMap;
StringBuilder sensitiveWord = new StringBuilder();
int depth = 0;
for (char word : checkWord.toCharArray()) {
for (char word : result.toCharArray()) {
String wordString = String.valueOf(word).toLowerCase();
Map<String, Object> wordMap = (Map<String, Object>) tempMap.get(wordString);
if (wordMap == null) {
@ -76,7 +104,7 @@ public class SensitiveWordsManager {
continue;
}
tempMap = wordMap;
if (depth == (int) wordMap.get("depth")) {
if (depth == (int) wordMap.get(KEY_DEPTH)) {
sensitiveWord.append(wordString);
depth++;
} else {
@ -86,13 +114,17 @@ public class SensitiveWordsManager {
depth++;
continue;
}
if ((int) wordMap.get("isEnd") == 1) {
System.out.println(sensitiveWord);
if ((int) wordMap.get(KEY_IS_END) == 1) {
sensitiveWordsResults.add(new SensitiveWordsResult(sensitiveWord.toString()));
depth = 0;
sensitiveWord = new StringBuilder();
}
}
return sensitiveResult;
}
private static class SensitiveWordsManagerBuilder {
public static SensitiveWordsManager sensitiveWordsManager = new SensitiveWordsManager();
}
}

View File

@ -0,0 +1,28 @@
package com.cm.plugin.manager;
/**
* When you feel like quitting. Think about why you started
* 当你想要放弃的时候想想当初你为何开始
*
* @ClassName: SensitiveWordResult
* @Description: 敏感词结果
* @Author: WangGeng
* @Date: 2020/6/17 20:05
* @Version: 1.0
**/
public class SensitiveWordsResult {
public SensitiveWordsResult(String sensitiveWord) {
this.sensitiveWord = sensitiveWord;
}
private String sensitiveWord;
public String getSensitiveWord() {
return sensitiveWord == null ? "" : sensitiveWord.trim();
}
public void setSensitiveWord(String sensitiveWord) {
this.sensitiveWord = sensitiveWord;
}
}

View File

@ -7,7 +7,9 @@ import com.cm.common.result.SuccessResult;
import com.cm.common.result.SuccessResultList;
import com.cm.plugin.pojo.dtos.SensitiveWordsDTO;
import com.cm.plugin.pojo.vos.SensitiveWordsVO;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@ -106,20 +108,21 @@ public interface ISensitiveWordsService {
*/
SuccessResultList<List<SensitiveWordsDTO>> listPageSensitiveWords(ListPage page) throws SearchException;
/**
* 缓存列表
*
* @param sensitiveWords
* @return
*/
List<SensitiveWordsDTO> listSensitiveWords(String sensitiveWords);
/**
* 刷新缓存
* 敏感词列表
*
* @param sensitiveWords
* @return
* @throws SearchException
*/
List<SensitiveWordsDTO> refreshListSensitiveWords(String sensitiveWords);
List<String> listSensitiveWordsForString() throws SearchException;
/**
* 导入敏感词
*
* @param excel
* @return
* @throws SearchException, IOException
*/
SuccessResult saveSensitiveWordsExcel(MultipartFile excel) throws SearchException, IOException;
}

View File

@ -1,5 +1,6 @@
package com.cm.plugin.service.impl;
import com.alibaba.excel.EasyExcel;
import com.cm.common.base.AbstractService;
import com.cm.common.exception.RemoveException;
import com.cm.common.exception.SearchException;
@ -9,6 +10,9 @@ import com.cm.common.result.SuccessResultList;
import com.cm.common.utils.HashMapUtil;
import com.cm.common.utils.UUIDUtil;
import com.cm.plugin.dao.ISensitiveWordsDao;
import com.cm.plugin.listener.excel.SensitiveWordsExcel;
import com.cm.plugin.listener.excel.SensitiveWordsExcelListener;
import com.cm.plugin.manager.SensitiveWordsManager;
import com.cm.plugin.pojo.dtos.SensitiveWordsDTO;
import com.cm.plugin.pojo.vos.SensitiveWordsVO;
import com.cm.plugin.service.ISensitiveWordsService;
@ -18,7 +22,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CachePut;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@ -64,6 +70,7 @@ public class SensitiveWordsServiceImpl extends AbstractService implements ISensi
setSaveInfo(params);
}
sensitiveWordsDao.saveSensitiveWords(params);
SensitiveWordsManager.getInstance().initSensitiveWords(listSensitiveWordsForString());
}
@Override
@ -93,6 +100,7 @@ public class SensitiveWordsServiceImpl extends AbstractService implements ISensi
setUpdateInfo(params);
}
sensitiveWordsDao.removeSensitiveWords(params);
SensitiveWordsManager.getInstance().initSensitiveWords(listSensitiveWordsForString());
}
@Override
@ -123,6 +131,7 @@ public class SensitiveWordsServiceImpl extends AbstractService implements ISensi
setUpdateInfo(params);
}
sensitiveWordsDao.updateSensitiveWords(params);
SensitiveWordsManager.getInstance().initSensitiveWords(listSensitiveWordsForString());
}
@Override
@ -138,15 +147,27 @@ public class SensitiveWordsServiceImpl extends AbstractService implements ISensi
}
@Override
@Cacheable(cacheNames = "sensitiveWords", key = "#sensitiveWords")
public List<SensitiveWordsDTO> listSensitiveWords(String sensitiveWords) throws SearchException {
return sensitiveWordsDao.listSensitiveWords(getHashMap(0));
public List<String> listSensitiveWordsForString() throws SearchException {
return sensitiveWordsDao.listSensitiveWordsForString();
}
@Override
@CachePut(cacheNames = "sensitiveWords", key = "#sensitiveWords")
public List<SensitiveWordsDTO> refreshListSensitiveWords(String sensitiveWords) {
return sensitiveWordsDao.listSensitiveWords(getHashMap(0));
public SuccessResult saveSensitiveWordsExcel(MultipartFile multipartFile) throws SearchException, IOException {
EasyExcel.read(multipartFile.getInputStream(), SensitiveWordsExcel.class, new SensitiveWordsExcelListener() {
@Override
public void listSensitiveWordsExcel(List<SensitiveWordsExcel> sensitiveWordsExcels) throws Exception {
Map<String, Object> params = getHashMap(7);
setSaveInfo(params);
for (SensitiveWordsExcel sensitiveWordsExcel : sensitiveWordsExcels) {
params.put("sensitiveWordsId", UUIDUtil.getUUID());
params.put("name", sensitiveWordsExcel.getSensitiveWord().trim());
params.put("message", "存在敏感词,请修改");
sensitiveWordsDao.saveSensitiveWords(params);
}
}
}).sheet().doRead();
SensitiveWordsManager.getInstance().initSensitiveWords(listSensitiveWordsForString());
return new SuccessResult();
}
@Override

View File

@ -0,0 +1,42 @@
package com.cm.plugin.startup;
import com.cm.plugin.manager.SensitiveWordsManager;
import com.cm.plugin.service.ISensitiveWordsService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;
import java.util.List;
/**
* When you feel like quitting. Think about why you started
* 当你想要放弃的时候想想当初你为何开始
*
* @ClassName: SensitiveWordsStartUp
* @Description: 敏感词启动
* @Author: WangGeng
* @Date: 2020/6/17 20:21
* @Version: 1.0
**/
@Component
public class SensitiveWordsStartUp implements ApplicationRunner {
private static final Logger LOG = LoggerFactory.getLogger(SensitiveWordsStartUp.class);
@Autowired
private ISensitiveWordsService sensitiveWordsService;
@Override
public void run(ApplicationArguments args) throws Exception {
LOG.debug("初始化敏感词开始");
long startTime = System.currentTimeMillis();
List<String> sensitiveWords = sensitiveWordsService.listSensitiveWordsForString();
SensitiveWordsManager.getInstance().initSensitiveWords(sensitiveWords);
long endTime = System.currentTimeMillis();
LOG.debug("初始化敏感词结束,耗时:{}ms", (endTime - startTime));
}
}

View File

@ -111,10 +111,20 @@
#{sensitiveWordsIds[${index}]}
</foreach>
</if>
</select>
<!-- 敏感词列表 -->
<select id="listSensitiveWordsForString" resultType="java.lang.String">
SELECT
t1.name
FROM
sys_sensitive_words t1
WHERE
t1.is_delete = 0
GROUP BY
t1.name,
t1.message,
t1.sensitive_words_id
t1.name
ORDER BY
t1.name
</select>
</mapper>

View File

@ -23,6 +23,9 @@
<button type="button" id="search" class="layui-btn layui-btn-sm">
<i class="fa fa-lg fa-search"></i> 搜索
</button>
<button type="button" class="layui-btn layui-btn-sm" id="uploadExcel">
<i class="fa fa-lg fa-cloud-upload"></i> 导入数据
</button>
</div>
<table class="layui-hide" id="dataTable" lay-filter="dataTable"></table>
<!-- 表头按钮组 -->
@ -171,6 +174,17 @@
$(document).on('click', '#search', function() {
reloadTable(1);
});
$(document).on('click', '#uploadExcel', function() {
top.dialog.open({
url: top.restAjax.path('route/sensitivewords/upload/saveuploadexcel', []),
title: '导入用户数据',
width: '300px',
height: '196px',
onClose: function() {
reloadTable();
}
});
});
// 事件 - 增删改
table.on('toolbar(dataTable)', function(obj) {
var layEvent = obj.event;

View File

@ -0,0 +1,73 @@
<!doctype html>
<html lang="en" xmlns:th="http://www.thymeleaf.org">
<head>
<base th:href="${#request.getContextPath() + '/'} ">
<meta charset="utf-8">
<meta name="renderer" content="webkit">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=0">
<link rel="stylesheet" href="assets/fonts/font-awesome/css/font-awesome.css"/>
<link rel="stylesheet" href="assets/layuiadmin/layui/css/layui.css" media="all">
<link rel="stylesheet" href="assets/layuiadmin/style/admin.css" media="all">
</head>
<body>
<div class="layui-anim layui-anim-fadein">
<div class="layui-card" style="text-align: center;">
<div class="layui-card-body" style="padding: 15px;">
<blockquote class="layui-elem-quote">第一行为表头第二行第一列开始为数据格式为xls或xlsx</blockquote>
<button type="button" class="layui-btn layui-btn" id="uploadExcel">
<i class="fa fa-lg fa-cloud-upload"></i> 导入数据
</button>
</div>
</div>
</div>
<script src="assets/layuiadmin/layui/layui.js"></script>
<script>
layui.config({
base: 'assets/layuiadmin/' //静态资源所在路径
}).extend({
index: 'lib/index' //主入口模块
}).use(['index', 'upload'], function(){
var $ = layui.$;
var form = layui.form;
function closeBox() {
parent.layer.close(parent.layer.getFrameIndex(window.name));
}
// 初始化Excel上传
function initExcelUpload() {
// Excel上传
var uploadLoading;
layui.upload.render({
elem: '#uploadExcel',
url: 'api/sensitivewords/savesensitivewordsexcel',
accept: 'file',
exts: 'xls|xlsx',
field: 'excel',
before: function() {
uploadLoading = layer.msg('正在上传,请稍后...', {icon: 16, time: 0, shade: 0.3});
},
done: function(data) {
layer.close(uploadLoading);
layer.msg('导入成功', {time: 2000}, function() {
closeBox();
});
},
error: function(data, index){
layer.close(uploadLoading);
if(data != null) {
top.dialog.msg(data.msg);
}
},
});
}
initExcelUpload();
$('.close').on('click', function() {
closeBox();
});
});
</script>
</body>
</html>