java过滤敏感词

java过滤敏感词

1、加载敏感词库的工具类：

package com.mdframework.utils;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.*;


public class SensitiveWordInit {
     private String ENCODING = "GBK";    //字符编码
     private String UTF8 = "utf-8";
        @SuppressWarnings("rawtypes")
        public HashMap sensitiveWordMap;
        public SensitiveWordInit(){
            super();
        }
        @SuppressWarnings("rawtypes")
        public Map initKeyWord(Set set){
            try {
                //读取敏感词库
                Set keyWordSet = readSensitiveWordFile();

//                keyWordSet.add("红客");
//                keyWordSet.add("法轮");
//                keyWordSet.add("手机卡");
                //将敏感词库加入到HashMap中
                addSensitiveWordToHashMap(set);
                //spring获取application，然后application.setAttribute("sensitiveWordMap",sensitiveWordMap);
            } catch (Exception e) {
                e.printStackTrace();
            }
            return sensitiveWordMap;
        }



    //将得到的敏感词库用一个DFA算法模型放到map中
        @SuppressWarnings({ "rawtypes", "unchecked" })
        private void addSensitiveWordToHashMap(Set keyWordSet) {
            sensitiveWordMap = new HashMap(keyWordSet.size());     //初始化敏感词容器，减少扩容 *** 作
            String key = null;
            Map nowMap = null;
            Map newWorMap = null;
            //迭代keyWordSet
            Iterator iterator = keyWordSet.iterator();
            while(iterator.hasNext()){
                key = iterator.next();    //关键字
                nowMap = sensitiveWordMap;
                for(int i = 0 ; i < key.length() ; i++){
                    char keyChar = key.charAt(i);       //转换成char型
                    Object wordMap = nowMap.get(keyChar);       //获取

                    if(wordMap != null){        //如果存在该key，直接赋值
                        nowMap = (Map) wordMap;
                    }
                    else{     //不存在则，则构建一个map，同时将isEnd设置为0，因为他不是最后一个
                        newWorMap = new HashMap();
                        newWorMap.put("isEnd", "0");     //不是最后一个
                        nowMap.put(keyChar, newWorMap);
                        nowMap = newWorMap;
                    }

                    if(i == key.length() - 1){
                        nowMap.put("isEnd", "1");    //最后一个
                    }
                }
            }
        }

//读取敏感词文件 加到set集合中
        @SuppressWarnings("resource")
        private Set readSensitiveWordFile() throws Exception{
            Set set = null;
             FileInputStream file = new FileInputStream("D:/SensitiveWord.txt");    //读取文件
            InputStreamReader read = new InputStreamReader(file,UTF8);
            try {
                   //文件流是否存在
                    set = new HashSet();
                    BufferedReader bufferedReader = new BufferedReader(read);
                    String s = bufferedReader.readLine();
                    String txt = null;
                        while((txt = bufferedReader.readLine()) != null){    //读取文件，将文件内容放入到set中
                            set.add(txt);

                }
            } catch (Exception e) {
                throw e;
            }finally{
                read.close();     //关闭文件流
            }
            return set;
        }
}

2、对比敏感词库的工具类：

package com.mdframework.utils;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;



public class SensitivewordFilter {

    @SuppressWarnings("rawtypes")
    public Map sensitiveWordMap = null;
    public static int minMatchTYpe = 1;      //最小匹配规则
    public static int maxMatchType = 2;      //最大匹配规则
    
    public SensitivewordFilter(Set set){
        sensitiveWordMap = new SensitiveWordInit().initKeyWord(set);
    }
    
    public boolean isContaintSensitiveWord(String txt,int matchType){
        boolean flag = false;
        for(int i = 0 ; i < txt.length() ; i++){
            int matchFlag = this.CheckSensitiveWord(txt, i, matchType); //判断是否包含敏感字符
            if(matchFlag > 0){    //大于0存在，返回true
                flag = true;
            }
        }
        return flag;
    }



    
    @SuppressWarnings({ "rawtypes"})
    public int CheckSensitiveWord(String txt,int beginIndex,int matchType){
        boolean  flag = false;    //敏感词结束标识位：用于敏感词只有1位的情况
        int matchFlag = 0;     //匹配标识数默认为0
        char word = 0;
        Map nowMap = sensitiveWordMap;
        for(int i = beginIndex; i < txt.length() ; i++){
            word = txt.charAt(i);
//            System.out.println(nowMap);
            nowMap = (Map) nowMap.get(word);     //获取指定key
//            System.out.println(nowMap);
            if(nowMap != null){     //存在，则判断是否为最后一个
                matchFlag++;     //找到相应key，匹配标识+1
                if("1".equals(nowMap.get("isEnd"))){       //如果为最后一个匹配规则,结束循环，返回匹配标识数
                    flag = true;       //结束标志位为true
                    if(SensitivewordFilter.minMatchTYpe == matchType){    //最小规则，直接返回,最大规则还需继续查找
                        break;
                    }
                }
            }
            else{     //不存在，直接返回
                break;
            }
        }
        if(matchFlag < 2 || !flag){        //长度必须大于等于1，为词
            matchFlag = 0;
        }
        return matchFlag;
    }


    
    public Set getSensitiveWord(String txt , int matchType){
        Set sensitiveWordList = new HashSet();

        for(int i = 0 ; i < txt.length() ; i++){
            int length = CheckSensitiveWord(txt, i, matchType);    //判断是否包含敏感字符
            if(length > 0){    //存在,加入list中
                sensitiveWordList.add(txt.substring(i, i+length));
                i = i + length - 1;    //减1的原因，是因为for会自增
            }
        }

        return sensitiveWordList;
    }}

3、测试

package com.mdframework.module.controller.test;

import com.mdframework.utils.SensitiveWordInit;
import com.mdframework.utils.SensitivewordFilter;

import java.io.Serializable;
import java.util.HashSet;
import java.util.Set;


public class SensitivewordController {
    public static void main(String[] args) {
        Set set = new HashSet<>();
        set.add("红客");
        set.add("手机卡");
        SensitivewordFilter filter = new SensitivewordFilter(set);
        System.out.println("敏感词的数量：" + filter.sensitiveWordMap.size());
        String string = "太多的伤感情怀也许只局限于饲养基地 荧幕中的情节，主人公尝试着去用某种方式渐渐的很潇洒地释自杀指南怀那些自己经历的伤感。"
                + "然后法.轮.功 我们的扮演的角色就是跟随着主人公的喜红客联盟 怒哀乐而过于牵强的把自己的情感也附加于银幕情节中，然后感动就流泪，"
                + "难过就躺在某一个人的怀里尽情的阐述心扉或者手机卡复制器一个人一杯红酒一部电影在夜三.级.片 深人静的晚上，关上电话静静的发呆着。";
        System.out.println("待检测语句字数：" + string.length());
        long beginTime = System.currentTimeMillis();
        Set sensitiveWordSet = filter.getSensitiveWord(string, 1);
        long endTime = System.currentTimeMillis();
        System.out.println("语句中包含敏感词的个数为：" + sensitiveWordSet.size() +","+sensitiveWordSet);
        System.out.println("总共消耗时间为：" + (endTime - beginTime));
    }
}