|
|
@@ -1,250 +0,0 @@ |
|
|
|
#!/usr/bin/env python |
|
|
|
# -*- coding:utf-8 -*- |
|
|
|
# ToolGood.Words.WordsSearch.py |
|
|
|
# 2020, Lin Zhijun, https://github.com/toolgood/ToolGood.Words |
|
|
|
# Licensed under the Apache License 2.0 |
|
|
|
# 更新日志 |
|
|
|
# 2020.04.06 第一次提交 |
|
|
|
# 2020.05.16 修改,支持大于0xffff的字符 |
|
|
|
|
|
|
|
__all__ = ['WordsSearch'] |
|
|
|
__author__ = 'Lin Zhijun' |
|
|
|
__date__ = '2020.05.16' |
|
|
|
|
|
|
|
class TrieNode(): |
|
|
|
def __init__(self): |
|
|
|
self.Index = 0 |
|
|
|
self.Index = 0 |
|
|
|
self.Layer = 0 |
|
|
|
self.End = False |
|
|
|
self.Char = '' |
|
|
|
self.Results = [] |
|
|
|
self.m_values = {} |
|
|
|
self.Failure = None |
|
|
|
self.Parent = None |
|
|
|
|
|
|
|
def Add(self,c): |
|
|
|
if c in self.m_values : |
|
|
|
return self.m_values[c] |
|
|
|
node = TrieNode() |
|
|
|
node.Parent = self |
|
|
|
node.Char = c |
|
|
|
self.m_values[c] = node |
|
|
|
return node |
|
|
|
|
|
|
|
def SetResults(self,index): |
|
|
|
if (self.End == False): |
|
|
|
self.End = True |
|
|
|
self.Results.append(index) |
|
|
|
|
|
|
|
class TrieNode2(): |
|
|
|
def __init__(self): |
|
|
|
self.End = False |
|
|
|
self.Results = [] |
|
|
|
self.m_values = {} |
|
|
|
self.minflag = 0xffff |
|
|
|
self.maxflag = 0 |
|
|
|
|
|
|
|
def Add(self,c,node3): |
|
|
|
if (self.minflag > c): |
|
|
|
self.minflag = c |
|
|
|
if (self.maxflag < c): |
|
|
|
self.maxflag = c |
|
|
|
self.m_values[c] = node3 |
|
|
|
|
|
|
|
def SetResults(self,index): |
|
|
|
if (self.End == False) : |
|
|
|
self.End = True |
|
|
|
if (index in self.Results )==False : |
|
|
|
self.Results.append(index) |
|
|
|
|
|
|
|
def HasKey(self,c): |
|
|
|
return c in self.m_values |
|
|
|
|
|
|
|
|
|
|
|
def TryGetValue(self,c): |
|
|
|
if (self.minflag <= c and self.maxflag >= c): |
|
|
|
if c in self.m_values: |
|
|
|
return self.m_values[c] |
|
|
|
return None |
|
|
|
|
|
|
|
|
|
|
|
class WordsSearch(): |
|
|
|
def __init__(self): |
|
|
|
self._first = {} |
|
|
|
self._keywords = [] |
|
|
|
self._indexs=[] |
|
|
|
|
|
|
|
def SetKeywords(self,keywords): |
|
|
|
self._keywords = keywords |
|
|
|
self._indexs=[] |
|
|
|
for i in range(len(keywords)): |
|
|
|
self._indexs.append(i) |
|
|
|
|
|
|
|
root = TrieNode() |
|
|
|
allNodeLayer={} |
|
|
|
|
|
|
|
for i in range(len(self._keywords)): # for (i = 0; i < _keywords.length; i++) |
|
|
|
p = self._keywords[i] |
|
|
|
nd = root |
|
|
|
for j in range(len(p)): # for (j = 0; j < p.length; j++) |
|
|
|
nd = nd.Add(ord(p[j])) |
|
|
|
if (nd.Layer == 0): |
|
|
|
nd.Layer = j + 1 |
|
|
|
if nd.Layer in allNodeLayer: |
|
|
|
allNodeLayer[nd.Layer].append(nd) |
|
|
|
else: |
|
|
|
allNodeLayer[nd.Layer]=[] |
|
|
|
allNodeLayer[nd.Layer].append(nd) |
|
|
|
nd.SetResults(i) |
|
|
|
|
|
|
|
|
|
|
|
allNode = [] |
|
|
|
allNode.append(root) |
|
|
|
for key in allNodeLayer.keys(): |
|
|
|
for nd in allNodeLayer[key]: |
|
|
|
allNode.append(nd) |
|
|
|
allNodeLayer=None |
|
|
|
|
|
|
|
for i in range(len(allNode)): # for (i = 0; i < allNode.length; i++) |
|
|
|
if i==0 : |
|
|
|
continue |
|
|
|
nd=allNode[i] |
|
|
|
nd.Index = i |
|
|
|
r = nd.Parent.Failure |
|
|
|
c = nd.Char |
|
|
|
while (r != None and (c in r.m_values)==False): |
|
|
|
r = r.Failure |
|
|
|
if (r == None): |
|
|
|
nd.Failure = root |
|
|
|
else: |
|
|
|
nd.Failure = r.m_values[c] |
|
|
|
for key2 in nd.Failure.Results : |
|
|
|
nd.SetResults(key2) |
|
|
|
root.Failure = root |
|
|
|
|
|
|
|
allNode2 = [] |
|
|
|
for i in range(len(allNode)): # for (i = 0; i < allNode.length; i++) |
|
|
|
allNode2.append( TrieNode2()) |
|
|
|
|
|
|
|
for i in range(len(allNode2)): # for (i = 0; i < allNode2.length; i++) |
|
|
|
oldNode = allNode[i] |
|
|
|
newNode = allNode2[i] |
|
|
|
|
|
|
|
for key in oldNode.m_values : |
|
|
|
index = oldNode.m_values[key].Index |
|
|
|
newNode.Add(key, allNode2[index]) |
|
|
|
|
|
|
|
for index in range(len(oldNode.Results)): # for (index = 0; index < oldNode.Results.length; index++) |
|
|
|
item = oldNode.Results[index] |
|
|
|
newNode.SetResults(item) |
|
|
|
|
|
|
|
oldNode=oldNode.Failure |
|
|
|
while oldNode != root: |
|
|
|
for key in oldNode.m_values : |
|
|
|
if (newNode.HasKey(key) == False): |
|
|
|
index = oldNode.m_values[key].Index |
|
|
|
newNode.Add(key, allNode2[index]) |
|
|
|
for index in range(len(oldNode.Results)): |
|
|
|
item = oldNode.Results[index] |
|
|
|
newNode.SetResults(item) |
|
|
|
oldNode=oldNode.Failure |
|
|
|
allNode = None |
|
|
|
root = None |
|
|
|
|
|
|
|
# first = [] |
|
|
|
# for index in range(65535):# for (index = 0; index < 0xffff; index++) |
|
|
|
# first.append(None) |
|
|
|
|
|
|
|
# for key in allNode2[0].m_values : |
|
|
|
# first[key] = allNode2[0].m_values[key] |
|
|
|
|
|
|
|
self._first = allNode2[0] |
|
|
|
|
|
|
|
|
|
|
|
def FindFirst(self,text): |
|
|
|
ptr = None |
|
|
|
for index in range(len(text)): # for (index = 0; index < text.length; index++) |
|
|
|
t =ord(text[index]) # text.charCodeAt(index) |
|
|
|
tn = None |
|
|
|
if (ptr == None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
else: |
|
|
|
tn = ptr.TryGetValue(t) |
|
|
|
if (tn==None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
|
|
|
|
|
|
|
|
if (tn != None): |
|
|
|
if (tn.End): |
|
|
|
item = tn.Results[0] |
|
|
|
keyword = self._keywords[item] |
|
|
|
return { "Keyword": keyword, "Success": True, "End": index, "Start": index + 1 - len(keyword), "Index": self._indexs[item] } |
|
|
|
ptr = tn |
|
|
|
return None |
|
|
|
|
|
|
|
def FindAll(self,text): |
|
|
|
ptr = None |
|
|
|
list = [] |
|
|
|
|
|
|
|
for index in range(len(text)): # for (index = 0; index < text.length; index++) |
|
|
|
t =ord(text[index]) # text.charCodeAt(index) |
|
|
|
tn = None |
|
|
|
if (ptr == None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
else: |
|
|
|
tn = ptr.TryGetValue(t) |
|
|
|
if (tn==None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
|
|
|
|
|
|
|
|
if (tn != None): |
|
|
|
if (tn.End): |
|
|
|
for j in range(len(tn.Results)): # for (j = 0; j < tn.Results.length; j++) |
|
|
|
item = tn.Results[j] |
|
|
|
keyword = self._keywords[item] |
|
|
|
list.append({ "Keyword": keyword, "Success": True, "End": index, "Start": index + 1 - len(keyword), "Index": self._indexs[item] }) |
|
|
|
ptr = tn |
|
|
|
return list |
|
|
|
|
|
|
|
|
|
|
|
def ContainsAny(self,text): |
|
|
|
ptr = None |
|
|
|
for index in range(len(text)): # for (index = 0; index < text.length; index++) |
|
|
|
t =ord(text[index]) # text.charCodeAt(index) |
|
|
|
tn = None |
|
|
|
if (ptr == None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
else: |
|
|
|
tn = ptr.TryGetValue(t) |
|
|
|
if (tn==None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
|
|
|
|
if (tn != None): |
|
|
|
if (tn.End): |
|
|
|
return True |
|
|
|
ptr = tn |
|
|
|
return False |
|
|
|
|
|
|
|
def Replace(self,text, replaceChar = '*'): |
|
|
|
result = list(text) |
|
|
|
|
|
|
|
ptr = None |
|
|
|
for i in range(len(text)): # for (i = 0; i < text.length; i++) |
|
|
|
t =ord(text[i]) # text.charCodeAt(index) |
|
|
|
tn = None |
|
|
|
if (ptr == None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
else: |
|
|
|
tn = ptr.TryGetValue(t) |
|
|
|
if (tn==None): |
|
|
|
tn = self._first.TryGetValue(t) |
|
|
|
|
|
|
|
if (tn != None): |
|
|
|
if (tn.End): |
|
|
|
maxLength = len( self._keywords[tn.Results[0]]) |
|
|
|
start = i + 1 - maxLength |
|
|
|
for j in range(start,i+1): # for (j = start; j <= i; j++) |
|
|
|
result[j] = replaceChar |
|
|
|
ptr = tn |
|
|
|
return ''.join(result) |