diff --git a/server/PAFunc.py b/server/PAFunc.py index 5501703..9fc6ccf 100644 --- a/server/PAFunc.py +++ b/server/PAFunc.py @@ -1,9 +1,68 @@ +#Personal Assistant Utility Functions + +from datetime import datetime +import time +import os +import shutil + def testworks(): print "\n it worked" +#returns id of command NOTE -1 is an error def parseComm(PAComm): - #print PAComm + PAComm=PAComm.strip() PAComm=PAComm.lower() - - print PAComm + + words = PAComm.split() + #for word in words: + #print(word) + #print len(words) + numWords=len(words) + if numWords <= 0: + print "Unknown Command" + return "-1" + + + if len(words) >= 3: + if words[0]== "graph" and words[1] == "observe": + if "colorapart" in words: + return "1.1 " + words[2] + " " + words[words.index("colorapart")+1] + if "accessorder" in words: + return "1.2 " + words[2] + return "1 " + words[2] + if words[0]== "graph" and words[1] == "formulate": + return "2 " + words[2] + if words[0]== "graph" and words[1] == "evaluate": + return "3 " + words[2] +" "+ words[3] + + elif len(words) ==2: + if words[0]== "delete" and words[1] == "all": + return "5" + + return "-1" + +#takes dictionary entry of node and returns a string of its time data +def getTimeStr(dictEntry): + nodeTime=str(dictEntry) + nodeTime=nodeTime.replace('\'weight\':', '') + nodeTime=nodeTime.replace('\'', '') + nodeTime=nodeTime.replace('{', '') + nodeTime=nodeTime.replace('}', '') + nodeTime=nodeTime.strip() + return nodeTime + #return "29 June 2017 14:15" + +#takes string of a node's time and returns it in Unix date/time formatting +def getDateTimeUnix(nodeTime): + fmt = '%d %B %Y %H:%M' + dt = datetime.strptime(nodeTime, fmt) + dtUnix = time.mktime(dt.timetuple()) + return dtUnix + +def deleteCaseFiles(): + dirList= os.listdir(os.getcwd()) + + for dir in dirList: + if "case" in dir: + shutil.rmtree(dir) diff --git a/server/PAFunc.pyc b/server/PAFunc.pyc index 71c3282..d2587b0 100644 Binary files a/server/PAFunc.pyc and b/server/PAFunc.pyc differ diff --git a/server/example.txt b/server/example.txt index 88f9e0a..d7c8d0b 100644 --- a/server/example.txt +++ b/server/example.txt @@ -1,9 +1,9 @@ -WmiPrvSE.exe -> explorer.exe @ 1 July 2018 14:15 -explorer.exe -> firefox.exe @ 1 July 2018 14:16 -explorer.exe -> AcroRd32.exe @ 1 July 2018 14:17 -AcroRd32.exe -> notepad.exe @ 1 July 2018 14:18 -AcroRd32.exe -> 192.168.1.115 @ 1 July 2018 14:19 -firefox.exe -> 23.209.190.51 @ 1 July 2018 14:20 -firefox.exe -> 54.201.155.11 @ 1 July 2018 14:21 -firefox.exe -> Dropbox @ 1 July 2018 14:22 +WmiPrvSE.exe -> explorer.exe @ 29 June 2017 14:15 +explorer.exe -> firefox.exe @ 29 June 2017 14:16 +explorer.exe -> AcroRd32.exe @ 29 June 2017 15:17 +AcroRd32.exe -> notepad.exe @ 29 June 2017 14:18 +AcroRd32.exe -> 192.168.1.115 @ 29 June 2017 14:19 +firefox.exe -> 23.209.190.51 @ 29 June 2017 15:20 +firefox.exe -> 54.201.155.11 @ 29 June 2017 15:21 +firefox.exe -> Dropbox @ 29 June 2017 15:22 diff --git a/server/sherlock.py b/server/sherlock.py index 4ee4d65..6ec283a 100755 --- a/server/sherlock.py +++ b/server/sherlock.py @@ -4,7 +4,8 @@ from networkx.drawing.nx_agraph import graphviz_layout from collections import Counter # builtin from collections import defaultdict # builtin - +from datetime import datetime +import time import csv # builtin import os # builtin @@ -19,7 +20,7 @@ import numpy as np import pandas as pd -from PAFunc import testworks, parseComm +from PAFunc import testworks, parseComm, getTimeStr, getDateTimeUnix, deleteCaseFiles # Input: Source Node, Target Node, Graph @@ -57,8 +58,39 @@ def evaluate(src, trg, grph): # Input: Source Node # Output: Edges in depth-first-search -def formulate(src, grph): - print list(nx.dfs_edges(grph, src)) +def formulate(src, grph, graph=-1): + formList=list(nx.dfs_edges(sG, src)) + print formList + + + if graph==-1: + return + + fG = nx.DiGraph() + + for node in formList: + fG.add_edge(node[0], node[1], weight=getTimeStr(sG[node[0]][node[1]])) + + + pos = nx.layout.circular_layout(fG) + nodes = nx.draw_networkx_nodes(fG, pos, node_size=100, node_color='blue', alpha=.5) + edges = nx.draw_networkx_edges(fG, pos, node_size=100, arrowstyle='->', arrowsize=10, width=2) + edge_labels = nx.get_edge_attributes(fG,'weight') + + nx.draw_networkx_labels(fG, pos, font_size=10) + nx.draw_networkx_edge_labels(fG, pos, edge_labels = edge_labels, font_size=5) + + + + ax = plt.gca() + ax.set_axis_off() + + plt.savefig(caseFile+'/formulate_graph.png') + #plt.show() + plt.clf() + plt.close() + ax.cla() + # Input: List of Dictionaries # Output: Two Directed Graphs @@ -80,9 +112,25 @@ def builder(glst): # Degree Centrality, Katz Centrality # TXT File - Data Mining Results # Location: case -def observe(filename): +caseFile="" +sG=None +def observe(filename, colorApart=-1, accessOrder=-1): + # Make a folder - os.makedirs("case") + + global caseFile + global sG + caseFile="case" + if not os.path.isdir(caseFile): + os.makedirs(caseFile) + else: + i=1 + while(1==1): + if not os.path.isdir(caseFile+str(i)): + os.makedirs(caseFile+str(i)) + caseFile=caseFile+str(i) + break + i+=1 # Parse the file glst = parser(filename) @@ -95,6 +143,7 @@ def observe(filename): #G.draw("case/directed_graph.png") #NEW Graph NOTE the old graphs still exist but are not displayed + #NOTE sG is now a global! sG = nx.DiGraph() for node in glst: @@ -104,25 +153,88 @@ def observe(filename): #pos = nx.layout.spectral_layout(sG) #pos = nx.layout.shell_layout(sG) pos = nx.layout.circular_layout(sG) - M = sG.number_of_edges() - edge_colors = range(2, M + 2) nodes = nx.draw_networkx_nodes(sG, pos, node_size=100, node_color='blue', alpha=.5) - edges = nx.draw_networkx_edges(sG, pos, node_size=100, arrowstyle='->', arrowsize=10, edge_color=edge_colors, width=2) + + + + + + nodeList=sG.nodes() + #NOTE this is NOT a node dfs it is an EDGE dfs + dfsList=list(nx.edge_dfs(sG,nodeList)) + accessOrderList = list() #ordered list of nodes by access time + accessOrderDict = {} #dict of (node, timeInUnix) + i = 0 + #insert sorted + #Can you tell I'm a c programmer? + while i < len(dfsList): + tNode= dfsList[i] + nodeTime=getTimeStr(sG[tNode[0]][tNode[1]]) + dtUnix=getDateTimeUnix(nodeTime) + + if i == 0: + accessOrderList.append(tNode) + accessOrderDict[tNode]=dtUnix + elif accessOrderDict[accessOrderList[i-1]]<= dtUnix: + accessOrderList.append(tNode) + accessOrderDict[tNode]=dtUnix + else : + j=0 + while j < len(accessOrderList): + if accessOrderDict[accessOrderList[j]] > dtUnix: + accessOrderDict[tNode]=dtUnix + accessOrderList.insert(j, tNode) + break + j+= 1 + i+= 1 + + #print accessOrderList + if accessOrder != -1: + for item in accessOrderList: + print item[1] + print sG[item[0]][item[1]] + + if colorApart != -1: + coloredEdges=list(); + regEdges=list(); + i = 0 + while i < len(accessOrderList): + if i==0: + regEdges.append(accessOrderList[i]) + elif (getDateTimeUnix(getTimeStr(sG[accessOrderList[i][0]][accessOrderList[i][1]]))- getDateTimeUnix(getTimeStr(sG[accessOrderList[i-1][0]][accessOrderList[i-1][1]])))/60 >= colorApart: + coloredEdges.append(accessOrderList[i]) + else: + regEdges.append(accessOrderList[i]) + + i+=1 + + edge_colors = range(2, len(regEdges) + 2) + edgesR = nx.draw_networkx_edges(sG, pos, edgelist=regEdges, node_size=100, arrowstyle='->', arrowsize=10, edge_color=edge_colors, width=2) + edgesC = nx.draw_networkx_edges(sG, pos, edgelist=coloredEdges, node_size=100, arrowstyle='->', arrowsize=10, edge_color='r', width=2) + else : + edge_colors = range(2, len(accessOrderList) + 2) + edges = nx.draw_networkx_edges(sG, pos, node_size=100, arrowstyle='->', arrowsize=10, edge_color=edge_colors, width=2) edge_labels = nx.get_edge_attributes(sG,'weight') nx.draw_networkx_labels(sG, pos, font_size=10) nx.draw_networkx_edge_labels(sG, pos, edge_labels = edge_labels, font_size=5) + + + ax = plt.gca() ax.set_axis_off() - plt.savefig('case/directed_graph.png') + plt.savefig(caseFile+'/directed_graph.png') + plt.clf() + plt.close() + ax.cla() #line below for debug #plt.show() # Determine HITs, PageRank, Katz Centrality, Degree Centrality - with open("case/graph_analysis.csv", "wb") as fh: + with open(caseFile+"/graph_analysis.csv", "wb") as fh: # Write header row writer=csv.writer(fh) writer.writerow(["NodeID", "PageRank","Hub", "KatzCentrality", "DegreeCentrality"]) @@ -151,8 +263,8 @@ def observe(filename): writer.writerow([k1,v1,v2,v3,v4]) # Observe with Data Mining - df = pd.read_csv('case/graph_analysis.csv') - df.describe().to_csv("case/data_mining.txt") + df = pd.read_csv(caseFile+ '/graph_analysis.csv') + df.describe().to_csv(caseFile+"/data_mining.txt") df.columns = ['NodeID', 'PageRank', 'Hub', 'KatzCentrality', 'DegreeCentrality'] @@ -163,7 +275,10 @@ def observe(filename): pr.set_title("Histogram of PageRank") hb.hist(df.Hub) hb.set_title("Histogram of Hubs") - plt.savefig("case/histo_regression_pagerank_hubs.png") + plt.savefig(caseFile+"/histo_regression_pagerank_hubs.png") + plt.clf() + plt.close() + ax.cla() fig = plt.figure(figsize=(12,6)) kc = fig.add_subplot(121) @@ -172,7 +287,10 @@ def observe(filename): kc.set_title("Histogram of Katz Centrality") dc.hist(df.DegreeCentrality) dc.set_title("Histogram of Degree Centrality") - plt.savefig("case/histo_regression_katzcentrality_degcentrality.png") + plt.savefig(caseFile+"/histo_regression_katzcentrality_degcentrality.png") + plt.clf() + plt.close() + ax.cla() return g @@ -216,17 +334,70 @@ def initPA(): while PAComm: PAComm=raw_input("Please state a command: ") if PAComm == "h": + #print(""" + #is [set] in [path/filename] + # checks if the set [set] is in the observed evidence in the file [path/filename] + #""") print(""" - -is [set] in [path/filename] - checks if the set [set] is in the observed evidence in the file [path/filename] + 1. graph observe [filename] + shows directed graph of [filename] + + 1.1 graph observe [filename] colorApart [minutes] + shows directed graph of [filename] and colors edges that are [minutes] apart from their previous access time. + + 1.2 graph observe [filename] accessOrder + prints the order in which each element was accessed + + 2. graph formulate [source node] + graphs the link of paths in the graph from [source node] + + 3. graph evaluate [source node] [target node] + evaluates source node and target node + + 5. delete all + deletes all case files + """) elif PAComm == "q": break elif PAComm == "exit": break else: - #print "\nCommand Unknown" - parseComm(PAComm) + + id=parseComm(PAComm) + idComponents = id.split() + + if idComponents[0] == "1": + grph = observe(idComponents[1]) + fileName=idComponents[1] + + elif idComponents[0] == "1.1": + grph = observe(idComponents[1], colorApart=int(idComponents[2])) + fileName=idComponents[1] + + elif idComponents[0] == "1.2": + grph = observe(idComponents[1], accessOrder=1) + fileName=idComponents[1] + + elif idComponents[0] == "2": + if not 'grph' in locals(): + print "plase observe evidence (step 1) first" + else: + formulate(idComponents[1], grph, graph=1) + + elif idComponents[0] == "3": + if not 'grph' in locals(): + print "plase observe evidence (step 1) first" + else: + evaluate(idComponents[1], idComponents[2], grph) + + elif idComponents[0] == "5": + deleteCaseFiles() + + + else: + print "\nCommand Unknown" + @@ -247,13 +418,21 @@ def initPA(): filename = raw_input("\nEnter Filename: ") grph = observe(filename) elif sciMthd == "2": - src = raw_input("\nEnter Source Node of your Hypothesis: ") - formulate(src, grph) + if not 'grph' in globals(): + print "plase observe evidence (step 1) first" + else: + src = raw_input("\nEnter Source Node of your Hypothesis: ") + formulate(src, grph) + elif sciMthd == "3": - src = raw_input("\nEnter Source Node of your Hypothesis: ") - trg = raw_input("\nEnter Targe Node of your Hypothesis:\n") - evaluate(src, trg, grph) - elif sciMthd == "4": + if not 'grph' in globals(): + print "plase observe evidence (step 1) first" + else: + src = raw_input("\nEnter Source Node of your Hypothesis: ") + trg = raw_input("\nEnter Targe Node of your Hypothesis:\n") + evaluate(src, trg, grph) + + elif sciMthd == "4" or sciMthd=="q": print "\nExit" break elif sciMthd == "5":