Skip to content

S
SMART
Commit 4ff86012 authored by Alexander Lercher's avatar Alexander Lercher
Browse files
Options

removed multilayer visualization

parent 1f67a66c
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 132954 deletions
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/community_detection/community/__init__.py deleted 100644 → 0
View file @ 1f67a66c
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This package implements community detection.
Package name is community but refer to python-louvain on pypi
"""
# from .community_louvain import (
# partition_at_level,
# modularity,
# best_partition,
# generate_dendrogram,
# induced_graph,
# load_binary,
# )
__author__ = """Thomas Aynaud (thomas.aynaud@lip6.fr)"""
# Copyright (C) 2009 by
# Thomas Aynaud <thomas.aynaud@lip6.fr>
# All rights reserved.
# BSD license.
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/community_detection/community/community_status.py deleted 100644 → 0
View file @ 1f67a66c
# coding=utf-8
class Status(object):
"""
To handle several data in one struct.
Could be replaced by named tuple, but don't want to depend on python 2.6
"""
node2com = {}
total_weight = 0
internals = {}
degrees = {}
gdegrees = {}
def __init__(self):
self.node2com = dict([])
self.total_weight = 0
self.degrees = dict([])
self.gdegrees = dict([])
self.internals = dict([])
self.loops = dict([])
def __str__(self):
return ("node2com : " + str(self.node2com) + " degrees : "
+ str(self.degrees) + " internals : " + str(self.internals)
+ " total_weight : " + str(self.total_weight))
def copy(self):
"""Perform a deep copy of status"""
new_status = Status()
new_status.node2com = self.node2com.copy()
new_status.internals = self.internals.copy()
new_status.degrees = self.degrees.copy()
new_status.gdegrees = self.gdegrees.copy()
new_status.total_weight = self.total_weight
def init(self, graph, weight, part=None):
"""Initialize the status of a graph with every node in one community"""
count = 0
self.node2com = dict([])
self.total_weight = 0
self.degrees = dict([])
self.gdegrees = dict([])
self.internals = dict([])
self.total_weight = graph.size(weight=weight)
if part is None:
for node in graph.nodes():
self.node2com[node] = count
deg = float(graph.degree(node, weight=weight))
if deg < 0:
error = "Bad node degree ({})".format(deg)
raise ValueError(error)
self.degrees[count] = deg
self.gdegrees[node] = deg
edge_data = graph.get_edge_data(node, node, default={weight: 0})
self.loops[node] = float(edge_data.get(weight, 1))
self.internals[count] = self.loops[node]
count += 1
else:
for node in graph.nodes():
com = part[node]
self.node2com[node] = com
deg = float(graph.degree(node, weight=weight))
self.degrees[com] = self.degrees.get(com, 0) + deg
self.gdegrees[node] = deg
inc = 0.
for neighbor, datas in graph[node].items():
edge_weight = datas.get(weight, 1)
if edge_weight <= 0:
error = "Bad graph type ({})".format(type(graph))
raise ValueError(error)
if part[neighbor] == com:
if neighbor == node:
inc += float(edge_weight)
else:
inc += float(edge_weight) / 2.
self.internals[com] = self.internals.get(com, 0) + inc
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/community_detection/community_ranking.py deleted 100644 → 0
View file @ 1f67a66c
## a framework for community-based node ranking
import networkx as nx
import numpy as np
from sklearn.cluster import AffinityPropagation
import multiprocessing as mp
from ..network_ranking.node_ranking import sparse_page_rank,modularity,stochastic_normalization
from scipy.cluster.hierarchy import dendrogram, linkage
from scipy.cluster.hierarchy import fcluster
import scipy.sparse as sp
def page_rank_kernel(index_row):
## call as results = p.map(pr_kernel, batch)
pr = sparse_page_rank(_RANK_GRAPH, [index_row],
epsilon=1e-6,
max_steps=100000,
damping=0.90,
spread_step=10,
spread_percent=0.1,
try_shrink=True)
norm = np.linalg.norm(pr, 2)
if norm > 0:
pr = pr / np.linalg.norm(pr, 2)
return (index_row,pr)
else:
return (index_row,np.zeros(G.shape[1]))
def create_tree(centers):
clusters = {}
to_merge = linkage(centers, method='single')
for i, merge in enumerate(to_merge):
if merge[0] <= len(to_merge):
# if it is an original point read it from the centers array
a = centers[int(merge[0]) - 1]
else:
# other wise read the cluster that has been created
a = clusters[int(merge[0])]
if merge[1] <= len(to_merge):
b = centers[int(merge[1]) - 1]
else:
b = clusters[int(merge[1])]
# the clusters are 1-indexed by scipy
clusters[1 + i + len(to_merge)] = {
'children' : [a, b]
}
# ^ you could optionally store other info here (e.g distances)
return clusters
def return_infomap_communities(network):
infomapWrapper = infomap.Infomap("--two-level --silent")
# Add link weight as an optional third argument
for e in network.edges():
infomapWrapper.addLink(e[0], e[1])
infomapWrapper.run()
tree = infomapWrapper.tree
print("Found %d modules with codelength: %f" % (tree.numTopModules(), tree.codelength()))
print("\n#node module")
part = defaultdict(list)
for node in tree.leafIter():
part[node.moduleIndex()].append(node.physIndex)
return list(part.values())
if __name__ == "__main__":
from infomap import infomap
from collections import defaultdict
from itertools import product
import community
from networkx.algorithms.community import LFR_benchmark_graph
from sklearn.cluster import AffinityPropagation,DBSCAN,MiniBatchKMeans
from scipy import cluster
from scipy.cluster.hierarchy import fcluster
from scipy.spatial.distance import pdist
global _RANK_GRAPH
print("Generating communities..")
n = 500
tau1 = 4
tau2 = 1.5
mu = 0.1
# _RANK_GRAPH = nx.windmill_graph(20, 5)
_RANK_GRAPH = LFR_benchmark_graph(n,
tau1,
tau2,
mu,
average_degree=5,
min_community=30,
seed=10)
print(nx.info(_RANK_GRAPH))
A = _RANK_GRAPH.copy()
_RANK_GRAPH = nx.to_scipy_sparse_matrix(_RANK_GRAPH)
_RANK_GRAPH = stochastic_normalization(_RANK_GRAPH) ## normalize
n = _RANK_GRAPH.shape[1]
with mp.Pool(processes=mp.cpu_count()) as p:
results = p.map(page_rank_kernel,range(n))
vectors = np.zeros((n, n))
for pr_vector in results:
if pr_vector != None:
vectors[pr_vector[0],:] = pr_vector[1]
vectors = np.nan_to_num(vectors)
option = "cpu"
dx_rc = defaultdict(list)
dx_lx = defaultdict(list)
dx_hc = defaultdict(list)
if option == "cpu":
mx_opt = 0
for nclust in range(2,_RANK_GRAPH.shape[0]):
clustering_algorithm = MiniBatchKMeans(n_clusters=nclust)
clusters = clustering_algorithm.fit_predict(vectors)
for a, b in zip(clusters,A.nodes()):
dx_rc[a].append(b)
partitions = dx_rc.values()
mx = modularity(A, partitions, weight='weight')
if mx > mx_opt:
mx_opt = mx
dx_rc = defaultdict(list)
print("KM: {}".format(mx_opt))
Z = linkage(vectors, 'ward')
mod_hc_opt = 0
for nclust in range(3,_RANK_GRAPH.shape[0]):
try:
k = nclust
cls = fcluster(Z, k, criterion='maxclust')
for a,b in zip(cls,A.nodes()):
dx_hc[a].append(b)
partition_hi = dx_hc.values()
mod = modularity(A, partition_hi, weight='weight')
if mod > mod_hc_opt:
mod_hc_opt = mod
except:
pass
print("Hierarchical: {}".format(mod))
## the louvain partition
partition = community.best_partition(A)
for a,b in partition.items():
dx_lx[b].append(a)
partition_louvain = dx_lx.values()
print("Louvain: {}".format(modularity(A, partition_louvain, weight='weight')))
parts_im = return_infomap_communities(A)
print("Infomap: {}".format(modularity(A, parts_im, weight='weight')))
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/community_detection/community_wrapper.py deleted 100644 → 0
View file @ 1f67a66c
## high level interface for community detection algorithms
from .community_louvain import *
def louvain_communities(network,input_type="mat",verbose=True):
# if input_type == "mat":
# network = nx.from_scipy_sparse_matrix(network)
if verbose:
print ("Detecting communities..")
partition = best_partition(network)
return partition
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/network_ranking/node_ranking.py deleted 100644 → 0
View file @ 1f67a66c
## node ranking algorithms
import numpy as np
import networkx as nx
import scipy.sparse as sp
#from networkx.algorithms.community.community_utils import is_partition
from itertools import product
# def stochastic_normalization(matrix):
# matrix = matrix.tolil()
# try:
# matrix.setdiag(0)
# except TypeError:
# matrix.setdiag(np.zeros(matrix.shape[0]))
# matrix = matrix.tocsr()
# d = matrix.sum(axis=1).getA1()
# nzs = np.where(d > 0)
# d[nzs] = 1 / d[nzs]
# matrix = (sp.diags(d, 0).tocsc().dot(matrix)).transpose()
# return matrix
def stochastic_normalization(matrix):
matrix = matrix.tolil()
try:
matrix.setdiag(0)
except TypeError:
matrix.setdiag(np.zeros(matrix.shape[0]))
matrix = matrix.tocsr()
d = matrix.sum(axis=1).getA1()
nzs = np.where(d > 0)
k = 1/d[nzs]
matrix = (sp.diags(k, 0).tocsc().dot(matrix)).transpose()
return matrix
def stochastic_normalization_hin(matrix):
matrix = matrix.tolil()
try:
matrix.setdiag(0)
except TypeError:
matrix.setdiag(np.zeros(matrix.shape[0]))
matrix = matrix.tocsr()
d = matrix.sum(axis=1).getA1()
nzs = np.where(d > 0)
d[nzs] = 1 / d[nzs]
matrix = (sp.diags(d, 0).tocsc().dot(matrix)).transpose()
return matrix
def modularity(G, communities, weight='weight'):
return 1
# if not is_partition(G, communities):
# raise NotAPartition(G, communities)
# multigraph = G.is_multigraph()
# directed = G.is_directed()
# m = G.size(weight=weight)
# if directed:
# out_degree = dict(G.out_degree(weight=weight))
# in_degree = dict(G.in_degree(weight=weight))
# norm = 1 / m
# else:
# out_degree = dict(G.degree(weight=weight))
# in_degree = out_degree
# norm = 1 / (2 * m)
# def val(u, v):
# try:
# if multigraph:
# w = sum(d.get(weight, 1) for k, d in G[u][v].items())
# else:
# w = G[u][v].get(weight, 1)
# except KeyError:
# w = 0
# # Double count self-loops if the graph is undirected.
# if u == v and not directed:
# w *= 2
# return w - in_degree[u] * out_degree[v] * norm
# Q = np.sum(val(u, v) for c in communities for u, v in product(c, repeat=2))
# return Q * norm
def page_rank_kernel(index_row):
## call as results = p.map(pr_kernel, batch)
pr = sparse_page_rank(G, [index_row],
epsilon=1e-6,
max_steps=100000,
damping=damping_hyper,
spread_step=spread_step_hyper,
spread_percent=spread_percent_hyper,
try_shrink=True)
norm = np.linalg.norm(pr, 2)
if norm > 0:
pr = pr / np.linalg.norm(pr, 2)
return (index_row,pr)
else:
return (index_row,np.zeros(graph.shape[1]))
def sparse_page_rank(matrix, start_nodes,
epsilon=1e-6,
max_steps=100000,
damping=0.5,
spread_step=10,
spread_percent=0.3,
try_shrink=False):
assert(len(start_nodes)) > 0
# this method assumes that column sums are all equal to 1 (stochastic normalizaition!)
size = matrix.shape[0]
if start_nodes is None:
start_nodes = range(size)
nz = size
else:
nz = len(start_nodes)
start_vec = np.zeros((size, 1))
start_vec[start_nodes] = 1
start_rank = start_vec / len(start_nodes)
rank_vec = start_vec / len(start_nodes)
# calculate the max spread:
shrink = False
which = np.zeros(0)
if try_shrink:
v = start_vec / len(start_nodes)
steps = 0
while nz < size * spread_percent and steps < spread_step:
steps += 1
v += matrix.dot(v)
nz_new = np.count_nonzero(v)
if nz_new == nz:
shrink = True
break
nz = nz_new
rr = np.arange(matrix.shape[0])
which = (v[rr] > 0).reshape(size)
if shrink:
start_rank = start_rank[which]
rank_vec = rank_vec[which]
matrix = matrix[:, which][which, :]
diff = np.Inf
steps = 0
while diff > epsilon and steps < max_steps: # not converged yet
steps += 1
new_rank = matrix.dot(rank_vec)
rank_sum = np.sum(new_rank)
if rank_sum < 0.999999999:
new_rank += start_rank * (1 - rank_sum)
new_rank = damping * new_rank + (1 - damping) * start_rank
new_diff = np.linalg.norm(rank_vec - new_rank, 1)
diff = new_diff
rank_vec = new_rank
if try_shrink and shrink:
ret = np.zeros(size)
rank_vec = rank_vec.T[0] ## this works for both python versions
ret[which] = rank_vec
ret[start_nodes] = 0
return ret.flatten()
else:
rank_vec[start_nodes] = 0
return rank_vec.flatten()
def hubs_and_authorities(graph):
return nx.hits_scipy(graph)
def hub_matrix(graph):
return nx.hub_matrix(graph)
def authority_matrix(graph):
return nx.authority_matrix(graph)
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/statistics/basic_statistics.py deleted 100644 → 0
View file @ 1f67a66c
## Compute many possible network statistics
import scipy.io
import networkx as nx
import pandas as pd
import numpy as np
from collections import Counter
from operator import itemgetter
def identify_n_hubs(G,top_n=100,node_type=None):
if node_type is not None:
target_nodes = []
for n in G.nodes(data=True):
try:
if n[1]['type'] == node_type:
target_nodes.append(n[0])
except:
pass
else:
target_nodes = G.nodes()
degree_dict = {x : G.degree(x) for x in target_nodes}
top_n_id = {x[0]:x[1] for e,x in enumerate(sorted(degree_dict.items(), key = itemgetter(1), reverse = True)) if e < top_n}
return top_n_id
def core_network_statistics(G,labels=None,name="example"):
rframe = pd.DataFrame(columns=["Name",
"classes",
"nodes",
"edges",
"degree",
"diameter",
"connected components",
"clustering coefficient",
"density",
"flow_hierarchy"])
nodes = len(G.nodes())
edges = len(G.edges())
cc = len(list(nx.connected_components(G.to_undirected())))
try:
cc = nx.average_clustering(G.to_undirected())
except:
cc = None
try:
dx = nx.density(G)
except:
dx = None
clustering = None
if labels is not None:
number_of_classes = labels.shape[1]
else:
number_of_classes = None
node_degree_vector = list(dict(nx.degree(G)).values())
mean_degree = np.mean(node_degree_vector)
try:
diameter = nx.diameter(G)
except:
diameter = "intractable"
try:
flow_hierarchy = nx.flow_hierarchy(G)
except:
flow_hierarchy = "intractable"
point = {"Name": name,
"classes":number_of_classes,
"nodes":nodes,
"edges":edges,
"diameter":diameter,
"degree":mean_degree,
"flow hierarchy":flow_hierarchy,
"connected components":cc,
"clustering coefficient":clustering,
"density":dx}
rframe = rframe.append(point,ignore_index=True)
return rframe
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/statistics/correlation_networks.py deleted 100644 → 0
View file @ 1f67a66c
## implement scale free network estimation
## do this according to the paper WCGA
import numpy as np
from collections import Counter
from scipy import stats
def pick_threshold(matrix):
current_r_opt = 0
rho, pval = stats.spearmanr(matrix)
for j in np.linspace(0,1,50):
tmp_array = rho.copy()
tmp_array[tmp_array > j] = 1
tmp_array[tmp_array < j] = 0
np.fill_diagonal(tmp_array, 0) ## self loops
rw_sum = np.sum(tmp_array,axis=0)
counts = Counter(rw_sum)
key_counts = np.log(list(counts.keys()))
counts = np.log(list(counts.values()))
slope, intercept, r_value, p_value, std_err = stats.linregress(key_counts,counts)
if r_value > current_r_opt:
print("Updating R^2: {}".format(r_value))
current_r_opt = r_value
if r_value > 0.80:
return j
return current_r_opt
def default_correlation_to_network(matrix,input_type="matrix",preprocess="standard"):
if preprocess == "standard":
matrix = (matrix - np.mean(matrix, axis=0)) / np.std(matrix, axis=0)
optimal_threshold = pick_threshold(matrix)
print("Rsq threshold {}".format(optimal_threshold))
matrix[matrix > optimal_threshold] = 1
matrix[matrix < optimal_threshold] = 0
return matrix
if __name__ == "__main__":
from numpy import genfromtxt
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--filename",default="/home/skblaz/Downloads/expression.tsv")
args = parser.parse_args()
datta = args.filename
a = genfromtxt(datta, delimiter='\t',skip_header=4)
a = np.nan_to_num(a)
print("Read the data..")
# idx = np.random.randint(1000,size=5000)
# a = a[:,idx]
print(default_correlation_to_network(a).shape)
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/statistics/enrichment.py deleted 100644 → 0
View file @ 1f67a66c
## compute enrichment via FET
## first for only simple terms,
## continue with communities
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/build/lib/statistics/statistics.py deleted 100644 → 0
View file @ 1f67a66c
## Compute many possible network statistics
import scipy.io
import networkx as nx
import pandas as pd
def core_network_statistics(G,labels=None,name="example"):
rframe = pd.DataFrame(columns=["Name",
"classes",
"nodes",
"edges",
"degree",
"diameter",
"connected components",
"clustering coefficient",
"density",
"flow_hierarchy"])
nodes = len(G.nodes())
edges = len(G.edges())
cc = len(list(nx.connected_components(G.to_undirected())))
try:
cc = nx.average_clustering(G.to_undirected())
except:
cc = None
try:
dx = nx.density(G)
except:
dx = None
clustering = None
if labels is not None:
number_of_classes = labels.shape[1]
else:
number_of_classes = None
mean_degree = np.mean(nx.degree(G).values())
diameter = nx.diameter(G)
flow_hierarchy = nx.flow_hierarchy(G)
point = {"Name": name,
"classes":number_of_classes,
"nodes":nodes,
"edges":edges,
"diameter":diameter,
"degree":mean_degree,
"flow hierarchy":flow_hierarchy,
"connected components":cc,
"clustering coefficient":clustering,
"density":dx}
rframe = rframe.append(point,ignore_index=True)
return rframe
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/community_detection/NoRC.py deleted 100644 → 0
View file @ 1f67a66c
### ncd
import networkx as nx
import numpy as np
import tqdm
from sklearn.cluster import AffinityPropagation
import sklearn.metrics.pairwise
import multiprocessing as mp
from .node_ranking import sparse_page_rank,modularity,stochastic_normalization
from scipy.cluster.hierarchy import dendrogram, linkage
from scipy.spatial.distance import pdist
from scipy.cluster.hierarchy import fcluster
import scipy.sparse as sp
from collections import defaultdict
from itertools import product
#import community
from networkx.algorithms.community import LFR_benchmark_graph
from sklearn.cluster import AffinityPropagation,DBSCAN,MiniBatchKMeans
from scipy import cluster
from scipy.spatial.distance import pdist
global _RANK_GRAPH
def page_rank_kernel(index_row):
## call as results = p.map(pr_kernel, batch)
pr = sparse_page_rank(_RANK_GRAPH, [index_row],
epsilon=1e-6,
max_steps=100000,
damping=0.90,
spread_step=10,
spread_percent=0.1,
try_shrink=True)
norm = np.linalg.norm(pr, 2)
if norm > 0:
pr = pr / np.linalg.norm(pr, 2)
return (index_row,pr)
else:
return (index_row,np.zeros(G.shape[1]))
def create_tree(centers):
clusters = {}
to_merge = linkage(centers, method='single')
for i, merge in enumerate(to_merge):
if merge[0] <= len(to_merge):
# if it is an original point read it from the centers array
a = centers[int(merge[0]) - 1]
else:
# other wise read the cluster that has been created
a = clusters[int(merge[0])]
if merge[1] <= len(to_merge):
b = centers[int(merge[1]) - 1]
else:
b = clusters[int(merge[1])]
# the clusters are 1-indexed by scipy
clusters[1 + i + len(to_merge)] = {
'children' : [a, b]
}
# ^ you could optionally store other info here (e.g distances)
return clusters
def sum(X,v):
rows, cols = X.shape
row_start_stop = as_strided(X.indptr, shape=(rows, 2),
strides=2*X.indptr.strides)
for row, (start, stop) in enumerate(row_start_stop):
data = X.data[start:stop]
data -= v[row]
def NoRC_communities_main(input_graph,clustering_scheme="hierarchical",max_com_num=100,verbose=False,sparisfy=True,parallel_step=6,prob_threshold=0.0005, community_range = [1,3,5,7,11,20,40,50,100,200,300],fine_range=3,lag_threshold=10):
if verbose:
print("Walking..")
global _RANK_GRAPH
_RANK_GRAPH = input_graph
A = _RANK_GRAPH.copy()
_RANK_GRAPH = nx.to_scipy_sparse_matrix(_RANK_GRAPH)
_RANK_GRAPH = stochastic_normalization(_RANK_GRAPH) ## normalize
n = _RANK_GRAPH.shape[1]
edgelist_triplets = []
jobs = [range(n)[i:i + parallel_step] for i in range(0, n, parallel_step)]
with mp.Pool(processes=parallel_step) as p:
for batch in tqdm.tqdm(jobs):
results = p.map(page_rank_kernel,batch)
for nid, result_vector in results:
cols = np.argwhere(result_vector > prob_threshold).flatten().astype(int)
vals = result_vector[cols].flatten()
ixx = np.repeat(nid,cols.shape[0]).flatten().astype(int)
arx = np.vstack((ixx,cols,vals)).T
edgelist_triplets.append(arx)
sparse_edgelist = np.concatenate(edgelist_triplets,axis=0)
print("Compressed to {}% of the initial size".format((sparse_edgelist.shape[0]*100)/n**2))
vectors = sp.coo_matrix((sparse_edgelist[:,2], (sparse_edgelist[:,0].astype(int),sparse_edgelist[:,1].astype(int)))).tocsr()
mx_opt = 0
if clustering_scheme == "kmeans":
if verbose:
print("Doing kmeans search")
nopt = 0
lag_num = 0
for nclust in tqdm.tqdm(community_range):
dx_hc = defaultdict(list)
clustering_algorithm = MiniBatchKMeans(n_clusters=nclust)
clusters = clustering_algorithm.fit_predict(vectors)
for a, b in zip(clusters,A.nodes()):
dx_hc[a].append(b)
partitions = dx_hc.values()
mx = modularity(A, partitions, weight='weight')
if mx > mx_opt:
lag_num = 0
if verbose:
print("Improved modularity: {}, found {} communities.".format(mx,len(partitions)))
mx_opt = mx
opt_clust = dx_hc
nopt = nclust
if mx == 1:
nopt = nclust
return opt_clust
else:
lag_num+=1
if verbose:
print("No improvement for {} iterations.".format(lag_num))
if lag_num > lag_threshold:
break
## fine grained search
if verbose:
print("Fine graining around {}".format(nopt))
for nclust in range(nopt-fine_range,nopt+fine_range,1):
if nclust != nopt:
dx_hc = defaultdict(list)
clustering_algorithm = MiniBatchKMeans(n_clusters=nclust)
clusters = clustering_algorithm.fit_predict(vectors)
for a, b in zip(clusters,A.nodes()):
dx_hc[a].append(b)
partitions = dx_hc.values()
mx = modularity(A, partitions, weight='weight')
if mx > mx_opt:
if verbose:
print("Improved modularity: {}, found {} communities.".format(mx,len(partitions)))
mx_opt = mx
opt_clust = dx_hc
if mx == 1:
nopt = nclust
return opt_clust
return opt_clust
if clustering_scheme == "hierarchical":
Z = linkage(vectors.todense(), 'average')
mod_hc_opt = 0
for nclust in tqdm.tqdm(community_range):
dx_hc = defaultdict(list)
try:
cls = fcluster(Z, nclust, criterion='maxclust')
for a,b in zip(cls,A.nodes()):
dx_hc[a].append(b)
partition_hi = dx_hc.values()
mod = modularity(A, partition_hi, weight='weight')
if mod > mod_hc_opt:
if verbose:
print("\nImproved modularity: {}, communities: {}".format(mod, len(partition_hi)))
mod_hc_opt = mod
opt_clust = dx_hc
if mod == 1:
return opt_clust
except Exception as es:
print (es)
return opt_clust
if __name__ == "__main__":
# n = 50
# tau1 = 4
# tau2 = 1.5
# mu = 0.1
# graph = LFR_benchmark_graph(n,
# tau1,
# tau2,
# mu,
# average_degree=5,
# min_community=30,
# seed=10)
graph = nx.powerlaw_cluster_graph(1000,5,0.1)
print(nx.info(graph))
communities1 = NoRC_communities_main(graph,verbose=True,clustering_scheme="kmeans")
communities1 = NoRC_communities_main(graph,verbose=True,clustering_scheme="hierarchical")
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/community_detection/build/lib/community/__init__.py deleted 100644 → 0
View file @ 1f67a66c
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This package implements community detection.
Package name is community but refer to python-louvain on pypi
"""
# from .community_louvain import (
# partition_at_level,
# modularity,
# best_partition,
# generate_dendrogram,
# induced_graph,
# load_binary,
# )
__author__ = """Thomas Aynaud (thomas.aynaud@lip6.fr)"""
# Copyright (C) 2009 by
# Thomas Aynaud <thomas.aynaud@lip6.fr>
# All rights reserved.
# BSD license.
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/community_detection/build/lib/community/community_status.py deleted 100644 → 0
View file @ 1f67a66c
# coding=utf-8
class Status(object):
"""
To handle several data in one struct.
Could be replaced by named tuple, but don't want to depend on python 2.6
"""
node2com = {}
total_weight = 0
internals = {}
degrees = {}
gdegrees = {}
def __init__(self):
self.node2com = dict([])
self.total_weight = 0
self.degrees = dict([])
self.gdegrees = dict([])
self.internals = dict([])
self.loops = dict([])
def __str__(self):
return ("node2com : " + str(self.node2com) + " degrees : "
+ str(self.degrees) + " internals : " + str(self.internals)
+ " total_weight : " + str(self.total_weight))
def copy(self):
"""Perform a deep copy of status"""
new_status = Status()
new_status.node2com = self.node2com.copy()
new_status.internals = self.internals.copy()
new_status.degrees = self.degrees.copy()
new_status.gdegrees = self.gdegrees.copy()
new_status.total_weight = self.total_weight
def init(self, graph, weight, part=None):
"""Initialize the status of a graph with every node in one community"""
count = 0
self.node2com = dict([])
self.total_weight = 0
self.degrees = dict([])
self.gdegrees = dict([])
self.internals = dict([])
self.total_weight = graph.size(weight=weight)
if part is None:
for node in graph.nodes():
self.node2com[node] = count
deg = float(graph.degree(node, weight=weight))
if deg < 0:
error = "Bad node degree ({})".format(deg)
raise ValueError(error)
self.degrees[count] = deg
self.gdegrees[node] = deg
edge_data = graph.get_edge_data(node, node, default={weight: 0})
self.loops[node] = float(edge_data.get(weight, 1))
self.internals[count] = self.loops[node]
count += 1
else:
for node in graph.nodes():
com = part[node]
self.node2com[node] = com
deg = float(graph.degree(node, weight=weight))
self.degrees[com] = self.degrees.get(com, 0) + deg
self.gdegrees[node] = deg
inc = 0.
for neighbor, datas in graph[node].items():
edge_weight = datas.get(weight, 1)
if edge_weight <= 0:
error = "Bad graph type ({})".format(type(graph))
raise ValueError(error)
if part[neighbor] == com:
if neighbor == node:
inc += float(edge_weight)
else:
inc += float(edge_weight) / 2.
self.internals[com] = self.internals.get(com, 0) + inc
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/community_detection/community/__init__.py deleted 100644 → 0
View file @ 1f67a66c
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This package implements community detection.
Package name is community but refer to python-louvain on pypi
"""
# from .community_louvain import (
# partition_at_level,
# modularity,
# best_partition,
# generate_dendrogram,
# induced_graph,
# load_binary,
# )
__author__ = """Thomas Aynaud (thomas.aynaud@lip6.fr)"""
# Copyright (C) 2009 by
# Thomas Aynaud <thomas.aynaud@lip6.fr>
# All rights reserved.
# BSD license.
src/data-hub/semantic-linking-microservice/app/multilayer/algorithms/community_detection/community_measures.py deleted 100644 → 0
View file @ 1f67a66c
## a set of measures for assessing community quality
import numpy as np
from sklearn.metrics import silhouette_score
from itertools import product
def modularity(G, communities, weight='weight'):
communities = list(communities.values())
multigraph = G.is_multigraph()
directed = G.is_directed()
m = G.size(weight=weight)
if directed:
out_degree = dict(G.out_degree(weight=weight))
in_degree = dict(G.in_degree(weight=weight))
norm = 1 / m
else:
out_degree = dict(G.degree(weight=weight))
in_degree = out_degree
norm = 1 / (2 * m)
def val(u, v):
try:
if multigraph:
w = sum(d.get(weight, 1) for k, d in G[u][v].items())
else:
w = G[u][v].get(weight, 1)
except KeyError:
w = 0
# Double count self-loops if the graph is undirected.
if u == v and not directed:
w *= 2
return w - in_degree[u] * out_degree[v] * norm
Q = np.sum(val(u, v) for c in communities for u, v in product(c, repeat=2))
return Q * norm
def size_distribution(network_partition):
return np.array([len(x) for x in network_partition.values()])
def number_of_communities(network_partition):
return len(network_partition)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment