Commit b5674148 authored by Christian Arnold's avatar Christian Arnold
Browse files

GRaNIEdev: Edge cases for network analyses

parent 3a32c960
Pipeline #28027 passed with stage
in 15 seconds
Package: GRaNIEdev
Title: GRaNIE: Reconstruction cell type specific gene regulatory networks including enhancers using chromatin accessibility and RNA-seq data
Version: 0.13.10
Version: 0.13.11
Encoding: UTF-8
Authors@R: c(person("Christian", "Arnold", email =
"christian.arnold@embl.de", role = c("cre","aut")),
......
......@@ -4303,6 +4303,8 @@ performAllNetworkAnalyses <- function(GRN, ontology = c("BP", "MF"),
GRN = plotCommunitiesEnrichment(GRN, outputFolder = outputFolder, display = display, communities = communities, forceRerun = forceRerun)
.printExecutionTime(start)
GRN
......@@ -4457,24 +4459,34 @@ calculateGeneralEnrichment <- function(GRN, ontology = c("BP", "MF"),
result.list = list()
geneList = factor(as.integer(unique(background) %in% unique(foreground)))
names(geneList) = unique(background)
for (ontology_cur in ontology){
data = suppressMessages(new("topGOdata",
ontology = ontology_cur,
allGenes = geneList,
description = description,
nodeSize = 5,
annot = topGO::annFUN.org,
mapping = mapping,
ID = "ensembl"))
result = suppressMessages(topGO::runTest(data, algorithm = algorithm, statistic = statistic))
# Dont trim GO terms here, happens later when plotting
result.tbl = unique(topGO::GenTable(data, pval = result, orderBy = "pval", numChar = 1000,
topNodes = length(topGO::score(result))) )
result.tbl$GeneRatio = result.tbl$Significant / length(unique(foreground))
if (length(geneList == 0)) {
result.tbl = NULL
} else {
data = suppressMessages(new("topGOdata",
ontology = ontology_cur,
allGenes = geneList,
description = description,
nodeSize = 5,
annot = topGO::annFUN.org,
mapping = mapping,
ID = "ensembl"))
result = suppressMessages(topGO::runTest(data, algorithm = algorithm, statistic = statistic))
# Dont trim GO terms here, happens later when plotting
result.tbl = unique(topGO::GenTable(data, pval = result, orderBy = "pval", numChar = 1000,
topNodes = length(topGO::score(result))) )
result.tbl$GeneRatio = result.tbl$Significant / length(unique(foreground))
}
result.list[["results"]][[ontology_cur]] = result.tbl
result.list[["parameters"]][[ontology_cur]] = stats::setNames(c(statistic, algorithm, backgroundStr), c("statistic", "algorithm", "background"))
......@@ -4533,61 +4545,70 @@ calculateCommunitiesStats <- function(GRN, clustering = "louvain", forceRerun =
futile.logger::flog.info(paste0("Calculating communities for clustering type ", clustering, "..."))
TF_gene.df = .generateGraphTables(GRN)$TF_gene.df
graph = igraph::graph_from_data_frame(d = TF_gene.df, directed = FALSE)
if (clustering == "louvain") {
communities_cluster = igraph::cluster_louvain(graph, weights = NA)
} else if (clustering == "leading_eigen") {
communities_cluster = igraph::cluster_leading_eigen(graph, ...)
} else if (clustering == "fast_greedy") {
communities_cluster = igraph::cluster_fast_greedy(graph, ...)
} else if (clustering == "optimal") {
communities_cluster = igraph::cluster_optimal(graph, ...)
} else if (clustering == "walktrap") {
communities_cluster = igraph::cluster_walktrap(graph, ...)
}
GRN@stats$graph_TF_gene = graph
GRN@stats$communityCluster_obj = communities_cluster
communities_count = sort(table(communities_cluster$membership), decreasing = TRUE)
communityVertices = tibble::tibble(vertex = communities_cluster$names,
community = factor(communities_cluster$membership, levels = names(communities_count)),
Class = dplyr::case_when(grepl("^ENS", vertex) ~ "gene",
TRUE ~ "TF"))
GRN@stats$communityVertices = communityVertices
communityGraphs = matrix(ncol=3, nrow =0, dimnames = list(c(), c("V1", "V2", "community")))
for (communityCur in stats::na.omit(names(communities_count))){ # change this to select communities
community_subgraph.df =
igraph::induced_subgraph(graph = graph,
vids = communityVertices$vertex[communityVertices$community==communityCur]) %>%
igraph::as_long_data_frame() %>%
dplyr::select(c(4,5,3)) %>%
dplyr::rename(V1 = names(.)[1], V2 = names(.)[2]) %>%
dplyr::mutate(community = communityCur)
communityGraphs = rbind(communityGraphs, community_subgraph.df[,c("V1", "V2", "community")])
if (nrow(TF_gene.df) == 0) {
GRN@stats$communities = NULL
GRN@stats$communityVertices = NULL
} else {
graph = igraph::graph_from_data_frame(d = TF_gene.df, directed = FALSE)
if (clustering == "louvain") {
communities_cluster = igraph::cluster_louvain(graph, weights = NA)
} else if (clustering == "leading_eigen") {
communities_cluster = igraph::cluster_leading_eigen(graph, ...)
} else if (clustering == "fast_greedy") {
communities_cluster = igraph::cluster_fast_greedy(graph, ...)
} else if (clustering == "optimal") {
communities_cluster = igraph::cluster_optimal(graph, ...)
} else if (clustering == "walktrap") {
communities_cluster = igraph::cluster_walktrap(graph, ...)
}
GRN@stats$graph_TF_gene = graph
GRN@stats$communityCluster_obj = communities_cluster
communities_count = sort(table(communities_cluster$membership), decreasing = TRUE)
communityVertices = tibble::tibble(vertex = communities_cluster$names,
community = factor(communities_cluster$membership, levels = names(communities_count)),
Class = dplyr::case_when(grepl("^ENS", vertex) ~ "gene",
TRUE ~ "TF"))
GRN@stats$communityVertices = communityVertices
communityGraphs = matrix(ncol=3, nrow =0, dimnames = list(c(), c("V1", "V2", "community")))
for (communityCur in stats::na.omit(names(communities_count))){ # change this to select communities
community_subgraph.df =
igraph::induced_subgraph(graph = graph,
vids = communityVertices$vertex[communityVertices$community==communityCur]) %>%
igraph::as_long_data_frame() %>%
dplyr::select(c(4,5,3)) %>%
dplyr::rename(V1 = names(.)[1], V2 = names(.)[2]) %>%
dplyr::mutate(community = communityCur)
communityGraphs = rbind(communityGraphs, community_subgraph.df[,c("V1", "V2", "community")])
}
GRN@stats$communities = communityGraphs %>% dplyr::mutate(community = as.factor(community))
}
GRN@stats$communities = communityGraphs %>% dplyr::mutate(community = as.factor(community))
} else {
futile.logger::flog.info(paste0("Data already exists in object, nothing to do"))
......@@ -4628,11 +4649,12 @@ calculateCommunitiesEnrichment <- function(GRN,
checkmate::assertSubset(selection, c("byRank", "byLabel"))
checkmate::assertFlag(forceRerun)
if (is.null(GRN@stats$communities)){
#message = "Could not find community results in GRN object. Make sure to generate the communities first using the communitiesStats function."
message = "Communities have not been previously calculated it. Calculating communities using the default parameters. \nIf you wish to modify those parameters run the communitiesStats function first."
.checkAndLogWarningsAndErrors(NULL, message, isWarning = TRUE)
GRN = calculateCommunitiesStats(GRN)
# If still NULL, no communities were found
if (is.null(GRN@stats$communities)) {
message = "No communities found, cannot calculate enrichment. Run the functioncalculateCommunitiesStats first. If you did already, it looks like no communities could be identified before"
.checkAndLogWarningsAndErrors(NULL, message, isWarning = TRUE)
return (GRN)
}
if (selection == "byLabel"){
......
......@@ -2019,56 +2019,76 @@ plotGeneralGraphStats <- function(GRN, outputFolder = NULL, forceRerun = FALSE)
ggtitle(paste0("Vertices (n=", sum(totalVerteces$Count), ")"))
geneDist = as.data.frame(table(droplevels(GRN@connections$all.filtered$`0`$gene.type)))
gGeneDist = ggplot(geneDist, aes(x= "", y = Freq, fill = Var1)) + geom_bar(stat="identity") +
coord_polar("y") +
geom_text(aes(label = paste0(round(Freq/sum(Freq) *100, digits = 1 ), "%")),
position = position_stack(vjust = 0.5)) +
theme_plots +
ggtitle(paste0("Genes (n=", sum(geneDist$Freq), ")"))
if (nrow(geneDist) == 0) {
gGeneDist = ggplot() + annotate("text", x = 4, y = 25, size=5, color = "red", label = "Nothing to plot:\nNo genes found") +
theme_void()
message = "No genes found in the GRN object. Make sure the filtered connections contain also genes."
.checkAndLogWarningsAndErrors(NULL, message, isWarning = TRUE)
} else {
gGeneDist = ggplot(geneDist, aes(x= "", y = Freq, fill = Var1)) + geom_bar(stat="identity") +
coord_polar("y") +
geom_text(aes(label = paste0(round(Freq/sum(Freq) *100, digits = 1 ), "%")),
position = position_stack(vjust = 0.5)) +
theme_plots +
ggtitle(paste0("Genes (n=", sum(geneDist$Freq), ")"))
}
totalEdges = as.data.frame(table(TF_peak_gene.df$connectionType))
gEdgeDist = ggplot(totalEdges, aes(x="", y=Freq, fill=Var1)) + geom_bar(stat="identity") +
coord_polar("y", start=0) +
scale_fill_manual(values = c("#BDC367", "#6BB1C1")) +
geom_text(aes(label = paste0(round(Freq/sum(Freq) *100, digits = 1 ), "%")),
position = position_stack(vjust = 0.5)) +
theme_plots +
ggtitle(paste0("Edges (n=", sum(totalEdges$Freq), ")"))
if (nrow(totalEdges) == 0) {
gEdgeDist= ggplot() + annotate("text", x = 4, y = 25, size=5, color = "red", label = "Nothing to plot:\nNo genes found") +
theme_void()
} else {
gEdgeDist = ggplot(totalEdges, aes(x="", y=Freq, fill=Var1)) + geom_bar(stat="identity") +
coord_polar("y", start=0) +
scale_fill_manual(values = c("#BDC367", "#6BB1C1")) +
geom_text(aes(label = paste0(round(Freq/sum(Freq) *100, digits = 1 ), "%")),
position = position_stack(vjust = 0.5)) +
theme_plots +
ggtitle(paste0("Edges (n=", sum(totalEdges$Freq), ")"))
}
print((gVertexDist + gEdgeDist)/gGeneDist + patchwork::plot_layout(widths = c(2,1), guides = "collect"))
# Get degree stats and central verteces in the TF-peak-gene graph
TF_peak_gene.degree.stats = .getDegreeStats(GRN, TF_peak_gene.df)
#degreeDist.tbl = TF_peak_gene.degree.stats$tbl$degrees
gDegrees = TF_peak_gene.degree.stats$figures$degreeDist + ggtitle("Distribution of vertex degrees in the filtered GRN")
gTopGenes = TF_peak_gene.degree.stats$figures$topGenes + ggtitle("Top degree-central genes in the filtered GRN")
gTopTFs = TF_peak_gene.degree.stats$figures$topTFs + ggtitle("Top degree-central TFs in the filtered GRN")
eigen_stats = .getEigenCentralVertices(GRN, TF_peak_gene.df)
gTopEigenGenes = eigen_stats$topGenes + ggtitle("Top eigenvector-central genes in the filtered GRN")
gTopEigenTFs = eigen_stats$topTFs + ggtitle("Top eigenvector-central TFs in the filtered GRN")
print(gDegrees)
print(gTopGenes/gTopTFs)
print(gTopEigenGenes/gTopEigenTFs)
# Get degree stats and central verteces in the filtered GRN
TF_gene.degree.stats = .getDegreeStats(GRN, TF_gene.df)
degreeDist.tbl = TF_gene.degree.stats$tbl$degrees
gDegrees = TF_gene.degree.stats$figures$degreeDist + ggtitle("Distribution of Vertex Degrees in the filtered GRN")
gTopGenes = TF_gene.degree.stats$figures$topGenes + ggtitle("Top degree-central genes in the filtered GRN")
gTopTFs = TF_gene.degree.stats$figures$topTFs + ggtitle("Top degree-central TFs in the filtered GRN")
# get top eigenvalue central TFs and genes
eigen_stats = .getEigenCentralVertices(GRN, TF_peak_gene.df)
gTopEigenGenes = eigen_stats$topGenes + ggtitle("Top eigenvector-central genes in the filtered GRN")
gTopEigenTFs = eigen_stats$topTFs + ggtitle("Top eigenvector-central TFs in the filtered GRN")
if (nrow(degreeDist.tbl) > 0) {
#degreeDist.tbl = TF_peak_gene.degree.stats$tbl$degrees
gDegrees = TF_peak_gene.degree.stats$figures$degreeDist + ggtitle("Distribution of vertex degrees in the filtered GRN")
gTopGenes = TF_peak_gene.degree.stats$figures$topGenes + ggtitle("Top degree-central genes in the filtered GRN")
gTopTFs = TF_peak_gene.degree.stats$figures$topTFs + ggtitle("Top degree-central TFs in the filtered GRN")
eigen_stats = .getEigenCentralVertices(GRN, TF_peak_gene.df)
gTopEigenGenes = eigen_stats$topGenes + ggtitle("Top eigenvector-central genes in the filtered GRN")
gTopEigenTFs = eigen_stats$topTFs + ggtitle("Top eigenvector-central TFs in the filtered GRN")
print(gDegrees)
print(gTopGenes/gTopTFs)
print(gTopEigenGenes/gTopEigenTFs)
# Get degree stats and central verteces in the filtered GRN
TF_gene.degree.stats = .getDegreeStats(GRN, TF_gene.df)
degreeDist.tbl = TF_gene.degree.stats$tbl$degrees
gDegrees = TF_gene.degree.stats$figures$degreeDist + ggtitle("Distribution of Vertex Degrees in the filtered GRN")
gTopGenes = TF_gene.degree.stats$figures$topGenes + ggtitle("Top degree-central genes in the filtered GRN")
gTopTFs = TF_gene.degree.stats$figures$topTFs + ggtitle("Top degree-central TFs in the filtered GRN")
# get top eigenvalue central TFs and genes
eigen_stats = .getEigenCentralVertices(GRN, TF_peak_gene.df)
gTopEigenGenes = eigen_stats$topGenes + ggtitle("Top eigenvector-central genes in the filtered GRN")
gTopEigenTFs = eigen_stats$topTFs + ggtitle("Top eigenvector-central TFs in the filtered GRN")
print(gDegrees)
print(gTopGenes/gTopTFs)
print(gTopEigenGenes/gTopEigenTFs)
}
print(gDegrees)
print(gTopGenes/gTopTFs)
print(gTopEigenGenes/gTopEigenTFs)
grDevices::dev.off()
......@@ -2216,57 +2236,70 @@ plotCommunitiesStats <- function(GRN, outputFolder = NULL, display = "byRank", c
futile.logger::flog.info(paste0("Plotting community statistics to file ", fileCur))
grDevices::pdf(fileCur, width = 10)
grDevices::pdf(fileCur, width = 10)
if (!is.null(communities)) {
if (display == "byRank"){
# Only display communities we have data for, in a reasonable order
communitiesDisplay = stats::na.omit(names(table(GRN@stats$communityVertices$community)[communities]))
} else{ # byLabel
communitiesDisplay = as.character(communities)
}
if (is.null(GRN@stats$communityVertices)) {
g = ggplot() + annotate("text", x = 4, y = 25, size=5, color = "red",
label = "Nothing to plot:\nNo communities found") + theme_void()
plot(g)
message = "No communities found in GRN object. Make sure the filtered connections contain also genes."
.checkAndLogWarningsAndErrors(NULL, message, isWarning = TRUE)
} else {
communitiesDisplay = unique(GRN@stats$communityVertices$community)
}
communityVertices = GRN@stats$communityVertices %>%
dplyr::filter(community %in% communitiesDisplay)
gCommunityVertices = ggplot(communityVertices, aes(x = community, fill = Class)) +
geom_bar(position = "stack") +
ggtitle("Vertices per community") +
xlab("community") +
ylab("vertex count") +
scale_fill_manual(values = c("#3B9AB2", "#E1AF00"))
plot(gCommunityVertices)
for (communityCur in as.character(communitiesDisplay)) { # change this to select communities
community_subgraph.df =
igraph::induced_subgraph(graph = GRN@stats$graph_TF_gene,
vids = communityVertices$vertex[as.character(communityVertices$community)==communityCur]) %>%
igraph::as_long_data_frame() %>%
dplyr::select(c(4,5,3)) %>%
dplyr::rename(V1 = names(.)[1], V2 = names(.)[2]) %>%
dplyr::mutate(community = communityCur)
community.degreeStats = .getDegreeStats(GRN, community_subgraph.df)
gDegreeDist = community.degreeStats$figures$degreeDist + ggtitle("Degree Distribution")
gTopGenes = community.degreeStats$figures$topGenes + ggtitle("Top degree-central genes")
gTopTFs = community.degreeStats$figures$topTFs + ggtitle("Top degree-central TFs")
community.eigenStats = .getEigenCentralVertices(GRN, df= community_subgraph.df)
gTopEigenGenes = community.eigenStats$topGenes + ggtitle("Top eigenvector-central genes")
gTopEigenTFs = community.eigenStats$topTFs + ggtitle("Top eigenvector-central genes")
plot(gDegreeDist + (gTopGenes/gTopTFs) +
patchwork::plot_layout(widths = c(2,2)) +
patchwork::plot_annotation(title = paste0("Community ", communityCur)) )
plot(gTopEigenGenes/gTopEigenTFs)
if (!is.null(communities)) {
if (display == "byRank"){
# Only display communities we have data for, in a reasonable order
communitiesDisplay = stats::na.omit(names(table(GRN@stats$communityVertices$community)[communities]))
} else{ # byLabel
communitiesDisplay = as.character(communities)
}
} else {
communitiesDisplay = unique(GRN@stats$communityVertices$community)
}
communityVertices = GRN@stats$communityVertices %>%
dplyr::filter(community %in% communitiesDisplay)
gCommunityVertices = ggplot(communityVertices, aes(x = community, fill = Class)) +
geom_bar(position = "stack") +
ggtitle("Vertices per community") +
xlab("community") +
ylab("vertex count") +
scale_fill_manual(values = c("#3B9AB2", "#E1AF00"))
plot(gCommunityVertices)
for (communityCur in as.character(communitiesDisplay)) { # change this to select communities
community_subgraph.df =
igraph::induced_subgraph(graph = GRN@stats$graph_TF_gene,
vids = communityVertices$vertex[as.character(communityVertices$community)==communityCur]) %>%
igraph::as_long_data_frame() %>%
dplyr::select(c(4,5,3)) %>%
dplyr::rename(V1 = names(.)[1], V2 = names(.)[2]) %>%
dplyr::mutate(community = communityCur)
community.degreeStats = .getDegreeStats(GRN, community_subgraph.df)
gDegreeDist = community.degreeStats$figures$degreeDist + ggtitle("Degree Distribution")
gTopGenes = community.degreeStats$figures$topGenes + ggtitle("Top degree-central genes")
gTopTFs = community.degreeStats$figures$topTFs + ggtitle("Top degree-central TFs")
community.eigenStats = .getEigenCentralVertices(GRN, df= community_subgraph.df)
gTopEigenGenes = community.eigenStats$topGenes + ggtitle("Top eigenvector-central genes")
gTopEigenTFs = community.eigenStats$topTFs + ggtitle("Top eigenvector-central genes")
plot(gDegreeDist + (gTopGenes/gTopTFs) +
patchwork::plot_layout(widths = c(2,2)) +
patchwork::plot_annotation(title = paste0("Community ", communityCur)) )
plot(gTopEigenGenes/gTopEigenTFs)
}
}
grDevices::dev.off()
} else {
......@@ -2313,8 +2346,9 @@ plotCommunitiesEnrichment <- function(GRN, outputFolder = NULL,
checkmate::assert(checkmate::testNull(outputFolder), checkmate::testDirectoryExists(outputFolder))
if (is.null(GRN@stats$communities)){
message = "No communities have been previously generated. TODO better message."
.checkAndLogWarningsAndErrors(NULL, message, isWarning = FALSE)
message = "No communities have been previously generated or none were found. make sure to run the function calculateCommunitiesEnrichment first."
.checkAndLogWarningsAndErrors(NULL, message, isWarning = TRUE)
return(GRN)
}
outputFolder = .checkOutputFolder(GRN, outputFolder)
......
......@@ -6,14 +6,14 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Page not found (404) • GRNdev</title>
<title>Page not found (404) • GRaNIEdev</title>
<!-- jquery -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<!-- Bootstrap -->
<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/flatly/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/css/bootstrap.min.css" integrity="sha256-bZLfwXAP04zRMK2BjiO8iu9pf4FbLqX6zitd+tIvLhE=" crossorigin="anonymous" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
......@@ -54,16 +54,6 @@
<![endif]-->
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=G-530L9SXFM1"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'G-530L9SXFM1');
</script>
</head>
......@@ -80,31 +70,34 @@
<span class="icon-bar"></span>
</button>
<span class="navbar-brand">
<a class="navbar-link" href="index.html">GRNdev</a>
<span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">0.13.4</span>
<a class="navbar-link" href="index.html">GRaNIEdev</a>
<span class="version label label-default" data-toggle="tooltip" data-placement="bottom" title="Released version">0.13.10</span>
</span>
</div>
<div id="navbar" class="navbar-collapse collapse">
<ul class="nav navbar-nav">
<li>
<a href="index.html"></a>
<a href="index.html">
<span class="fas fa-home fa-lg"></span>
</a>
</li>
<li>
<a href="articles/quickStart.html">Getting Started</a>
<a href="reference/index.html">Reference</a>
</li>
<li class="dropdown">
<a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
Vignettes
Articles
<span class="caret"></span>
</a>
<ul class="dropdown-menu" role="menu">
<li>
<a href="articles/quickStart.html">Getting Started</a>
<a href="articles/Introduction.html">Introduction and Methodological Details</a>
</li>
<li>
<a href="articles/Introduction.html">Introduction</a>
<a href="articles/quickStart.html">Get Started with the *GRaNIE* packages from the Zaugg Lab</a>
</li>
<li>
<a href="articles/workflow.html">Workflow example</a>
......@@ -112,10 +105,7 @@
</ul>
</li>
<li>
<a href="reference/index.html">Reference</a>
</li>
<li>
<a href="news/index.html">Changelog &amp; News</a>
<a href="news/index.html">Changelog</a>
</li>
</ul>
<ul class="nav navbar-nav navbar-right">
......@@ -152,7 +142,7 @@ Content not found. Please use links in the navbar.
<footer>
<div class="copyright">
<p>Developed by Christian Arnold, Judith Zaugg.</p>
<p>Developed by Christian Arnold, Rim Moussa.</p>
</div>
<div class="pkgdown">
......
......@@ -6,14 +6,14 @@
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Articles • GRNdev</title>
<title>Articles • GRaNIEdev</title>
<!-- jquery -->
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.4.1/jquery.min.js" integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo=" crossorigin="anonymous"></script>
<!-- Bootstrap -->
<link href="https://cdnjs.cloudflare.com/ajax/libs/bootswatch/3.4.0/flatly/bootstrap.min.css" rel="stylesheet" crossorigin="anonymous" />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/css/bootstrap.min.css" integrity="sha256-bZLfwXAP04zRMK2BjiO8iu9pf4FbLqX6zitd+tIvLhE=" crossorigin="anonymous" />
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha256-nuL8/2cJ5NDSSwnKD8VqreErSWHtnEP9E7AySL+1ev4=" crossorigin="anonymous"></script>
......@@ -54,16 +54,6 @@
<![endif]-->