Version 1.1.6, see Changelog for details

VERSION

-1.1.5
+1.1.6

docs/projectInfo.rst

@@ -31,6 +31,12 @@ We also put the paper on *bioRxiv*, please read all methodological details here:
 Change log
 ============================
 
+Version 1.1.6 (2018-10-11)
+    - fixed small issue in ``checkParameterValidity.R`` when not having sufficient permissions for the folder in which the fasta file is located
+    - updated the ``summaryFinal.R`` script. Now, for the Volcano plot PDF, in addition to adj. p-values, also the raw p-values are plotted in the end. This might be helpful for datasets with small signal when no adj. p-value is significant. In addition, labeling of TFs is now skipped when the number of TFs to label exceeds 150. THis makes the step faster and the PDF smaller and less crowded.
+    - small updates to the translation table for mm10
+    - adding two local rules to the Snakefile for potential minor speed improvements when running in cluster mode
+
 Version 1.1.5 (2018-08-14)
     - optimized ``checkParameterValidity.R`` script, only TFBS files for TFs included in the analysis are now checked
     - addressed an R library compatibility issue independent of *diffTF* that users reported. In some cases, for particular versions of R and Bioconductor, R exited with a *segfault* (memory not mapped) error in the ``checkParameterValidity.R`` that seems to be caused by the combination of *DiffBind* and *DESeq2*. Specifically, when *DiffBind* is loaded *before* *DESeq2*, R crashes with a segmentation fault upon exiting, whereas loading *DiffBind* *after* *DESeq2* causes no issue. If there are further issues, please let us know. Thanks to Gyan Prakash Mishra, who first reported this.

src/R/analyzeTF.R

@@ -210,8 +210,7 @@ nTFBS = nrow(overlapsAll.df)
 if (nTFBS >= par.l$minNoDatapoints) {
 
   
-  # Group by peak ID
-  # take only the maximum row mean of all samples, sample with biggest coverage
+  # Group by peak ID: To avoid biases and dependencies based on TFBS clustering within peaks, we then select the TFBS per TF per peak with the highest average read count across all samples.
   coverageAll_grouped.df = overlapsAll.df %>%
     dplyr::group_by(peakID) %>%
     dplyr::slice(which.max(mean)) %>%
@@ -395,7 +394,12 @@ if (skipTF) {
         }
         )
         
-        if (class(res_DESeq) == "character") skipTF = TRUE
+        if (class(res_DESeq) == "character") {
+          
+          skipTF = TRUE
+          TF_outputInclPerm.df = as.data.frame(matrix(nrow = 0, ncol = 2 + par.l$nPermutations + 1))
+          colnames(TF_outputInclPerm.df) = c("TF", "TFBSID", paste0("log2fc_perm", 0:par.l$nPermutations))
+        }
         
         if (!skipTF) {
             res_DESeq.df <- as.data.frame(DESeq2::results(res_DESeq))

src/R/checkParameterValidity.R

@@ -52,7 +52,7 @@ assertDirectoryExists(dirname(TFBS_dir), access = "r")
 
 fastaFile = snakemake@config$additionalInputFiles$refGenome_fasta
 assertFileExists(fastaFile)
-assertDirectoryExists(dirname(fastaFile), access = "w")
+assertDirectoryExists(dirname(fastaFile), access = "r")
 
 allTFs = strsplit(snakemake@config$par_general$TFs, ",")[[1]]
 

src/R/summaryFinal.R

@@ -47,6 +47,10 @@ par.l$extension_x_limits = 0.15 # 15 % x axis extension, regardless of the limit
 par.l$extension_y_limits = 1.1
 par.l$plot_grayColor = "grey50"
 
+par.l$maxTFsToDraw = 150
+
+par.l$pseudocountLogTransform = 0.0001
+
 # diverging, modified
 par.l$colorCategories = c("activator" = "#d7191c", "undetermined" = "black", "repressor" = "#2b83ba", "not-expressed" = "slategrey")
 par.l$colorCategories = c("activator" = "#4daf4a", "undetermined" = "black", "repressor" = "#e41a1c", "not-expressed" = "Snow3")
@@ -63,8 +67,10 @@ par.l$legend_position = c(0.1, 0.9)
 
 # Which transformation of the y values to do?
 
-transform_yValues <- function(values) {
-    -log10(values + 0.0001)
+# TODO: Pseudocount also for line?
+transform_yValues <- function(values, addPseudoCount = TRUE) {
+    
+    -log10(values + ifelse(addPseudoCount, par.l$pseudocountLogTransform, 0))
 }
 
 transform_yValues_caption <- function() {
@@ -465,7 +471,7 @@ if (length(TF_NA) > 0) {
     output.global.TFs.orig = output.global.TFs.orig[-TF_NA,]
   
   TFs_NA = output.global.TFs.orig$TF[TF_NA]
-  message = paste0("The following TF have been removed from the data due to NA values in weighted_meanDifference (insufficient data in previous steps): ", paste0(TFs_NA, collapse = ", "))
+  message = paste0("The following TF have been removed from the data due to NA values in weighted_meanDifference (insufficient data in previous steps): ", paste0(unique(TFs_NA), collapse = ", "))
   checkAndLogWarningsAndErrors(NULL, message, isWarning = TRUE)
 }
 
@@ -1024,7 +1030,7 @@ output.global.TFs.origReal = output.global.TFs
 # Set the page dimensions to the maximum across all plotted variants
 output.global.TFs.filteredSummary = filter(output.global.TFs, pvalue <= max(par.l$significanceThresholds))
 
-nTF_label = nrow(output.global.TFs.filteredSummary)
+nTF_label = min(par.l$maxTFsToDraw, nrow(output.global.TFs.filteredSummary))
 
 TFLabelSize = ifelse(nTF_label < 20, 8,
                      ifelse(nTF_label < 40, 7,
@@ -1065,16 +1071,39 @@ for (significanceThresholdCur in par.l$significanceThresholds) {
     # VOLCANO PLOT #
     ################
     
-    output.global.TFs = output.global.TFs %>%
+    output.global.TFs = output.global.TFs.origReal %>%
         mutate( pValueAdj_log10 = transform_yValues(pvalueAdj),
-              pValue_sig = pvalueAdj <= significanceThresholdCur)
+                pValue_log10 = transform_yValues(pvalue),
+                pValueAdj_sig = pvalueAdj <= significanceThresholdCur,
+                pValue_sig = pvalue <= significanceThresholdCur)  %>%
+        filter(classification %in% showClasses)
+    
+    
+    for (pValueStrCur in c("pvalue", "pvalueAdj")) {
+        
+        
+        if (pValueStrCur == "pvalue") {
+            
+            pValueScoreCur = "pValue_log10"
+            pValueSigCur = "pValue_sig"
+            pValueStrLabel = "raw p-value"
             
             ggrepel_df = filter(output.global.TFs, pValue_sig == TRUE)
+            maxPValue = max(output.global.TFs$pValue_log10, na.rm = TRUE)
+            
+        } else {
             
+            pValueScoreCur = "pValueAdj_log10"
+            pValueSigCur = "pValueAdj_sig"
+            pValueStrLabel = "adj. p-value"
+            
+            ggrepel_df = filter(output.global.TFs, pValueAdj_sig == TRUE)
+            maxPValue = max(output.global.TFs$pValueAdj_log10, na.rm = TRUE)
+        }
     
         
         # Increase the ymax a bit more
-    ymax = max(transform_yValues(significanceThresholdCur), max(output.global.TFs$pValueAdj_log10, na.rm = TRUE)) * 1.1
+        ymax = max(transform_yValues(significanceThresholdCur), maxPValue) * 1.1
         alphaValueNonSign = 0.3
         
             
@@ -1085,7 +1114,7 @@ for (significanceThresholdCur in par.l$significanceThresholds) {
         g = ggplot()
         
         if (par.l$plotRNASeqClassification) {
-      g = g + geom_point(data = output.global.TFs, aes(weighted_meanDifference, pValueAdj_log10, alpha = pValue_sig, size = TFBS, fill = classification), shape=21, stroke = 0.5, color = "black") +  scale_fill_manual("TF class", values = par.l$colorCategories)
+          g = g + geom_point(data = output.global.TFs, aes_string("weighted_meanDifference", pValueScoreCur, alpha = pValueSigCur, size = "TFBS", fill = "classification"), shape=21, stroke = 0.5, color = "black") +  scale_fill_manual("TF class", values = par.l$colorCategories)
           
           g = g +
               geom_rect(aes(xmin = -Inf,xmax = 0,ymin = -Inf, ymax = Inf, color = par.l$colorConditions[2]),
@@ -1095,7 +1124,7 @@ for (significanceThresholdCur in par.l$significanceThresholds) {
           
         } else {
             
-      g = g + geom_point(data = output.global.TFs, aes(weighted_meanDifference, pValueAdj_log10, alpha = pValue_sig, size = TFBS), shape=21, stroke = 0.5, color = "black")
+          g = g + geom_point(data = output.global.TFs, aes_string("weighted_meanDifference", pValueScoreCur, alpha = pValueSigCur, size = "TFBS"), shape=21, stroke = 0.5, color = "black")
           g = g + geom_rect(aes(xmin = -Inf,
                                 xmax = 0,
                                 ymin = -Inf, 
@@ -1111,26 +1140,46 @@ for (significanceThresholdCur in par.l$significanceThresholds) {
      
     
         g = g + ylim(-0.1,ymax) + 
-        ylab(paste0(transform_yValues_caption(), " (adj. p-value)")) + 
+            ylab(paste0(transform_yValues_caption(), " (", pValueStrLabel, ")")) + 
             xlab("weighted mean difference") + 
-        scale_alpha_manual(paste0("adj. p-value < ", significanceThresholdCur), values = c(alphaValueNonSign, 1), labels = c("no", "yes")) + 
+            scale_alpha_manual(paste0(pValueStrLabel, " < ", significanceThresholdCur), values = c(alphaValueNonSign, 1), labels = c("no", "yes")) + 
             geom_hline(yintercept = transform_yValues(significanceThresholdCur), linetype = "dotted") 
         
+        if (nrow(ggrepel_df) <= par.l$maxTFsToDraw) {
+            
             if (par.l$plotRNASeqClassification) {
-      g = g +  geom_label_repel(data = ggrepel_df, aes(weighted_meanDifference, pValueAdj_log10, label = TF, fill = classification),
+                g = g +  geom_label_repel(data = ggrepel_df, aes_string("weighted_meanDifference", pValueScoreCur, label = "TF", fill = "classification"),
                                           size = TFLabelSize, fontface = 'bold', color = 'white',
                                           segment.size = 0.3, box.padding = unit(0.2, "lines"), max.iter = 5000,
                                           label.padding = unit(0.2, "lines"), # how thick is connectin line
                                           nudge_y = 0.05, nudge_x = 0,  # how far from center points
                                           segment.alpha = .8, segment.color = par.l$plot_grayColor, show.legend = FALSE)
             } else {
-      g = g +  geom_label_repel(data = ggrepel_df, aes(weighted_meanDifference, pValueAdj_log10, label = TF),
+                g = g +  geom_label_repel(data = ggrepel_df, aes_string("weighted_meanDifference", pValueScoreCur, label = "TF"),
                                           size = TFLabelSize, fontface = 'bold', color = 'black',
                                           segment.size = 0.3, box.padding = unit(0.2, "lines"), max.iter = 5000,
                                           label.padding = unit(0.2, "lines"), # how thick is connectin line
                                           nudge_y = 0.05, nudge_x = 0,  # how far from center points
                                           segment.alpha = .8, segment.color = par.l$plot_grayColor, show.legend = FALSE)
             }
+        } else {
+            
+            flog.warn(paste0("Not labeling significant TFs, maximum of ", par.l$maxTFsToDraw, " exceeded for ", pValThrStr, " and ", pValueStrCur))
+            
+            
+            if (nrow(ggrepel_df) > par.l$maxTFsToDraw) {
+                
+                labelPlot = paste0("*TF labeling skipped because number of significant TFs\nexceeds the maximum of ", par.l$maxTFsToDraw, " (", nrow(ggrepel_df), ")")
+                flog.warn(labelPlot)
+                
+                g = g + annotate("text", label = labelPlot, x = 0, y = ymax, size = 3)
+                
+            }
+            
+            
+            
+        }
+        
       
           g = g + theme_bw() + 
               theme(axis.text.x = element_text(size=rel(1.5)),
@@ -1145,23 +1194,27 @@ for (significanceThresholdCur in par.l$significanceThresholds) {
                            fill = guide_legend(override.aes = list(size=5), order = 3),
                            color = guide_legend(override.aes = list(size=5), order = 1))
             
-        allPlots.l[["volcano"]] [[pValThrStr]] [[paste0(showClasses,collapse = "-")]] = g
+            allPlots.l[["volcano"]] [[pValThrStr]] [[paste0(showClasses,collapse = "-")]] [[pValueStrCur]] = g
           } else {
             g = g + guides(alpha = guide_legend(override.aes = list(size=5), order = 2),
                            fill = guide_legend(override.aes = list(size=5), order = 3))
      
             
-        allPlots.l[["volcano"]] [[pValThrStr]] = g
+            allPlots.l[["volcano"]] [[pValThrStr]] [[pValueStrCur]] = g
           }
           
-
+      } # end separately for raw and adjusted p-values
         
     } # end for all showClasses
 
 
+
 } # end for different significance thresholds
 
 
+#####################
+# CIRCULAR PLOT PDF #
+#####################
 height = width = max(nTF_label / 5, par.l$circularPlot_minDimensions)
 
 # Increase a bit towards smaller heights by a factor, empirical observation
@@ -1182,18 +1235,30 @@ for (significanceThresholdCur in par.l$significanceThresholds) {
 }
 dev.off()
 
+
+####################
+# VOLCANO PLOT PDF #
+####################
 height = width = max(nTF_label / 15 , par.l$circularPlot_minDimensions)
 pdf(file = par.l$file_plotVolcano, height = height, width = width, useDingbats = FALSE)
 
-for (significanceThresholdCur in par.l$significanceThresholds) {
+
+for (pValueStrCur in c("pvalueAdj", "pvalue")) {
+    
+    for (significanceThresholdCur in par.l$significanceThresholds) {
       
       for (showClasses in classesList.l) {
+
               if (par.l$plotRNASeqClassification) {
-      plot(allPlots.l[["volcano"]] [[as.character(significanceThresholdCur)]] [[paste0(showClasses,collapse = "-")]])
+                  plot(allPlots.l[["volcano"]] [[as.character(significanceThresholdCur)]] [[paste0(showClasses,collapse = "-")]] [[pValueStrCur]])
               } else {
-      plot(allPlots.l[["volcano"]] [[as.character(significanceThresholdCur)]])
+                  plot(allPlots.l[["volcano"]] [[as.character(significanceThresholdCur)]] [[pValueStrCur]])
               }
           
+          
+        
+        
+      }
     }
 }
 dev.off()

src/Snakefile

@@ -301,7 +301,7 @@ script_summaryFinal         = "summaryFinal.R"
 #########################################
 
 # For cluster usage:  The keyword localrules allows to mark a rule as local, so that it is not submitted to the cluster and instead executed on the host node
-localrules: all
+localrules: all,cleanUpLogFiles,filterSexChromosomesAndSortPeaks
 
 rule all:
     input:

src/TF_Gene_TranslationTables/HOCOMOCO_v10/translationTable_mm10.csv

@@ -28,6 +28,10 @@ Cdc5l	ENSMUSG00000023932	CDC5L
 Cdx1	ENSMUSG00000024619	CDX1
 Cdx2	ENSMUSG00000029646	CDX2
 Cdx4	ENSMUSG00000031326	CDX4
+Cdx4Hoxc9	ENSMUSG00000036139	CDX4HOXC9
+Cdx4HT	ENSMUSG00000031326	CDX4HT
+Cdx4met1	ENSMUSG00000031326	CDX4MET1
+Cdx4met2	ENSMUSG00000031326	CDX4MET2
 Cebpa	ENSMUSG00000034957	CEBPA
 Cebpb	ENSMUSG00000056501	CEBPB
 Cebpd	ENSMUSG00000071637	CEBPD