Merged main

.gitlab-ci.yml

@@ -33,33 +33,59 @@ cluster_access_test:
   rules:
     - when: always  # TODO change where necessary
 
+build_base_jbrowse_config_file:
+  stage: build
+  image: gcr.io/kaniko-project/executor:debug-v0.21.0
+  before_script:
+    # yamllint disable-line rule:line-length
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+  script:
+    - /kaniko/executor --cache=true --context $CI_PROJECT_DIR/image
+      --dockerfile $CI_PROJECT_DIR/image/Dockerfile_base
+      --destination $CI_REGISTRY_IMAGE:jbrowse2-base-$CI_COMMIT_SHORT_SHA
+    - echo "Pushed $CI_REGISTRY_IMAGE:jbrowse2-base-$CI_COMMIT_SHORT_SHA"
+
+export_artifacts_via_inception:
+  stage: build
+  image: $CI_REGISTRY_IMAGE:jbrowse2-base-$CI_COMMIT_SHORT_SHA
+  needs:
+    - build_base_jbrowse_config_file
+  script:
+    - cp /app/annelids/index.html image/
+    - cp /app/annelids/config.json image/
+  artifacts:
+    paths:
+      - image/index.html
+      - image/config.json
+
 build_nginx_with_jbrowse_config_file:
   stage: build
   image: gcr.io/kaniko-project/executor:debug-v0.21.0
+  needs:
+    - build_base_jbrowse_config_file
+    - export_artifacts_via_inception
   before_script:
     # yamllint disable-line rule:line-length
     - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
   script:
+    - echo "JBROWSE2_BASE='$CI_REGISTRY_IMAGE:jbrowse2-base-$CI_COMMIT_SHORT_SHA'"
     - /kaniko/executor --cache=true --context $CI_PROJECT_DIR/image
-      --dockerfile $CI_PROJECT_DIR/image/Dockerfile
+      --build-arg "JBROWSE2_BASE='$CI_REGISTRY_IMAGE:jbrowse2-base-$CI_COMMIT_SHORT_SHA'"
+      --dockerfile $CI_PROJECT_DIR/image/Dockerfile_advanced
       --destination $CI_REGISTRY_IMAGE:jbrowse2-$CI_COMMIT_SHORT_SHA
     - echo "Destination follows"
     - echo $CI_REGISTRY_IMAGE:jbrowse2-$CI_COMMIT_SHORT_SHA
 
-build_export_artifacts_via_inception:
+export_advanced_artifacts_via_inception:
   stage: build
-  image:
-    name: $CI_REGISTRY_IMAGE:jbrowse2-$CI_COMMIT_SHORT_SHA
+  image: $CI_REGISTRY_IMAGE:jbrowse2-$CI_COMMIT_SHORT_SHA
   needs:
     - build_nginx_with_jbrowse_config_file
   script:
-    - ls /usr/share/nginx/html
-    - cp /usr/share/nginx/html/index.html .
-    - cp /usr/share/nginx/html/config.json .
+    - cp /usr/share/nginx/html/config.json image/config_advanced.json
   artifacts:
     paths:
-    - index.html
-    - config.json
+      - image/config_advanced.json
 
 deploy_to_k8s:
   stage: deploy

image/Dockerfile → image/Dockerfile_advanced

-# Stage 1: Jbrowse config file generated
-FROM quay.io/biocontainers/jbrowse2:2.15.3--hbbc3ce4_0 as jbrowse_builder
+# Stage 1: Jbrowse previous image
+ARG JBROWSE2_BASE
+FROM ${JBROWSE2_BASE} as jbrowse_builder
 
-WORKDIR /app
-COPY jbrowse_script.sh .
-RUN ./jbrowse_script.sh
+WORKDIR /app/annelids
+COPY jbrowse_script_advanced.sh .
+RUN ./jbrowse_script_advanced.sh
 RUN rm -rf annelids/node_modules
 
 # Stage 2: Load into nginx

image/Dockerfile_base

+# Stage 1: Jbrowse config file generated
+FROM quay.io/biocontainers/jbrowse2:2.15.3--hbbc3ce4_0 
+WORKDIR /app
+COPY jbrowse_script_base.sh .
+RUN ./jbrowse_script_base.sh

image/jbrowse_script.sh → image/jbrowse_script_advanced.sh

 #!/bin/bash -eux
 # Platy only for now
-jbrowse create annelids
-cd annelids
-jbrowse add-assembly https://s3.embl.de/annelids/pdumv021/pdumv021.fa.gz -n "pdumv021" --displayName "Platynereis dumerilii genome v2.1 (pdumv021)" --gziLocation https://s3.embl.de/annelids/pdumv021/pdumv021.fa.gz.gzi --faiLocation https://s3.embl.de/annelids/pdumv021/pdumv021.fa.gz.fai -t bgzipFasta  
-jbrowse add-assembly https://s3.embl.de/annelids/pmassv1_masked/pmassv1_masked.fa.gz -n "pmassv1_masked" --displayName "Platynereis massiliensis genome v1 - masked (pmassv1)" --gziLocation https://s3.embl.de/annelids/pmassv1_masked/pmassv1_masked.fa.gz.gzi --faiLocation https://s3.embl.de/annelids/pmassv1_masked/pmassv1_masked.fa.gz.fai -t bgzipFasta  
-jbrowse add-track https://s3.embl.de/annelids/pdumv021/Annotation/pdumv021_Annotation_annotation.gff.gz -a "pdumv021" -d "Kevin's GTF file" -n "Gene models" --category "Annotation" --trackId "annotation"  --indexFile https://s3.embl.de/annelids/pdumv021/Annotation/pdumv021_Annotation_annotation.gff.gz.tbi
-jbrowse add-track https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_masked_Annotation_annotation.gff.gz -a "pmassv1_masked" -d "Kevin's GTF file" -n "Gene models" --category "Annotation" --trackId "annotation-pmass"  --indexFile https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_masked_Annotation_annotation.gff.gz.tbi
 jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/2.final.bam -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 2 BAM" --category "ATACseq" --trackId "library_bam_2"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/2.final.bam.bai
 jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/2.final.bigwig -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 2 BigWig" --category "ATACseq" --trackId "library_bigwig_2"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/
 jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/3.final.bam -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 3 BAM" --category "ATACseq" --trackId "library_bam_3"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/3.final.bam.bai

image/jbrowse_script_base.sh

+#!/bin/bash -eux
+# Platy only for now
+jbrowse create annelids
+cd annelids
+# Assemblies
+jbrowse add-assembly https://s3.embl.de/annelids/pdumv021/pdumv021.fa.gz -n "pdumv021" --displayName "Platynereis dumerilii genome v2.1 (pdumv021)" --gziLocation https://s3.embl.de/annelids/pdumv021/pdumv021.fa.gz.gzi --faiLocation https://s3.embl.de/annelids/pdumv021/pdumv021.fa.gz.fai -t bgzipFasta
+jbrowse add-assembly https://s3.embl.de/annelids/pmassv1_masked/pmassv1_masked.fa.gz -n "pmassv1_masked" --displayName "Platynereis massiliensis genome v1 - masked (pmassv1)" --gziLocation https://s3.embl.de/annelids/pmassv1_masked/pmassv1_masked.fa.gz.gzi --faiLocation https://s3.embl.de/annelids/pmassv1_masked/pmassv1_masked.fa.gz.fai -t bgzipFasta
+# Reference annotation
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/Annotation/pdumv021_Annotation_annotation.gff.gz -a "pdumv021" -d "Kevin's GTF file" -n "Gene models" --category "Annotation" --trackId "annotation"  --indexFile https://s3.embl.de/annelids/pdumv021/Annotation/pdumv021_Annotation_annotation.gff.gz.tbi
+jbrowse add-track https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_masked_Annotation_annotation.gff.gz -a "pmassv1_masked" -d "Kevin's GTF file" -n "Gene models" --category "Annotation" --trackId "annotation-pmass"  --indexFile https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_masked_Annotation_annotation.gff.gz.tbi
+# Text index
+jbrowse text-index

supplementary_resources/pdumv21/convert_to_gff.sh

-#!/bin/bash
-FILE='./Supp_miRNA_Annotation_Table.xlsx - miRNA_annotated.csv'
-OUT_FILE='./miRNA.gff'
-echo "##gff-version 3" > $OUT_FILE
-tail -n +2 "$FILE" | awk -F ',' '!($5 == "")' | cut -d',' -f 1,5,9 | sed -E -e 's/(:|\.\.)/,/g' | \
- awk 'BEGIN {FS=","; OFS="\t"} {print $2, MirDeep2, miRNA, $3, $4, $6, $5, ".", "gene_name=\""$1"\""}' >> $OUT_FILE