Merge branch 'merging_back_to_main' into 'main'

Merging back to main See merge request !8

Merge branch 'merging_back_to_main' into 'main'
Merging back to main See merge request !8
63b0be82 · Cyril Christophe Daniel Robert Cros · 6431ee0f · 14a95984 · 63b0be82 · 63b0be82
Commit 63b0be82 authored 7 months ago by Cyril Christophe Daniel Robert Cros
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,7 @@ tmp/*
 */.nextflow.log*
 */.nextflow
 .nextflow.log*
-.nextflow
\ No newline at end of file
+.nextflow
+supplementary_resources/*/*.gff
+supplementary_resources/*/*.gff.gz
+supplementary_resources/*/*.gff.gz.tbi
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -12,15 +12,6 @@ stages:
  - deploy_to_cluster
  - clean_up_on_failure

-s3_access_test:
-  image: minio/mc
-  stage: access_test
-  before_script:
-    - mc alias set genomes $S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET_KEY
-  script:
-    - mc ls genomes/$S3_BUCKET
-    - cd image
-
 cluster_access_test:
  image: line/kubectl-kustomize:1.30.1-5.4.2
  stage: access_test
@@ -30,6 +21,17 @@ cluster_access_test:
    - export KUBECONFIG=/tmp/kubeconf
  script:
    - kubectl config current-context
+  rules:
+    - if: '$CI_COMMIT_REF_NAME  == "main"'
+      variables:
+        JBROWSE_HOSTNAME: "genomes.arendt.embl.de"
+        RELEASE_LABEL: "prod"
+        INGRESS_CLASS: "external-users"
+    - if: '$CI_COMMIT_REF_NAME  == "dev"'
+      variables:
+        JBROWSE_HOSTNAME: "genomes-dev.arendt.embl.de"
+        RELEASE_LABEL: "dev"
+        INGRESS_CLASS: "internal-users"

 build_nginx_with_jbrowse_config_file:
  image: gcr.io/kaniko-project/executor:debug-v0.21.0
@@ -41,9 +43,11 @@ build_nginx_with_jbrowse_config_file:
    - /kaniko/executor --cache=true --context $CI_PROJECT_DIR/image
      --dockerfile $CI_PROJECT_DIR/image/Dockerfile
      --destination $CI_REGISTRY_IMAGE:jbrowse2-$CI_COMMIT_SHORT_SHA
+    - echo "Destination follows"
+    - echo $CI_REGISTRY_IMAGE:jbrowse2-$CI_COMMIT_SHORT_SHA

 kustomize_build_apply:
-  image: line/kubectl-kustomize:1.30.1-5.4.2
+  image: dtzar/helm-kubectl:3.8.2
  stage: deploy_to_cluster
  before_script:
    - echo $KUBECONFIG | base64 -d > /tmp/kubeconf
@@ -52,9 +56,7 @@ kustomize_build_apply:
    - echo "You are on branch $CI_COMMIT_REF_NAME"
    - envsubst < deployment/jbrowse2-dep.yml > deployment/jbrowse2-dep.yaml
    - cat deployment/jbrowse2-dep.yaml
-    - kustomize deployment > resources.yaml
-    - cat resources.yaml
-    - kubectl apply -f resources.yaml
+    - kubectl apply -k deployment
    - kubectl -n $NS rollout restart deployment/jbrowse2-$RELEASE_LABEL
    - kubectl -n $NS wait --for=condition=available
      --timeout=60s deploy/jbrowse2-$RELEASE_LABEL
@@ -68,15 +70,17 @@ kustomize_build_apply:
        JBROWSE_HOSTNAME: "genomes.arendt.embl.de"
        RELEASE_LABEL: "prod"
        INGRESS_CLASS: "external-users"
+      when: on_success
    - if: '$CI_COMMIT_REF_NAME  == "dev"'
      variables:
        JBROWSE_HOSTNAME: "genomes-dev.arendt.embl.de"
        RELEASE_LABEL: "dev"
        INGRESS_CLASS: "internal-users"
-    - when: on_success
+      when: on_success
+    - when: never

 clean_up_k8s_deployment:
-  image: line/kubectl-kustomize:1.30.1-5.4.2
+  image: dtzar/helm-kubectl:3.8.2
  stage: clean_up_on_failure
  before_script:
    - echo $KUBECONFIG | base64 -d > /tmp/kubeconf
@@ -84,8 +88,7 @@ clean_up_k8s_deployment:
  script:
    - echo 'Failed job, deleting resources'
    - envsubst < deployment/jbrowse2-dep.yml > deployment/jbrowse2-dep.yaml
-    - kustomize deployment > resources.yaml
-    - kubectl delete -f resources.yaml
+    - kubectl delete -k deployment
  dependencies:
    - kustomize_build_apply
  rules:
@@ -101,4 +104,5 @@ clean_up_k8s_deployment:
        JBROWSE_HOSTNAME: "genomes-dev.arendt.embl.de"
        RELEASE_LABEL: "dev"
        INGRESS_CLASS: "internal-users"
-    - when: on_success
+    - when: never
+
--- a/deployment/jbrowse2-dep.yml
+++ b/deployment/jbrowse2-dep.yml
@@ -20,7 +20,7 @@ spec:
        environment: ${RELEASE_LABEL}
    spec:
      imagePullSecrets:
-        - name: gitlab-token-auth
+        - name: registry-credentials-jbrowse2-new
      containers:
        - name: jbrowse2
          image: ${CI_REGISTRY_IMAGE}:jbrowse2-${CI_COMMIT_SHORT_SHA}

--- a/image/jbrowse_script.sh
+++ b/image/jbrowse_script.sh
@@ -8,3 +8,16 @@ jbrowse add-track https://s3.embl.de/annelids/pdumv021/Annotation/pdumv021_Annot
 jbrowse add-track https://s3.embl.de/annelids/pdumv021/Annotation/pdumv021_EMAPPER.emapper.gff.gz -a "pdumv021" -d "Kevin's GFF file from EMAPPER" -n "EMAPPER models" --category "Annotation" --trackId "emapper"  --indexFile https://s3.embl.de/annelids/pdumv021/Annotation/pdumv021_EMAPPER.emapper.gff.gz.tbi
 jbrowse add-track https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_EMAPPER.gff.gz -a "pmassv1_masked" -d "Kevin's GFF file from EMAPPER" -n "EMAPPER models" --category "Annotation" --trackId "emapper-pmass"  --indexFile https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_EMAPPER.gff.gz.tbi
 jbrowse add-track https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_masked_Annotation_annotation.gff.gz -a "pmassv1_masked" -d "Kevin's GTF file" -n "Gene models" --category "Annotation" --trackId "annotation-pmass"  --indexFile https://s3.embl.de/annelids/pmassv1_masked/Annotation/pmassv1_masked_Annotation_annotation.gff.gz.tbi
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/2.final.bam -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 2" --category "ATACseq" --trackId "library_bam_2"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/2.final.bam.bai
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/2.final.bigwig -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 2" --category "ATACseq" --trackId "library_bigwig_2"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/3.final.bam -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 3" --category "ATACseq" --trackId "library_bam_3"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/3.final.bam.bai
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/3.final.bigwig -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 3" --category "ATACseq" --trackId "library_bigwig_3"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/4.final.bam -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 4" --category "ATACseq" --trackId "library_bam_4"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/4.final.bam.bai
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/4.final.bigwig -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 4" --category "ATACseq" --trackId "library_bigwig_4"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/5.final.bam -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 5" --category "ATACseq" --trackId "library_bam_5"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/5.final.bam.bai
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/ATACseq/5.final.bigwig -a "pdumv021" -d "Lara's Snakemake outputs" -n "Library 5" --category "ATACseq" --trackId "library_bigwig_5"  --indexFile https://s3.embl.de/annelids/pdumv021/ATACseq/
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/MASseq/collapsed.gff.gz -a "pdumv021" -d "MASseq outputs from running isoseq collapse against the Pdum genome" -n "SN073 MASseq isoforms from the 15ng prep" --category "MASseq" --trackId "SN073_15ng_collapsed"  --indexFile https://s3.embl.de/annelids/pdumv021/MASseq/collapsed.gff.gz.tbi
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/MASseq/collapsed.gff.gz -a "pdumv021" -d "MASseq outputs from running isoseq collapse against the Pdum genome" -n "SN073 MASseq isoforms" --category "MASseq" --trackId "SN073_60ng_collapsed"  --indexFile https://s3.embl.de/annelids/pdumv021/MASseq/collapsed.gff.gz.tbi
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/Variants/pdumv021_Variants_SN035_vcf.vcf.gz -a "pdumv021" -d "Tobi's variants" -n "SN035" --category "Variants" --trackId "SN035_vcf"  --indexFile https://s3.embl.de/annelids/pdumv021/Variants/pdumv021_Variants_SN035_vcf.vcf.gz.tbi
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/Variants/pdumv021_Variants_SN054_vcf.vcf.gz -a "pdumv021" -d "Tobi's variants" -n "SN054" --category "Variants" --trackId "SN054_vcf"  --indexFile https://s3.embl.de/annelids/pdumv021/Variants/pdumv021_Variants_SN054_vcf.vcf.gz.tbi
+jbrowse add-track https://s3.embl.de/annelids/pdumv021/Variants/pdumv021_Variants_SN055_vcf.vcf.gz -a "pdumv021" -d "Tobi's variants" -n "SN055" --category "Variants" --trackId "SN055_vcf"  --indexFile https://s3.embl.de/annelids/pdumv021/Variants/pdumv021_Variants_SN055_vcf.vcf.gz.tbi
--- a/resources_to_s3_NF/README.md
+++ b/resources_to_s3_NF/README.md
@@ -11,6 +11,15 @@ This folder contains a Nextflow pipeline that will:

 In theory you would have to edit those CSV files, rerun the pipeline and commit the resulting files in order to update Jbrowse2 with new tracks.

+## S3 configuration
+
+- `nextflow secrets set ACCESS_KEY ""`
+- `nextflow secrets set SECRET_KEY ""`
+
+## Launching the workflow
+
 To launch:
 - `module load Nextflow`
- `nextflow run -c nextflow.config -w /scratch/cros/nextflow-workdir -params-file params.yaml jbrowse2_processing.nf -resume`
\ No newline at end of file
+- `nextflow run -c nextflow.config -w /scratch/cros/nextflow-workdir -params-file params.yaml jbrowse2_processing.nf -resume`
+
+
--- a/resources_to_s3_NF/tracks.csv
+++ b/resources_to_s3_NF/tracks.csv
@@ -6,29 +6,10 @@ pdumv021,emapper,EMAPPER models,GFF,Annotation,Kevin's GFF file from EMAPPER,/g/
 pdumv021,SN035_vcf,SN035,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_PlanetaryBiology/Variants_SN035.vcf,FALSE
 pdumv021,SN054_vcf,SN054,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_PlanetaryBiology/Variants_SN054.vcf,FALSE
 pdumv021,SN055_vcf,SN055,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_PlanetaryBiology/Variants_SN055.vcf,FALSE
-pdumv021,SN056_vcf,SN056,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_PlanetaryBiology/Variants_SN056.vcf,FALSE
-pdumv021,SN057_vcf,SN057,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_PlanetaryBiology/Variants_SN057.vcf,FALSE
-pdumv021,SN058_vcf,SN058,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_PlanetaryBiology/Variants_SN058.vcf,FALSE
-pdumv021,SN001_vcf,SN001,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN001.vcf,FALSE
-pdumv021,SN002_vcf,SN002,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN002.vcf,FALSE
-pdumv021,SN003_vcf,SN003,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN003.vcf,FALSE
-pdumv021,SN004_vcf,SN004,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN004.vcf,FALSE
-pdumv021,SN015_vcf,SN015,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN015.vcf,FALSE
-pdumv021,SN016_vcf,SN016,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN016.vcf,FALSE
-pdumv021,SN018_vcf,SN018,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN018.vcf,FALSE
-pdumv021,SN019_vcf,SN019,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN019.vcf,FALSE
-pdumv021,SN020_vcf,SN020,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN020.vcf,FALSE
-pdumv021,SN021_vcf,SN021,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN021.vcf,FALSE
-pdumv021,SN028_vcf,SN028,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN028.vcf,FALSE
-pdumv021,SN029_vcf,SN029,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN029.vcf,FALSE
-pdumv021,SN030_vcf,SN030,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN030.vcf,FALSE
-pdumv021,SN031_vcf,SN031,VCF,Variants,Tobi's variants,/g/arendt/gerber/Processed_Files/Variants_Atlas/Variants_Libs/Variants_SN031.vcf,FALSE
-pdumv021,library_bam_1,Library 1,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/1.final.bam,FALSE
-pdumv021,library_bam_2,Library 2,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/2.final.bam,FALSE
-pdumv021,library_bam_3,Library 3,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/3.final.bam,FALSE
-pdumv021,library_bam_4,Library 4,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/4.final.bam,FALSE
-pdumv021,library_bam_5,Library 5,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/5.final.bam,FALSE
-pdumv021,library_bigwig_1,Library 1,BigWig,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/1.final.bigwig,FALSE
+pdumv021,library_bam_2,Library 2 BAM,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/2.final.bam,FALSE
+pdumv021,library_bam_3,Library 3 BAM,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/3.final.bam,FALSE
+pdumv021,library_bam_4,Library 4 BAM,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/4.final.bam,FALSE
+pdumv021,library_bam_5,Library 5 BAM,BAM,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/5.final.bam,FALSE
 pdumv021,library_bigwig_2,Library 2,BigWig,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/2.final.bigwig,FALSE
 pdumv021,library_bigwig_3,Library 3,BigWig,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/3.final.bigwig,FALSE
 pdumv021,library_bigwig_4,Library 4,BigWig,ATACseq,Lara's Snakemake outputs,/g/arendt/Lara/8.FinalOutput/4.final.bigwig,FALSE