feat: update of dockerfile to use more secure image;

INSTALL.pl

@@ -150,20 +150,20 @@ BEGIN
 my $config = {};
 GetOptions(
   $config,
-  'DESTDIR|d=s',
+  'DESTDIR|DEST_DIR|d=s',
   'CACHE_VERSION|e=i',
   'ASSEMBLY|y=s',
-  'BIOPERL|b=s',
-  'CACHEURL|u=s',
-  'CACHEDIR|c=s',
-  'FASTAURL|f=s',
+  'BIOPERL|BIOPERL_URL|b=s',
+  'CACHEURL|CACHE_URL|u=s',
+  'CACHEDIR|CACHE_DIR|c=s',
+  'FASTAURL|FASTA_URL|f=s',
   'HELP|h',
   'NO_UPDATE|n',
   'SPECIES|s=s',
   'NO_PLUGINS',
   'PLUGINS|g=s',
-  'PLUGINSDIR|r=s',
-  'PLUGINURL=s',
+  'PLUGINSDIR|PLUGINS_DIR|r=s',
+  'PLUGINURL|PLUGIN_URL=s',
   'AUTO|a=s',
   'QUIET|q',
   'PREFER_BIN|p',

README.md

@@ -1,7 +1,7 @@
 # ensembl-vep
 
-[![GitHub](https://img.shields.io/github/license/Ensembl/ensembl-vep.svg)](https://github.com/Ensembl/ensembl-vep/blob/release/110/LICENSE)
-[![Coverage Status](https://coveralls.io/repos/github/Ensembl/ensembl-vep/badge.svg?branch=release/110)](https://coveralls.io/github/Ensembl/ensembl-vep?branch=release/110)
+[![GitHub](https://img.shields.io/github/license/Ensembl/ensembl-vep.svg)](https://github.com/Ensembl/ensembl-vep/blob/main/LICENSE)
+[![Coverage Status](https://coveralls.io/repos/github/Ensembl/ensembl-vep/badge.svg?branch=main)](https://coveralls.io/github/Ensembl/ensembl-vep?branch=main)
 [![Docker Build Status](https://img.shields.io/github/actions/workflow/status/Ensembl/ensembl-vep/docker.yml?label=docker%20build)](https://hub.docker.com/r/ensemblorg/ensembl-vep)
 [![Docker Hub Pulls](https://img.shields.io/docker/pulls/ensemblorg/ensembl-vep.svg)](https://hub.docker.com/r/ensemblorg/ensembl-vep)
 
@@ -10,17 +10,23 @@
 * **Variant Recoder** translates between different variant encodings.
 
 ##### Table of contents
-* [Installation and requirements](#install)
-* [VEP](#vep)
-  * [Usage](#vepusage)
-* [Haplosaurus](#haplo)
-  * [Usage](#haplousage)
-  * [Output](#haplooutput)
-  * [REST](#haploREST)
-  * [Flags](#haploflags)
-* [Variant Recoder](#recoder)
-  * [Usage](#recoderusage)
-  * [Output](#recoderoutput) 
+- [ensembl-vep](#ensembl-vep)
+        - [Table of contents](#table-of-contents)
+    - [Installation and requirements](#installation-and-requirements)
+      - [Additional CPAN modules](#additional-cpan-modules)
+      - [Docker](#docker)
+  - [VEP](#vep)
+    - [Usage](#usage)
+  - [Haplosaurus](#haplosaurus)
+    - [Usage](#usage-1)
+    - [Output](#output)
+    - [REST service](#rest-service)
+    - [Flags](#flags)
+    - [bioperl-ext](#bioperl-ext)
+  - [Variant Recoder](#variant-recoder)
+    - [Usage](#usage-2)
+    - [Output](#output-1)
+    - [Options](#options)
 
 ---
 <a name="install"></a>

cpanfile

@@ -2,7 +2,7 @@ requires 'DBI';
 requires 'Set::IntervalTree';
 requires 'JSON';
 requires 'Text::CSV';
-recommends 'DBD::mysql';
+recommends 'DBD::mysql', '<= 4.050'; # newer versions do not support MySQL 5
 recommends 'PerlIO::gzip';
 recommends 'IO::Uncompress::Gunzip';
 recommends 'Bio::DB::BigFile';

docker/Dockerfile

@@ -1,13 +1,15 @@
-ARG BRANCH=release/110
+ARG BRANCH=main
 
 ###################################################
 # Stage 1 - docker container to build ensembl-vep #
 ###################################################
-FROM ubuntu:18.04 as builder
+FROM ubuntu:23.04 as builder
+ARG DEBIAN_FRONTEND=noninteractive
 
 # Update aptitude and install some required packages
 # a lot of them are required for Bio::DB::BigFile
-RUN apt-get update && apt-get -y install \
+# many dependencies are needed to ensure that that vulnerabilities are squashed
+RUN apt-get update && apt-get -y install --no-install-recommends \
     build-essential \
     git \
     libpng-dev \
@@ -16,15 +18,23 @@ RUN apt-get update && apt-get -y install \
     liblzma-dev \
     perl \
     perl-base \
+    python3 \
+    python3-pip \
     unzip \
     wget && \
+    mv /usr/lib/python3.11/EXTERNALLY-MANAGED /usr/lib/python3.11/EXTERNALLY-MANAGED.old && \
+    python3 -m pip install --no-cache-dir --upgrade pip && \
+    python3 -m pip uninstall cryptography && \
+    python3 -m pip install --no-cache-dir cryptography==41.0.4 setuptools==65.5.1 && \
+    apt-get -y purge manpages-dev && \
+    apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
 # Setup VEP environment
 ENV OPT /opt/vep
 ENV OPT_SRC $OPT/src
 ENV HTSLIB_DIR $OPT_SRC/htslib
-ARG BRANCH
+ENV BRANCH
 
 # Working directory
 WORKDIR $OPT_SRC
@@ -51,7 +61,7 @@ RUN if [ "$BRANCH" = "main" ]; \
     unzip -q ensembl-xs.zip && mv ensembl-xs-2.3.2 ensembl-xs && rm -rf ensembl-xs.zip && \
     # Clone/Download other repositories: bioperl-live is needed so the cpanm dependencies installation from the ensembl-vep/cpanfile file takes less disk space
     ensembl-vep/travisci/get_dependencies.sh && \
-    # Only keep the bioperl-live "Bio" library
+    # Only keep the bioperl-live"Bio" library
     mv bioperl-live bioperl-live_bak && mkdir bioperl-live && mv bioperl-live_bak/Bio bioperl-live/ && rm -rf bioperl-live_bak && \
     ## A lot of cleanup on the imported libraries, in order to reduce the docker image ##
     rm -rf Bio-HTS/.??* Bio-HTS/Changes Bio-HTS/DISCLAIMER Bio-HTS/MANIFEST* Bio-HTS/README Bio-HTS/scripts Bio-HTS/t Bio-HTS/travisci \
@@ -65,6 +75,14 @@ RUN if [ "$BRANCH" = "main" ]; \
     cp kent-335_base_bak/src/*.sh kent-335_base/src/ && \
     rm -rf kent-335_base_bak
 
+# # Build LOFTEE for GRCh37 & GRRCh38
+# # This section is included for completion but commented out as LOFTEE is not a cononical plugin
+# RUN mkdir /loftee-GRCh37 /loftee-GRCh38 && \
+#     cd /loftee-GRCh37 && \
+#     git clone https://github.com/konradjk/loftee.git && \
+#     cd /loftee-GRCh38 && \
+#     git clone -b ${LOFTEE_GRCh38_BRANCH} https://github.com/konradjk/loftee.git
+
 # Setup bioperl-ext
 WORKDIR bioperl-ext/Bio/Ext/Align/
 RUN perl -pi -e"s|(cd libs.+)CFLAGS=\\\'|\$1CFLAGS=\\\'-fPIC |" Makefile.PL
@@ -82,14 +100,19 @@ RUN make && rm -f Makefile *.c
 ###################################################
 # Stage 2 - docker container to build ensembl-vep #
 ###################################################
-FROM ubuntu:18.04
+FROM ubuntu:23.04
+ARG DEBIAN_FRONTEND=noninteractive
 
 # Update aptitude and install some required packages
 # a lot of them are required for Bio::DB::BigFile
-RUN apt-get update && apt-get -y install \
+# some packages have been moved here form the ubuntu-packages.txt
+# due to the need to clear vulnerabilities
+RUN apt-get update && apt-get -y install --no-install-recommends \
+    software-properties-common \
     build-essential \
     cpanminus \
     curl \
+    libgd-dev \
     libmysqlclient-dev \
     libpng-dev \
     libssl-dev \
@@ -100,19 +123,31 @@ RUN apt-get update && apt-get -y install \
     openssl \
     perl \
     perl-base \
+    pkg-config \
+    python3 \
+    python3-pip \
+    sqlite3 \
+    uuid-dev \
     unzip \
     vim && \
+    # following lines are needed to install updates to compromiosed python packages
+    mv /usr/lib/python3.11/EXTERNALLY-MANAGED /usr/lib/python3.11/EXTERNALLY-MANAGED.old && \
+    python3 -m pip install --no-cache-dir --upgrade pip && \
+    python3 -m pip uninstall cryptography && \
+    python3 -m pip install --no-cache-dir cryptography==41.0.4 setuptools==65.5.1 && \
     apt-get -y purge manpages-dev && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+
 # Setup VEP environment
 ENV OPT /opt/vep
 ENV OPT_SRC $OPT/src
 ENV PERL5LIB_TMP $PERL5LIB:$OPT_SRC/ensembl-vep:$OPT_SRC/ensembl-vep/modules
 ENV PERL5LIB $PERL5LIB_TMP:$OPT_SRC/bioperl-live
 ENV KENT_SRC $OPT/src/kent-335_base/src
 ENV HTSLIB_DIR $OPT_SRC/htslib
+ENV MACHTYPE x86_64
 ENV DEPS $OPT_SRC
 ENV PATH $OPT_SRC/ensembl-vep:$OPT_SRC/var_c_code:$PATH
 ENV LANG_VAR en_US.UTF-8
@@ -146,6 +181,8 @@ RUN export MACHTYPE=$(uname -m) &&\
     # Install ensembl perl dependencies (cpanm)
     cpanm --installdeps --with-recommends --notest --cpanfile ensembl_cpanfile . && \
     cpanm --installdeps --with-recommends --notest --cpanfile ensembl-vep/cpanfile . && \
+    # additional perl dependencies
+    cpanm Try::Tiny Module::Build && \
     # Delete bioperl and cpanfiles after the cpanm installs as bioperl will be reinstalled by the INSTALL.pl script
     rm -rf bioperl-live ensembl_cpanfile ensembl-vep/cpanfile && \
     # Configure "locale", see https://github.com/rocker-org/rocker/issues/19
@@ -162,6 +199,8 @@ ENV LANG $LANG_VAR
 # Switch back to vep user
 USER vep
 ENV PERL5LIB $PERL5LIB_TMP
+# Removing FATHMM, FATHMM_MKL & PON_P2 as these plugins require python 2.x which is End-Of-Life
+ENV PLUGIN_LIST=AlphaMisssense,AncestralAllele,Blosum62,CADD,Carol,Condel,Conservation,dbNSFP,dbscSNV,DisGeNET,Downstream,Draw,ExAC,ExACpLi,FlagLRG,FunMotifs,G2P,GeneSplicer,gnomADc,GO,HGVSIntronOffset,LD,LocalID,LoFtool,LOVD,Mastermind,MaxEntScan,MPC,MTR,NearestExonJB,NearestGene,neXtProt,Phenotypes,PON_P2,PostGAP,ProteinSeqs,ReferenceQuality,REVEL,SameCodon,satMutMPRA,SingleLetterAA,SpliceAI,SpliceRegion,StructuralVariantOverlap,SubsetVCF,TSSDistance,UTRannotator
 
 # Setup Docker environment for when users run VEP and INSTALL.pl in Docker image:
 #   - skip VEP updates in INSTALL.pl
@@ -181,26 +220,39 @@ RUN echo >> $OPT/.profile && \
     echo PATH=$PATH:\$PATH >> $OPT/.profile && \
     echo export PATH >> $OPT/.profile && \
     # Install Ensembl API and plugins
-    ./INSTALL.pl --auto ap --plugins all --pluginsdir $VEP_DIR_PLUGINS --no_update --no_htslib && \
+    ./INSTALL.pl --auto ap --plugins all --pluginsdir ${VEP_DIR_PLUGINS} --no_update --no_htslib && \
     # Remove the ensemb-vep tests and travis
     rm -rf t travisci .travis.yml
+    # # needed to prevent LoF compilation errors at runtime
+    # # only required if LOFTEE is being intalled
+    # rm -rf $VEP_DIR_PLUGINS/LoF.pm
+
+# # Copy over LOFTEE build files
+# # Only needed if LOFTEE is being installed
+# COPY --from=builder /loftee-GRCh37/loftee /plugins/loftee-GRCh37
+# COPY --from=builder /loftee-GRCh38/loftee /plugins/loftee-GRCh38
 
 # Install dependencies for VEP plugins:
 USER root
 ENV PLUGIN_DEPS "https://raw.githubusercontent.com/Ensembl/VEP_plugins/$BRANCH/config"
 #   - Ubuntu packages
-RUN curl -O "$PLUGIN_DEPS/ubuntu-packages.txt" && \
-    apt-get update && apt-get install -y --no-install-recommends \
-    $(sed s/\#.*//g ubuntu-packages.txt) && \
-    rm -rf /var/lib/apt/lists/* ubuntu-packages.txt
+# Commented out as the packages in question are explicitly installed above.
+# Retained to ensure that future updates can more easily reintegrate the commands
+# RUN curl -O "$PLUGIN_DEPS/ubuntu-packages.txt" && \
+#     apt-get update && apt-get install -y --no-install-recommends \
+#     "$(sed s/\#.*//g ubuntu-packages.txt)" && \
+#     rm -rf /var/lib/apt/lists/* ubuntu-packages.txt
 #   - Perl modules
 RUN curl -O "$PLUGIN_DEPS/cpanfile" && \
     cpanm --installdeps --with-recommends . && \
     rm -rf /root/.cpanm cpanfile
 #   - Python packages
-RUN curl -O "$PLUGIN_DEPS/requirements.txt" && \
-    python -m pip install --no-cache-dir -r requirements.txt && \
-    rm requirements.txt
+# Commented out as Python2 has been removed from the container.
+# Retained to ensure that future updates can more easily reintegrate the commands
+# RUN curl -O "$PLUGIN_DEPS/requirements.txt" && \
+#     python -m pip install --no-cache-dir -r requirements.txt && \
+#     python -m pip uninstall urllib && \
+#     rm requirements.txt
 
 # Set working directory as symlink to $OPT/.vep (containing VEP cache and data)
 USER root

modules/Bio/EnsEMBL/VEP/AnnotationSource.pm

@@ -184,7 +184,7 @@ sub get_regions_from_coords {
   # log min/max
   my ($min, $max) = defined $min_max->{$source_chr} ? @{$min_max->{$source_chr}} : (1e10, 0);
   $min = $start if $start < $min;
-  $max = $end if $end > $max;
+  $max = $end   if   $end > $max;
 
   # convert to region-size
   my ($r_s, $r_e) = map {int(($_ - 1) / $cache_region_size)} ($start - $up_down_size, $end + $up_down_size);
@@ -245,6 +245,25 @@ sub get_all_regions_by_InputBuffer {
     push @regions, @new_regions;
     $min_max->{$chr} = [$min, $max];
 
+    # process alternative alleles from breakend structural variants
+    next unless Scalar::Util::blessed($vf) and $vf->can('get_breakends');
+    foreach my $alt (@{$vf->get_breakends}) {
+      ($chr, $min, $max, $seen, @new_regions) = $self->get_regions_from_coords(
+        $alt->{chr}, $alt->{pos}, $alt->{pos},
+        $min_max, $cache_region_size, $up_down_size, $seen);
+      push @regions, @new_regions;
+      $min_max->{$chr} = [$min, $max];
+    }
+
+    $chr = $vf->{chr} || $vf->slice->seq_region_name;
+    throw("ERROR: Cannot get chromosome $chr from VariationFeature") unless $chr;
+
+    ($chr, $min, $max, $seen, @new_regions) = $self->get_regions_from_coords(
+      $chr, $vf->{start}, $vf->{end},
+      $min_max, $cache_region_size, $up_down_size, $seen);
+    push @regions, @new_regions;
+    $min_max->{$chr} = [$min, $max];
+
     # process alternative alleles from breakend structural variants
     foreach my $alt (@{$vf->{_parsed_allele}}) {
       ($chr, $min, $max, $seen, @new_regions) = $self->get_regions_from_coords(
@@ -327,7 +346,12 @@ sub filter_features_by_min_max {
   my $up_down_size = defined($self->{up_down_size}) ? $self->{up_down_size} : $self->up_down_size();
 
   return [
-    grep { $self->_check_overlap($_, $min_max, $up_down_size) } @$features
+    grep {
+      exists $min_max->{$_->{slice}->{seq_region_name}} &&
+      overlap($_->{start}, $_->{end},
+              @{$min_max->{$_->{slice}->{seq_region_name}}}[0] - $up_down_size,
+              @{$min_max->{$_->{slice}->{seq_region_name}}}[1] + $up_down_size)}
+    @$features
   ];
 }
 

modules/Bio/EnsEMBL/VEP/AnnotationSource/Database/Variation.pm

@@ -190,8 +190,8 @@ sub get_features_by_regions_uncached {
       my %clin_sigs;
       foreach my $pfa(@{$v_clinsigs})
       {
-	if(defined($pfa->{clinvar_clin_sig}) && $v_copy{variation_name} eq $pfa->{id})
-	{
+        if(defined($pfa->{clinvar_clin_sig}) && $v_copy{variation_name} eq $pfa->{id})
+        {
           $pfa->{clinvar_clin_sig}=~s/ /_/g;
           $clin_sigs{$pfa->{risk_allele} . ':' .$pfa->{clinvar_clin_sig}} = 1;
         }