Update Singularity file definition to avoid keyserver errors

_pytadbit/mapping/full_mapper.py

@@ -277,7 +277,6 @@ def _sam_filter(fnam, fastq_path, unmap_out, map_out):
     """
     Divides reads in a map file in two categories: uniquely mapped, and not.
     Writes them in two files
-
     """
     try:
         fhandler = Samfile(fnam)
@@ -506,8 +505,7 @@ def _gem_mapping(gem_index_path, fastq_path, out_map_path, fastq_path2 = None,
     print(' '.join(gem_cmd))
     try:
         # check_call(gem_cmd, stdout=PIPE, stderr=PIPE)
-        out, err = Popen(gem_cmd, stdout=PIPE, stderr=PIPE,
-                         universal_newlines=True).communicate()
+        out, err = Popen(gem_cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate()
     except CalledProcessError as e:
         print(out)
         print(err)
@@ -566,8 +564,7 @@ def full_mapping(mapper_index_path, fastq_path, out_map_dir, mapper='gem',
     suffix = ('_' * (suffix != '')) + suffix
     nthreads = kwargs.get('nthreads', 8)
     outfiles = []
-    temp_dir = os.path.abspath(os.path.expanduser(
-        kwargs.get('temp_dir', gettempdir())))
+    temp_dir = os.path.abspath(os.path.expanduser(kwargs.get('temp_dir', gettempdir())))
     if mapper == 'gem':
         gem_version = None
         # check that we have the GEM binary:
@@ -651,8 +648,7 @@ def full_mapping(mapper_index_path, fastq_path, out_map_dir, mapper='gem',
                 if gem_version >= 3:
                     _sam_filter(out_map_path, curr_map,
                                 curr_map + '_filt_%s-%s%s.map' % (beg, end, suffix),
-                                os.path.join(out_map_dir,
-                                             base_name + '_full_%s-%s%s.map' % (beg, end, suffix)))
+                                os.path.join(out_map_dir, base_name + '_full_%s-%s%s.map' % (beg, end, suffix)))
                 else:
                     _gem_filter(out_map_path,
                                 curr_map + '_filt_%s-%s%s.map' % (beg, end, suffix),
@@ -929,4 +925,4 @@ def fast_fragment_mapping(mapper_index_path, fastq_path1, fastq_path2, r_enz,
     if get_nread:
         return [(out_map,count_fastq)]
     return out_map
-
+

_pytadbit/parsers/map_parser.py

@@ -112,13 +112,12 @@ def parse_map(f_names1, f_names2=None, out_file1=None, out_file2=None,
             # start parsing
             read_count = 0
             try:
-                while not False:
+                while True:
                     for _ in range(max_size):
                         try:
-                            reads.append(read_read(next(fhandler), frags,
-                                                   frag_chunk))
+                            reads.append(read_read(next(fhandler), frags, frag_chunk))
                         except KeyError:
-                            # Chromosome not in hash
+                            print("Chromosome not in hash")
                             continue
                         read_count += 1
                     nfile += 1
@@ -260,6 +259,8 @@ def read_read_nofrags(r, _, __):
     try:
         crm, strand, pos = ali.split(':')[:3]
     except ValueError:
+        print(r)
+        print(ali)
         raise KeyError()
     positive = strand == '+'
     crm = crm.split()[0]

_pytadbit/tools/tadbit_normalize.py

@@ -793,7 +793,12 @@ def read_bam(inbam, filter_exclude, resolution, min_count=2500, biases_path='',
     pool = mu.Pool(ncpus)
     procs = []
     read_bam_frag = read_bam_frag_valid if only_valid else read_bam_frag_filter
+
+    num_regs = len(regs)
+    regs = map(lambda r: r.replace("|", "_"), regs)
+
     for i, (region, start, end) in enumerate(zip(regs, begs, ends)):
+        num_regs += 1
         procs.append(pool.apply_async(
             read_bam_frag, args=(inbam, filter_exclude, bins, bins_dict,
                                  resolution, outdir, extra_out,
@@ -803,13 +808,13 @@ def read_bam(inbam, filter_exclude, resolution, min_count=2500, biases_path='',
     pool.join()
     ## COLLECT RESULTS
     cisprc = {}
-    printime('  - Collecting cis and total interactions per bin (%d chunks)' % (len(regs)))
+    printime('  - Collecting cis and total interactions per bin (%d chunks)' % num_regs)
     stdout.write('     ')
     for countbin, (region, start, end) in enumerate(zip(regs, begs, ends)):
         if not countbin % 10 and countbin:
             stdout.write(' ')
         if not countbin % 50 and countbin:
-            stdout.write(' %9s\n     ' % ('%s/%s' % (countbin , len(regs))))
+            stdout.write(' %9s\n     ' % ('%s/%s' % (countbin , num_regs)))
         stdout.write('.')
         stdout.flush()
 

_pytadbit/utils/sqlite_utils.py

@@ -36,6 +36,9 @@ def digest_parameters(opts, get_md5=True, extra=None):
          if k not in ['fastq', 'index', 'renz', 'iterative', 'workdir',
                       'skip', 'func', 'tmp', 'keep_tmp'] + extra and v is not None])
     parameters = parameters.replace("'", "")
+    parameters = parameters.replace("|", "_")
+    parameters = parameters.replace(";", "_")
+    parameters = parameters.replace("#", "_")
     return parameters
 
 

containers/TADbit_singularity.def

@@ -1,95 +1,63 @@
 BootStrap: docker
-From:neurodebian:buster
+From: debian:latest
 
 %help
   minimal TADbit container
 
+%files
+  . /TADbit
+
 %environment
   LANG=C.UTF-8 
   LC_ALL=C.UTF-8
-  PATH="/usr/local/anaconda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+  PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
   export PATH LANG LC_ALL
 
 %post
-
-  # for R
-  echo "deb-src http://ftp.de.debian.org/debian/ buster main contrib" >> /etc/apt/sources.list
-
-  apt-get update --fix-missing
-
-  apt install -y dirmngr apt-transport-https ca-certificates software-properties-common gnupg2 wget
-  apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF'
-  add-apt-repository 'deb https://cloud.r-project.org/bin/linux/debian buster-cran35/'
-
-  apt-get update --fix-missing
-  apt-get -y install unzip build-essential wget bzip2 libxml2-dev libssl-dev libcurl4-gnutls-dev --no-install-recommends
+  apt-get update
+  apt-get -y install unzip \
+  build-essential \
+  wget \
+  librust-bzip2-dev \
+  libxml2-dev \
+  libssl-dev \
+  libcurl4-gnutls-dev \
+  liblzma-dev \
+  libncurses-dev \
+  git \
+  cmake \
+  libboost-all-dev \
+  r-base \
+  r-cran-devtools \
+  pigz \
+  zlib1g-dev \
+  libbz2-dev \
+  swig \
+  libhdf5-dev \
+  libeigen3-dev \
+  libcereal-dev \
+  mcl \
+  samtools
 
-  apt-get -y install r-base r-cran-devtools
+  apt-get -y install python3 python3-pip
+  python3 -m pip install pip --upgrade
+  python3 -m pip install numpy scipy future h5py xsamtools pysam matplotlib multiprocess mcl
 
-  #############################################################################
-  # DSRC
-  wget --quiet --no-check-certificate https://github.com/refresh-bio/DSRC/releases/download/v2.0.2/dsrc-linux-x64-static.tar.gz &&\
-  tar xzvf dsrc-linux-x64-static.tar.gz &&\
-  mv bin/dsrc /usr/local/bin && rm -f dsrc-linux-x64-static.tar.gz && chmod +x /usr/local/bin/dsrc
+  git clone https://github.com/salilab/imp.git
+  cd imp && ./setup_git.py
+  mkdir ../imp_release && cd ../imp_release && cmake ../imp -DCMAKE_BUILD_TYPE=Release && make -j 8 && cd ..
 
-  #############################################################################
-  # dryhic / oneD
-  R --slave -e 'options(repos="http://cran.rstudio.com/"); devtools::install_github("qenvio/dryhic")'
+  wget --quiet --no-check-certificate https://github.com/refresh-bio/DSRC/releases/download/v2.0.2/dsrc-linux-x64-static.tar.gz && \
+  tar xzvf dsrc-linux-x64-static.tar.gz && \
+  mv bin/dsrc /usr/local/bin && rm -f dsrc-linux-x64-static.tar.gz && chmod +x /usr/local/bin/dsrc
 
-  #############################################################################
-  # conda
-  echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \
-  wget --quiet --no-check-certificate https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \
-      -O $HOME/miniconda.sh
-  /bin/bash $HOME/miniconda.sh -b -p /usr/local/anaconda
-  rm $HOME/miniconda.sh
+  git clone --recursive https://github.com/smarco/gem3-mapper.git gem3-mapper
+  cd gem3-mapper && ./configure && make && cd ..
 
-  export PATH="/usr/local/anaconda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-
-  conda config --set always_yes yes --set changeps1 no
-  conda update -q conda
-  conda info -a
-
-  #############################################################################
-  # install python packages
-  conda config --add channels salilab && conda config --add channels bioconda && \
-  conda config --add channels hargup/label/pypi && conda config --add channels conda-forge
-  conda install -y -q imp
-  conda install -y -q future
-  conda install -y -q scipy
-  conda install -y -q numpy
-  conda install -y -q mcl
-  conda install -y -q matplotlib
-  conda install -y -q samtools
-  conda install -y -q pysam
-  conda install -y -q ucsc-bigWigToBedGraph
-  conda install -y -q ucsc-wigtobigwig
-  conda install -y -q gem3-mapper
-  conda install -y -q hisat2
-  # to run jupyter the notebok has to be built and installed as writable (with sudo)
-### conda install -y -q jupyter
-  conda clean -y --all  && rm -rf /opt/conda/pkgs/*
+  R --slave -e 'options(repos="http://cran.rstudio.com/"); devtools::install_github("qenvio/dryhic")'	
 
-  #############################################################################
-  # GEM
-  wget --quiet --no-check-certificate https://sourceforge.net/projects/gemlibrary/files/gem-library/Binary%20pre-release%203/GEM-binaries-Linux-x86_64-core_i3-20130406-045632.tbz2/download -O GEM.tbz2 && \
-  tar -xvjf GEM.tbz2 && \
-  rm -f GEM-binaries-*/bin/LICENCE && \
-  mv GEM-binaries-*/bin/* /usr/local/bin/ && rm -rf GEM*
-
-  #############################################################################
-  # TADbit
-  wget --quiet --no-check-certificate https://github.com/3DGenomes/TADbit/archive/master.zip && unzip master.zip && \
-  cd TADbit-master && yes | python setup.py install && cd .. && rm -rf TADbit-master master.zip
+  cd TADbit && PYTHONPATH=/imp_release/lib/:$PYTHONPATH python3 setup.py install && cd ..
 
-  ############################################################################
-  # Clean
-  rm -rf /var/lib/apt/lists/*
-
 %runscript
-  echo "Wecome to TADbit Singularity container"
+  echo "Welcome to TADbit Singularity container"
   exec /bin/bash --norc
-  # to run jupyter the notebok has to be built and installed as writable (with sudo)
-  # echo "Starting notebook..."
-  # echo "Open browser to localhost:8888"
-  # exec /opt/conda/bin/jupyter notebook --notebook-dir=/opt/notebooks --ip='*' --allow-root --port=8888 --no-browser