diff --git a/_pytadbit/mapping/full_mapper.py b/_pytadbit/mapping/full_mapper.py index 2abf6959..c0fa9d4e 100644 --- a/_pytadbit/mapping/full_mapper.py +++ b/_pytadbit/mapping/full_mapper.py @@ -277,7 +277,6 @@ def _sam_filter(fnam, fastq_path, unmap_out, map_out): """ Divides reads in a map file in two categories: uniquely mapped, and not. Writes them in two files - """ try: fhandler = Samfile(fnam) @@ -506,8 +505,7 @@ def _gem_mapping(gem_index_path, fastq_path, out_map_path, fastq_path2 = None, print(' '.join(gem_cmd)) try: # check_call(gem_cmd, stdout=PIPE, stderr=PIPE) - out, err = Popen(gem_cmd, stdout=PIPE, stderr=PIPE, - universal_newlines=True).communicate() + out, err = Popen(gem_cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True).communicate() except CalledProcessError as e: print(out) print(err) @@ -566,8 +564,7 @@ def full_mapping(mapper_index_path, fastq_path, out_map_dir, mapper='gem', suffix = ('_' * (suffix != '')) + suffix nthreads = kwargs.get('nthreads', 8) outfiles = [] - temp_dir = os.path.abspath(os.path.expanduser( - kwargs.get('temp_dir', gettempdir()))) + temp_dir = os.path.abspath(os.path.expanduser(kwargs.get('temp_dir', gettempdir()))) if mapper == 'gem': gem_version = None # check that we have the GEM binary: @@ -651,8 +648,7 @@ def full_mapping(mapper_index_path, fastq_path, out_map_dir, mapper='gem', if gem_version >= 3: _sam_filter(out_map_path, curr_map, curr_map + '_filt_%s-%s%s.map' % (beg, end, suffix), - os.path.join(out_map_dir, - base_name + '_full_%s-%s%s.map' % (beg, end, suffix))) + os.path.join(out_map_dir, base_name + '_full_%s-%s%s.map' % (beg, end, suffix))) else: _gem_filter(out_map_path, curr_map + '_filt_%s-%s%s.map' % (beg, end, suffix), @@ -929,4 +925,4 @@ def fast_fragment_mapping(mapper_index_path, fastq_path1, fastq_path2, r_enz, if get_nread: return [(out_map,count_fastq)] return out_map - \ No newline at end of file + diff --git a/_pytadbit/parsers/map_parser.py b/_pytadbit/parsers/map_parser.py index 37fd5d7d..de36ab41 100644 --- a/_pytadbit/parsers/map_parser.py +++ b/_pytadbit/parsers/map_parser.py @@ -112,13 +112,12 @@ def parse_map(f_names1, f_names2=None, out_file1=None, out_file2=None, # start parsing read_count = 0 try: - while not False: + while True: for _ in range(max_size): try: - reads.append(read_read(next(fhandler), frags, - frag_chunk)) + reads.append(read_read(next(fhandler), frags, frag_chunk)) except KeyError: - # Chromosome not in hash + print("Chromosome not in hash") continue read_count += 1 nfile += 1 @@ -260,6 +259,8 @@ def read_read_nofrags(r, _, __): try: crm, strand, pos = ali.split(':')[:3] except ValueError: + print(r) + print(ali) raise KeyError() positive = strand == '+' crm = crm.split()[0] diff --git a/_pytadbit/tools/tadbit_normalize.py b/_pytadbit/tools/tadbit_normalize.py index ff33b6c0..ecfe5f70 100644 --- a/_pytadbit/tools/tadbit_normalize.py +++ b/_pytadbit/tools/tadbit_normalize.py @@ -793,7 +793,12 @@ def read_bam(inbam, filter_exclude, resolution, min_count=2500, biases_path='', pool = mu.Pool(ncpus) procs = [] read_bam_frag = read_bam_frag_valid if only_valid else read_bam_frag_filter + + num_regs = len(regs) + regs = map(lambda r: r.replace("|", "_"), regs) + for i, (region, start, end) in enumerate(zip(regs, begs, ends)): + num_regs += 1 procs.append(pool.apply_async( read_bam_frag, args=(inbam, filter_exclude, bins, bins_dict, resolution, outdir, extra_out, @@ -803,13 +808,13 @@ def read_bam(inbam, filter_exclude, resolution, min_count=2500, biases_path='', pool.join() ## COLLECT RESULTS cisprc = {} - printime(' - Collecting cis and total interactions per bin (%d chunks)' % (len(regs))) + printime(' - Collecting cis and total interactions per bin (%d chunks)' % num_regs) stdout.write(' ') for countbin, (region, start, end) in enumerate(zip(regs, begs, ends)): if not countbin % 10 and countbin: stdout.write(' ') if not countbin % 50 and countbin: - stdout.write(' %9s\n ' % ('%s/%s' % (countbin , len(regs)))) + stdout.write(' %9s\n ' % ('%s/%s' % (countbin , num_regs))) stdout.write('.') stdout.flush() diff --git a/_pytadbit/utils/sqlite_utils.py b/_pytadbit/utils/sqlite_utils.py index 6ce644f9..c27e5f66 100644 --- a/_pytadbit/utils/sqlite_utils.py +++ b/_pytadbit/utils/sqlite_utils.py @@ -36,6 +36,9 @@ def digest_parameters(opts, get_md5=True, extra=None): if k not in ['fastq', 'index', 'renz', 'iterative', 'workdir', 'skip', 'func', 'tmp', 'keep_tmp'] + extra and v is not None]) parameters = parameters.replace("'", "") + parameters = parameters.replace("|", "_") + parameters = parameters.replace(";", "_") + parameters = parameters.replace("#", "_") return parameters diff --git a/containers/TADbit_singularity.def b/containers/TADbit_singularity.def index 7c153cc2..9a8ae884 100644 --- a/containers/TADbit_singularity.def +++ b/containers/TADbit_singularity.def @@ -1,95 +1,63 @@ BootStrap: docker -From:neurodebian:buster +From: debian:latest %help minimal TADbit container +%files + . /TADbit + %environment LANG=C.UTF-8 LC_ALL=C.UTF-8 - PATH="/usr/local/anaconda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export PATH LANG LC_ALL %post - - # for R - echo "deb-src http://ftp.de.debian.org/debian/ buster main contrib" >> /etc/apt/sources.list - - apt-get update --fix-missing - - apt install -y dirmngr apt-transport-https ca-certificates software-properties-common gnupg2 wget - apt-key adv --keyserver keys.gnupg.net --recv-key 'E19F5F87128899B192B1A2C2AD5F960A256A04AF' - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/debian buster-cran35/' - - apt-get update --fix-missing - apt-get -y install unzip build-essential wget bzip2 libxml2-dev libssl-dev libcurl4-gnutls-dev --no-install-recommends + apt-get update + apt-get -y install unzip \ + build-essential \ + wget \ + librust-bzip2-dev \ + libxml2-dev \ + libssl-dev \ + libcurl4-gnutls-dev \ + liblzma-dev \ + libncurses-dev \ + git \ + cmake \ + libboost-all-dev \ + r-base \ + r-cran-devtools \ + pigz \ + zlib1g-dev \ + libbz2-dev \ + swig \ + libhdf5-dev \ + libeigen3-dev \ + libcereal-dev \ + mcl \ + samtools - apt-get -y install r-base r-cran-devtools + apt-get -y install python3 python3-pip + python3 -m pip install pip --upgrade + python3 -m pip install numpy scipy future h5py xsamtools pysam matplotlib multiprocess mcl - ############################################################################# - # DSRC - wget --quiet --no-check-certificate https://github.com/refresh-bio/DSRC/releases/download/v2.0.2/dsrc-linux-x64-static.tar.gz &&\ - tar xzvf dsrc-linux-x64-static.tar.gz &&\ - mv bin/dsrc /usr/local/bin && rm -f dsrc-linux-x64-static.tar.gz && chmod +x /usr/local/bin/dsrc + git clone https://github.com/salilab/imp.git + cd imp && ./setup_git.py + mkdir ../imp_release && cd ../imp_release && cmake ../imp -DCMAKE_BUILD_TYPE=Release && make -j 8 && cd .. - ############################################################################# - # dryhic / oneD - R --slave -e 'options(repos="http://cran.rstudio.com/"); devtools::install_github("qenvio/dryhic")' + wget --quiet --no-check-certificate https://github.com/refresh-bio/DSRC/releases/download/v2.0.2/dsrc-linux-x64-static.tar.gz && \ + tar xzvf dsrc-linux-x64-static.tar.gz && \ + mv bin/dsrc /usr/local/bin && rm -f dsrc-linux-x64-static.tar.gz && chmod +x /usr/local/bin/dsrc - ############################################################################# - # conda - echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh && \ - wget --quiet --no-check-certificate https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh \ - -O $HOME/miniconda.sh - /bin/bash $HOME/miniconda.sh -b -p /usr/local/anaconda - rm $HOME/miniconda.sh + git clone --recursive https://github.com/smarco/gem3-mapper.git gem3-mapper + cd gem3-mapper && ./configure && make && cd .. - export PATH="/usr/local/anaconda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - - conda config --set always_yes yes --set changeps1 no - conda update -q conda - conda info -a - - ############################################################################# - # install python packages - conda config --add channels salilab && conda config --add channels bioconda && \ - conda config --add channels hargup/label/pypi && conda config --add channels conda-forge - conda install -y -q imp - conda install -y -q future - conda install -y -q scipy - conda install -y -q numpy - conda install -y -q mcl - conda install -y -q matplotlib - conda install -y -q samtools - conda install -y -q pysam - conda install -y -q ucsc-bigWigToBedGraph - conda install -y -q ucsc-wigtobigwig - conda install -y -q gem3-mapper - conda install -y -q hisat2 - # to run jupyter the notebok has to be built and installed as writable (with sudo) -### conda install -y -q jupyter - conda clean -y --all && rm -rf /opt/conda/pkgs/* + R --slave -e 'options(repos="http://cran.rstudio.com/"); devtools::install_github("qenvio/dryhic")' - ############################################################################# - # GEM - wget --quiet --no-check-certificate https://sourceforge.net/projects/gemlibrary/files/gem-library/Binary%20pre-release%203/GEM-binaries-Linux-x86_64-core_i3-20130406-045632.tbz2/download -O GEM.tbz2 && \ - tar -xvjf GEM.tbz2 && \ - rm -f GEM-binaries-*/bin/LICENCE && \ - mv GEM-binaries-*/bin/* /usr/local/bin/ && rm -rf GEM* - - ############################################################################# - # TADbit - wget --quiet --no-check-certificate https://github.com/3DGenomes/TADbit/archive/master.zip && unzip master.zip && \ - cd TADbit-master && yes | python setup.py install && cd .. && rm -rf TADbit-master master.zip + cd TADbit && PYTHONPATH=/imp_release/lib/:$PYTHONPATH python3 setup.py install && cd .. - ############################################################################ - # Clean - rm -rf /var/lib/apt/lists/* - %runscript - echo "Wecome to TADbit Singularity container" + echo "Welcome to TADbit Singularity container" exec /bin/bash --norc - # to run jupyter the notebok has to be built and installed as writable (with sudo) - # echo "Starting notebook..." - # echo "Open browser to localhost:8888" - # exec /opt/conda/bin/jupyter notebook --notebook-dir=/opt/notebooks --ip='*' --allow-root --port=8888 --no-browser