Repository: LabShengLi/nanome (language: python)

Issue:
Adds GPU drivers (CUDA) in the container
:warning: Do not merge yet - to be discussed.

This PR is addressing the issue found `libcuda.so.1` not found that affects `guppy_basecaller` command.


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
old mode 100644
new mode 100755
diff --git a/Dockerfile b/Dockerfile
old mode 100644
new mode 100755
index f44cefcf..b2cc6b01
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,9 +2,18 @@ FROM genomicpariscentre/guppy-gpu:4.2.2
 LABEL description="Nanocompare project" \
       author="yang.liu@jax.org"
 RUN apt-get update -y \
-  && apt-get install procps git -y \
+  && DEBIAN_FRONTEND=noninteractive apt-get install procps git curl -y \
   && rm -rf /var/lib/apt/lists/*
  
+RUN apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y software-properties-common && \
+    curl -O http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-ubuntu1604.pin && \
+    mv cuda-ubuntu1604.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \
+    apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/7fa2af80.pub && \
+    add-apt-repository "deb http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/ /" && \
+    apt update && \
+    apt -y install cuda
+
 #Install miniconda
 RUN wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O Miniconda.sh && \
     /bin/bash Miniconda.sh -b -p /opt/conda && \
@@ -23,4 +32,16 @@ SHELL ["conda", "run", "-n", "nanocompare", "/bin/bash", "-c"]
 ENV PATH /opt/conda/envs/nanocompare/bin:$PATH
 USER root
 WORKDIR /data/
+
+# Copy additonal scripts
+RUN mkdir /opt/bin
+ADD utils/ /opt/bin/utils
+ADD src/ /opt/bin/src
+RUN chmod +x /opt/bin/utils/*
+RUN chmod +x /opt/bin/src/*
+RUN chmod +x /opt/bin/src/nanocompare/*
+ENV PATH="$PATH:/opt/bin/utils"
+ENV PATH="$PATH:/opt/bin/src"
+ENV PATH="$PATH:/opt/bin/src/nanocompare"
+
 CMD ["bash"]
diff --git a/README.md b/README.md
old mode 100644
new mode 100755
index af84ba87..3abf0f69
--- a/README.md
+++ b/README.md
@@ -73,6 +73,10 @@ Our HPC hardware specifications are as follows:
 
 
 ## Installation
+The 'nanome' pipeline support running with various ways:
+* Singularity container
+* Docker container
+* Conda environment
 
 We will update within a short time.
 
diff --git a/conf/NA12878.config b/conf/NA12878.config
new file mode 100755
index 00000000..71b7ae13
--- /dev/null
+++ b/conf/NA12878.config
@@ -0,0 +1,20 @@
+/*
+ * -------------------------------------------------
+ *  Nextflow config file for running NA12878
+ * -------------------------------------------------
+ * Defines bundled input files and everything required
+ * to run a fast and simple test.
+ */
+
+params {
+	// Methylation calling params
+	dsname = 'NA12878CHR20'
+	input = 'inputs/NA12878CHR20.filelist.txt'
+
+	// For evaluation/analysis configurations
+	eval=false // true if running evaluation steps
+
+	// NA12878 is basecalled from downloaded fast5, we need to clean previous analysis
+	cleanAnalyses=true // clean previous basecalled analyses name
+}
+
diff --git a/conf/benchmarking.config b/conf/benchmarking.config
old mode 100644
new mode 100755
diff --git a/conf/ecoli_demo.config b/conf/ecoli_demo.config
old mode 100644
new mode 100755
diff --git a/conf/google.config b/conf/google.config
old mode 100644
new mode 100755
index 0a2f7e21..46f787f9
--- a/conf/google.config
+++ b/conf/google.config
@@ -13,17 +13,18 @@ google {
 }
 
 process {
-  container = "quay.io/lifebitai/nanome"
+  container = "quay.io/lifebitai/nanome:shebang-cuda"
   echo = params.echo
 
   withLabel: with_gpus {
     cpus = params.with_gpus_process_cpus
     memory =  params.with_gpus_process_memory
-    container = "quay.io/lifebitai/nanome"
+    container = "quay.io/lifebitai/nanome:shebang-cuda"
     containerOptions = params.containerOptions
     accelerator = [request: params.n_accelerators, type: params.accelerator ]
     params.GuppyGPUOptions="--gpu_runners_per_device 16 --device auto"
-    params.DeepSignal_isgpu = 'yes'
     params.GuppyNumCallers=16
+    params.megalodonGPUOptions="--devices 0"
+    params.DeepSignal_isgpu = 'yes'
 	}
 }
diff --git a/conf/hpc.config b/conf/hpc.config
old mode 100644
new mode 100755
diff --git a/conf/local.config b/conf/local.config
old mode 100644
new mode 100755
diff --git a/conf/standard.config b/conf/standard.config
old mode 100644
new mode 100755
diff --git a/docs/Fig1.jpg b/docs/Fig1.jpg
old mode 100644
new mode 100755
diff --git a/docs/Fig1.png b/docs/Fig1.png
old mode 100644
new mode 100755
diff --git a/docs/Fig2.jpg b/docs/Fig2.jpg
old mode 100644
new mode 100755
diff --git a/docs/Usage.md b/docs/Usage.md
old mode 100644
new mode 100755
index df25b142..cbbc315c
--- a/docs/Usage.md
+++ b/docs/Usage.md
@@ -16,6 +16,14 @@ curl -fsSL get.nextflow.io | bash
    --dsname TestData --eval true -profile winter
 ```
 
+You can also running the pipeline on CloudOS, using following command options.
+```angular2html
+nextflow run https://github.com/liuyangzzu/nanome
+    --config 'conf/google.config'
+    --dsname TestData
+    --input 'https://github.com/liuyangzzu/nanome/raw/master/test_data/demo.fast5.reads.tar.gz'
+```
+
 Command running results is below.
 
 ```angular2html
diff --git a/docs/fig.s2.workflow.jpg b/docs/fig.s2.workflow.jpg
old mode 100644
new mode 100755
diff --git a/docs/nanome.pipeline_trace.tsv b/docs/nanome.pipeline_trace.tsv
old mode 100644
new mode 100755
diff --git a/docs/nanome.pipeline_trace.txt b/docs/nanome.pipeline_trace.txt
old mode 100644
new mode 100755
diff --git a/docs/report.html b/docs/report.html
old mode 100644
new mode 100755
diff --git a/docs/reports.pdf b/docs/reports.pdf
old mode 100644
new mode 100755
diff --git a/docs/reports2.pdf b/docs/reports2.pdf
old mode 100644
new mode 100755
diff --git a/docs/timeline.html b/docs/timeline.html
old mode 100644
new mode 100755
diff --git a/docs/timeline.pdf b/docs/timeline.pdf
old mode 100644
new mode 100755
diff --git a/environment.yml b/environment.yml
old mode 100644
new mode 100755
diff --git a/inputs/NA12878CHR20.filelist.txt b/inputs/NA12878CHR20.filelist.txt
new file mode 100755
index 00000000..81c45fbe
--- /dev/null
+++ b/inputs/NA12878CHR20.filelist.txt
@@ -0,0 +1,5 @@
+http://s3.amazonaws.com/nanopore-human-wgs/rel3-fast5-chr20.part01.tar
+http://s3.amazonaws.com/nanopore-human-wgs/rel3-fast5-chr20.part02.tar
+http://s3.amazonaws.com/nanopore-human-wgs/rel3-fast5-chr20.part03.tar
+http://s3.amazonaws.com/nanopore-human-wgs/rel3-fast5-chr20.part04.tar
+http://s3.amazonaws.com/nanopore-human-wgs/rel3-fast5-chr20.part05.tar
diff --git a/inputs/benchmark.filelist.txt b/inputs/benchmark.filelist.txt
old mode 100644
new mode 100755
diff --git a/main.nf b/main.nf
old mode 100644
new mode 100755
index df6f77d3..aa8c612c
--- a/main.nf
+++ b/main.nf
@@ -2,7 +2,7 @@
 
 log.info """\
 NANOME - NF PIPELINE (v1.0)
-by Li Lab at The Jackon Laboratory
+by Li Lab at The Jackson Laboratory
 http://nanome.jax.org
 =================================
 dsname			:${params.dsname}
@@ -16,6 +16,12 @@ evaluation		:${params.eval}
 """
 .stripIndent()
 
+projectDir = workflow.projectDir
+ch_utils = Channel.fromPath("${projectDir}/utils",  type: 'dir', followLinks: false)
+ch_src   = Channel.fromPath("${projectDir}/src",  type: 'dir', followLinks: false)
+
+ch_utils.into{ch_utils1; ch_utils2; ch_utils3}
+ch_src.into{ch_src1; ch_src2}
 
 // We collect all folders of fast5 files, and send into Channels for pipelines
 if (params.input.endsWith(".filelist.txt")) { // filelist
@@ -39,9 +45,10 @@ process EnvCheck {
 	errorStrategy 'terminate'
 	label 'with_gpus'
 
-	"""
-	set -x
+	output:
+	file nanome into nanome_ch
 
+	"""
 	which guppy_basecaller
 	guppy_basecaller -v
 
@@ -59,9 +66,12 @@ process EnvCheck {
 
 	which DeepMod.py
 	DeepMod.py
+
+	git clone https://github.com/liuyangzzu/nanome.git
 	"""
 }
 
+// nanome_ch.into{nanome_ch0; nanome_ch1; nanome_ch2; nanome_ch3; nanome_ch4}
 
 // basecall of subfolders named 'M1', ..., 'M10', etc.
 process Basecall {
@@ -71,6 +81,7 @@ process Basecall {
 
 	input: // input here is not passed
 	file fast5_tar from fast5_tar_ch1
+	each file("*") from ch_utils1
 
 	output:
 	file "${fast5_tar.simpleName}_basecalled" into basecall_out_ch  // try to fix the christina proposed problems
@@ -80,8 +91,6 @@ process Basecall {
 	params.runBasecall
 
 	"""
-	set -x
-
 	mkdir -p untarDir
 
 	infn=${fast5_tar}
@@ -100,7 +109,8 @@ process Basecall {
 
 	## Clean old analyses in input fast5 files
 	if [[ "${params.cleanAnalyses}" = true ]] ; then
-		python ${workflow.projectDir}/utils/clean_old_basecall_in_fast5.py -i untarDir1 --is-indir
+		echo "### Start cleaning old analysis"
+		python utils/clean_old_basecall_in_fast5.py -i untarDir1 --is-indir
 	fi
 
 	mkdir -p ${fast5_tar.simpleName}_basecalled
@@ -143,7 +153,7 @@ basecall_out_ch
 	.into { resquiggle_in_ch; nanopolish_in_ch; deepmod_in_ch }
 
 
-// resquiggle on basecalled subfolders named 'M1', ..., 'M10', etc.
+// Resquiggle on basecalled subfolders named 'M1', ..., 'M10', etc.
 process Resquiggle {
 	tag "${basecallIndir.simpleName}"
 
@@ -158,14 +168,15 @@ process Resquiggle {
 	params.runMethcall
 
 	"""
-	set -x
-
 	## untar reference_genome
 	tar -xzf ${reference_genome_tar}
 	refGenome=${params.referenceGenome}
 
 	### copy basecall workspace files, due to tombo resquiggle modify base folder
 	mkdir -p ${basecallIndir.simpleName}_resquiggle_dir
+
+	### TODO: check if it is ok? We need to duplicate basecall results,
+	### original basecalled results will be parrallelly used by other processes
 	cp -rf ${basecallIndir}/* ${basecallIndir.simpleName}_resquiggle_dir/
 
 	### Now set processes=1, to avoid Tombo bug of BrokenPipeError, it is very fast even set to 1.
@@ -195,7 +206,7 @@ process DeepSignal {
 	label 'with_gpus'
 
 	input:
-	file ttt from deepsignal_in2_ch //[basecallDir, reference_genome]
+	file ttt from deepsignal_in2_ch // ttt is [basecallDir, reference_genome]
 	each file(deepsignal_model_tar) from Channel.fromPath(params.deepsignel_model_tar)
 
 	output:
@@ -229,6 +240,7 @@ process Tombo {
 
 	input:
 	each file(reference_genome_tar) from Channel.fromPath(params.reference_genome_tar)
+	each file("*") from ch_utils2
 	file resquiggleDir from tombo_in_ch
 
 	output:
@@ -238,7 +250,6 @@ process Tombo {
 	params.runMethcall
 
 	"""
-	set -x
 	tar -xzf ${reference_genome_tar}
 
 	tombo detect_modifications alternative_model \
@@ -251,7 +262,7 @@ process Tombo {
 		--processes ${params.processors} \
 		--corrected-group ${params.resquiggleCorrectedGroup}
 
-	python ${workflow.projectDir}/utils/tombo_extract_per_read_stats.py \
+	python utils/tombo_extract_per_read_stats.py \
 		${params.chromSizesFile} \
 		"batch_${resquiggleDir.simpleName}.CpG.tombo.per_read_stats" \
 		"batch_${resquiggleDir.simpleName}.CpG.tombo.per_read_stats.bed"
@@ -277,8 +288,6 @@ process Megalodon {
 	params.runMethcall
 
 	"""
-	set -x
-
 	## Get raw fast5 files
 	mkdir -p untarDir
 	infn=${fast5_tar}
@@ -339,8 +348,6 @@ process DeepMod {
 	params.runDeepMod
 
 	"""
-	set -x
-
 	wget ${params.DeepModGithub} --no-verbose
 	tar -xzf v0.1.3.tar.gz
 	DeepModProjectDir="DeepMod-0.1.3"
@@ -372,6 +379,7 @@ process Nanopolish {
 	input:
 	file basecallDir from nanopolish_in_ch
 	each file (reference_genome_tar) from Channel.fromPath(params.reference_genome_tar)
+	each file("*") from ch_utils3
 
 	output:
 	file "batch_${basecallDir.simpleName}.nanopolish.methylation_calls.tsv" into nanopolish_out_ch
@@ -380,8 +388,6 @@ process Nanopolish {
 	params.runMethcall
 
 	"""
-	set -x
-
 	tar -xzf ${reference_genome_tar}
 	refGenome=${params.referenceGenome}
 
@@ -403,7 +409,7 @@ process Nanopolish {
 		# echo "cat \$f >> \$fastqFile - COMPLETED"
 	done
 
-	python ${workflow.projectDir}/utils/nanopore_nanopolish_preindexing_checkDups.py \${fastqFile} \${fastqNoDupFile}
+	python utils/nanopore_nanopolish_preindexing_checkDups.py \${fastqFile} \${fastqNoDupFile}
 
 	# Index the raw read with fastq
 	nanopolish index -d ${basecallDir}/workspace \${fastqNoDupFile}
@@ -524,8 +530,6 @@ process NplshComb {
 	x.size() >= 1
 
 	"""
-	set -x
-
 	> ${params.dsname}.Nanopolish.combine.tsv
 
 	for fn in $x
@@ -560,8 +564,6 @@ process DpmodComb {
 	x.size() >= 1
 
 	"""
-	set -x
-
 	wget ${params.DeepModGithub} --no-verbose
 	tar -xzf v0.1.3.tar.gz
 	DeepModProjectDir="DeepMod-0.1.3"
@@ -620,6 +622,7 @@ process ReadLevelPerf {
 
 	input: // TODO: I can not sort fileList by name, seems sorted by date????
 	file fileList from readlevel_in_ch
+	each file("*") from ch_src1
 
 	output:
 	file "MethPerf-*" into readlevel_out_ch
@@ -628,8 +631,6 @@ process ReadLevelPerf {
 	params.eval && (fileList.size() >= 1)
 
 	"""
-	set -x
-
 	# Sort files
 	flist=(\$(ls *.combine.{tsv,bed}.gz))
 
@@ -644,7 +645,9 @@ process ReadLevelPerf {
 	export PYTHONPATH=${workflow.projectDir}:\${PYTHONPATH}
 
 	## Read level evaluations
-	python ${workflow.projectDir}/src/nanocompare/read_level_eval.py \
+	### python ${workflow.projectDir}/src/nanocompare/read_level_eval.py \
+	python src/nanocompare/read_level_eval.py \
+
 		--calls DeepSignal:\${deepsignalFile} \
 				Tombo:\${tomboFile} \
 				Nanopolish:\${nanopolishFile} \
@@ -668,6 +671,7 @@ process SiteLevelCorr {
 	input:
 	file perfDir from readlevel_out_ch
 	file fileList from sitelevel_in_ch
+	each file("*") from ch_src2
 
 	output:
 	file "MethCorr-*" into sitelevel_out_ch
@@ -676,8 +680,6 @@ process SiteLevelCorr {
 	params.eval && (fileList.size() >= 1)
 
 	"""
-	set -x
-
 	# Sort file by my self
 	flist=(\$(ls *.combine.{tsv,bed}.gz))
 	echo \${flist[@]}
@@ -691,7 +693,8 @@ process SiteLevelCorr {
 	export PYTHONPATH=${workflow.projectDir}:\${PYTHONPATH}
 
 	## Site level evaluations
-	python ${workflow.projectDir}/src/nanocompare/site_level_eval.py \
+	### python ${workflow.projectDir}/src/nanocompare/site_level_eval.py \
+	python src/nanocompare/site_level_eval.py \
 		--calls DeepSignal:\${deepsignalFile} \
 				Tombo:\${tomboFile} \
 				Nanopolish:\${nanopolishFile} \
diff --git a/main.sbatch b/main.sbatch
old mode 100644
new mode 100755
index 1eb716f4..700cd0d3
--- a/main.sbatch
+++ b/main.sbatch
@@ -6,11 +6,10 @@
 #SBATCH -N 1 # number of nodes
 #SBATCH -n 2 # number of cores
 #SBATCH --mem=70G # memory pool for all cores
-#SBATCH --time=08:00:00 # time
+#SBATCH --time=18:00:00 # time
 #SBATCH -o log/%x.%j.out # STDOUT
 #SBATCH -e log/%x.%j.err # STDERR
 
-
 set -x
 
 #cd $SLURM_SUBMIT_DIR
@@ -32,10 +31,14 @@ workingDir=/fastscratch/liuya/nanocompare/work
 # Running nanome pipeline on test data (from NA19240) in Human Genome
 #rm -rf ${workingDir}-test
 ./nextflow run main.nf -profile winter \
-    -work-dir ${workingDir}-test3 \
+    -work-dir ${workingDir}-test4 \
     -config conf/test.config
 
 
+#./nextflow run main.nf -profile winter \
+#    -work-dir ${workingDir}-test3 \
+#    -config conf/NA12878.config
+
 ###############################################################
 ###############################################################
 ###############################################################
diff --git a/nextflow.config b/nextflow.config
old mode 100644
new mode 100755
index 60e76398..4a2f98b5
--- a/nextflow.config
+++ b/nextflow.config
@@ -77,6 +77,7 @@ params {
 	GUPPY_TIMEOUT = 240
 	READS_PER_GUPPY_BATCH = 100
 	SAMTOOLS_PATH="samtools"
+	megalodonGPUOptions=""
 
 	// DeepMod default used model specifications
 	DeepModGithub="https://github.com/WGLab/DeepMod/archive/refs/tags/v0.1.3.tar.gz"
diff --git a/src/nanocompare/Makefile b/src/nanocompare/Makefile
old mode 100644
new mode 100755
diff --git a/src/nanocompare/__init__.py b/src/nanocompare/__init__.py
old mode 100644
new mode 100755
diff --git a/src/nanocompare/benchmarkingDataConstruct.py b/src/nanocompare/benchmarkingDataConstruct.py
old mode 100644
new mode 100755
index a17a8f06..a21237d4
--- a/src/nanocompare/benchmarkingDataConstruct.py
+++ b/src/nanocompare/benchmarkingDataConstruct.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 Construct the benchmarking data for resource usage evaluation
 """
diff --git a/src/nanocompare/benchmarkingDataConstruct.sh b/src/nanocompare/benchmarkingDataConstruct.sh
old mode 100644
new mode 100755
diff --git a/src/nanocompare/computeRawReadsCoverage.py b/src/nanocompare/computeRawReadsCoverage.py
old mode 100644
new mode 100755
index 35989eb5..aafe9b29
--- a/src/nanocompare/computeRawReadsCoverage.py
+++ b/src/nanocompare/computeRawReadsCoverage.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import glob
 import os
 import re
diff --git a/src/nanocompare/computeRawReadsCoverage.sh b/src/nanocompare/computeRawReadsCoverage.sh
old mode 100644
new mode 100755
diff --git a/src/nanocompare/eval_common.py b/src/nanocompare/eval_common.py
index b1d018e7..09193fa0 100755
--- a/src/nanocompare/eval_common.py
+++ b/src/nanocompare/eval_common.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 Common functions used by read level and site level evaluations in Nanocompare paper.
 
diff --git a/src/nanocompare/eval_common.sh b/src/nanocompare/eval_common.sh
old mode 100644
new mode 100755
diff --git a/src/nanocompare/global_config.py b/src/nanocompare/global_config.py
old mode 100644
new mode 100755
index 56f9e3c6..2297d836
--- a/src/nanocompare/global_config.py
+++ b/src/nanocompare/global_config.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 =================================================================
 Global variable initializations
diff --git a/src/nanocompare/global_settings.py b/src/nanocompare/global_settings.py
old mode 100644
new mode 100755
index ec4546af..1d242591
--- a/src/nanocompare/global_settings.py
+++ b/src/nanocompare/global_settings.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 Define names and global variables
 
diff --git a/src/nanocompare/read_level_eval.py b/src/nanocompare/read_level_eval.py
index 6e3438a4..18b705f6 100755
--- a/src/nanocompare/read_level_eval.py
+++ b/src/nanocompare/read_level_eval.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 Read-level evaluation on methylation calls of tools, compute the performance results(F1, accuracy, ROC-AUC, etc.) in Nanocompare paper.
 
diff --git a/src/nanocompare/read_level_eval_submit.py b/src/nanocompare/read_level_eval_submit.py
index 4269612d..87717048 100755
--- a/src/nanocompare/read_level_eval_submit.py
+++ b/src/nanocompare/read_level_eval_submit.py
@@ -1,4 +1,5 @@
-#!/home/liuya/anaconda3/envs/nmf/bin/python
+#!/usr/bin/env python3
+
 """
 Submit read level evaluation jobs for each line input of tsv.
 """
diff --git a/src/nanocompare/resource/__init__.py b/src/nanocompare/resource/__init__.py
old mode 100644
new mode 100755
diff --git a/src/nanocompare/resource/resource_summary.py b/src/nanocompare/resource/resource_summary.py
old mode 100644
new mode 100755
diff --git a/src/nanocompare/site_level_eval.py b/src/nanocompare/site_level_eval.py
index bb25ba36..b61ee9da 100755
--- a/src/nanocompare/site_level_eval.py
+++ b/src/nanocompare/site_level_eval.py
@@ -1,4 +1,4 @@
-#!/home/liuya/anaconda3/envs/nmf/bin/python
+#!/usr/bin/env python3
 
 """
 Generate site-level methylation correlation results in Nanocompare paper.
diff --git a/src/nanocompare/site_level_eval_submit.py b/src/nanocompare/site_level_eval_submit.py
index b13b38a0..e554d1d7 100755
--- a/src/nanocompare/site_level_eval_submit.py
+++ b/src/nanocompare/site_level_eval_submit.py
@@ -1,4 +1,5 @@
-#!/home/liuya/anaconda3/envs/nmf/bin/python
+#!/usr/bin/env python3
+
 """
 Submit site level evaluation jobs for each line input of tsv.
 """
diff --git a/src/nanocompare/tss_eval.py b/src/nanocompare/tss_eval.py
index 711bc1d0..a71609db 100755
--- a/src/nanocompare/tss_eval.py
+++ b/src/nanocompare/tss_eval.py
@@ -1,4 +1,4 @@
-#!/home/liuya/anaconda3/envs/nmf/bin/python
+#!/usr/bin/env python3
 
 """
 Generate site-level methylation correlation results in Nanocompare paper.
diff --git a/src/nanocompare/tss_eval_submit.py b/src/nanocompare/tss_eval_submit.py
index 493aec25..aafc0568 100755
--- a/src/nanocompare/tss_eval_submit.py
+++ b/src/nanocompare/tss_eval_submit.py
@@ -1,4 +1,4 @@
-#!/home/liuya/anaconda3/envs/nmf/bin/python
+#!/usr/bin/env python3
 
 import csv
 import os
diff --git a/src/performance_of_read_level_in_paper.tsv b/src/performance_of_read_level_in_paper.tsv
old mode 100644
new mode 100755
diff --git a/src/plot_figure.py b/src/plot_figure.py
old mode 100644
new mode 100755
index c9b6bfa9..5b7bf006
--- a/src/plot_figure.py
+++ b/src/plot_figure.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 Plots and export data for nanome paper
 """
diff --git a/src/plot_figure.sh b/src/plot_figure.sh
old mode 100644
new mode 100755
diff --git a/src/site_level_correlation_submit.sh b/src/site_level_correlation_submit.sh
old mode 100644
new mode 100755
diff --git a/src/tss_sites_analysis_submit.sh b/src/tss_sites_analysis_submit.sh
old mode 100644
new mode 100755
diff --git a/utils/FilesSeparatorNew.py b/utils/FilesSeparatorNew.py
index 136435ec..465cdb7d 100755
--- a/utils/FilesSeparatorNew.py
+++ b/utils/FilesSeparatorNew.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 Seperate all fast5 file into many subfolders 1--N
 """
diff --git a/utils/clean_old_basecall_in_fast5.py b/utils/clean_old_basecall_in_fast5.py
old mode 100644
new mode 100755
index c6152508..3051fa33
--- a/utils/clean_old_basecall_in_fast5.py
+++ b/utils/clean_old_basecall_in_fast5.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 This tool is to clean old basecall analyses for NA12878, etc.
 """
diff --git a/utils/hm_cluster_predict.py b/utils/hm_cluster_predict.py
old mode 100644
new mode 100755
index 25b21d82..9a25f606
--- a/utils/hm_cluster_predict.py
+++ b/utils/hm_cluster_predict.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os, sys, time
 from collections import defaultdict
diff --git a/utils/nanopore_nanopolish_preindexing_checkDups.py b/utils/nanopore_nanopolish_preindexing_checkDups.py
old mode 100644
new mode 100755
index cc4a943d..a9c2ab3f
--- a/utils/nanopore_nanopolish_preindexing_checkDups.py
+++ b/utils/nanopore_nanopolish_preindexing_checkDups.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 """
 remove duplicate reads of $1 fastq file, save it into $2 fastq file
 """
diff --git a/utils/sum_chr_mod.py b/utils/sum_chr_mod.py
old mode 100644
new mode 100755
index e5691867..90d28146
--- a/utils/sum_chr_mod.py
+++ b/utils/sum_chr_mod.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import os, sys, string
 import multiprocessing
diff --git a/utils/tombo_extract_per_read_stats.py b/utils/tombo_extract_per_read_stats.py
old mode 100644
new mode 100755
index f72ff2aa..829e0ea3
--- a/utils/tombo_extract_per_read_stats.py
+++ b/utils/tombo_extract_per_read_stats.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 from sys import argv
 
 import numpy as np


Test patch:
diff --git a/conf/test.config b/conf/test.config
old mode 100644
new mode 100755
diff --git a/src/nanocompare/test_script.py b/src/nanocompare/test_script.py
old mode 100644
new mode 100755
index d5324b3e..d20b2b3f
--- a/src/nanocompare/test_script.py
+++ b/src/nanocompare/test_script.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import os, psutil
 process = psutil.Process(os.getpid())
 print(process.memory_info().rss/1000/1000)  # in bytes GB
diff --git a/test_data/demo.fast5.reads.tar.gz b/test_data/demo.fast5.reads.tar.gz
old mode 100644
new mode 100755