hdgwas/hdregression.py
def C_matrix(phenotype):
	C=np.einsum('ij,ji->i', phenotype.T, phenotype)
	return C
hdgwas/hdregression.py
def B_covariates(covariates, phenotype, intercept=True):

	S,N=covariates.shape

	b_cov = np.dot(covariates.T, phenotype)
	if intercept:
		b1 = np.sum(phenotype, axis=0).reshape(1, phenotype.shape[1])
		B13 = np.append(b1, b_cov, axis=0)
		return B13
	else:
		return b_cov
hdgwas/converter.py
class GenotypePLINK(GenotypeHDF5):

	def __init__(self, name, reader=None):
		super(GenotypePLINK, self).__init__(name)
		self.reader=reader
		self.split_size=None


	def convert_individuals(self):

		individuals=self.reader.folder.get_fam()

		self.h5_ind_file.create_table(self.h5_ind_file.root, 'individuals', individuals,
									 title='Individuals', filters=self.pytable_filters)
		self.h5_ind_file.root.individuals[:] = individuals
		self.individuals = self.h5_ind_file.root.individuals[:]
		self.n_ind=len(individuals)


	#@profile
	def convert_probes(self, chunk_size=100000):

		if os.path.isfile(os.path.join(self.out,'probes',self.h5_name)):
			os.remove(os.path.join(self.out,'probes',self.h5_name))
		hash_table={'keys':np.array([],dtype=np.int),'allele':np.array([])}
		i=0
		chunk=np.array([])
		while True:
			chunk=self.reader.folder.get_bim(chunk_size)
			if isinstance(chunk,type(None)):
				break
			print i
			chunk.columns=['CHR', 'ID', 'distance', 'bp', 'allele1', 'allele2']
			hash_1=chunk.allele1.apply(hash)
			hash_2=chunk.allele2.apply(hash)
			k,indices=np.unique(np.append(hash_1,hash_2),return_index=True)
			s=np.append(chunk.allele1,chunk.allele2)[indices]
			ind=np.invert(np.in1d(k,hash_table['keys']))
			hash_table['keys']=np.append(hash_table['keys'],k[ind])
			hash_table['allele']=np.append(hash_table['allele'],s[ind])
			chunk.allele1=hash_1
			chunk.allele2=hash_2
			# WARNING!!! doesn't work on windows
			chunk.to_hdf(os.path.join(self.out,'probes',self.h5_name), key='probes',format='table',data_columns=True, append=True,
						 complib='zlib',complevel=9, min_itemsize = 45)
			gc.collect()
			i+=1
		pd.DataFrame.from_dict(hash_table).to_csv(os.path.join(self.out,'probes',self.file_name+'_hash_table.csv.gz'),index=False,compression='gzip', sep='\t')
		print('Number of Probes {} converted'.format(self.reader.folder.N_probes))

	#@profile
	def convert_genotypes(self):


		chunk_size=self.split_size
		if chunk_size is None:
			raise ValueError('CONVERTER_SPLIT_SIZE does not define in config file!')
		G=np.array([])
		#self.reader.folder.processed=0
		while True:
			with Timer() as t:
				G=self.reader.folder.get_bed(chunk_size)
				if isinstance(G,type(None)):
					break

			print ('Time to read {} SNPs is {} s'.format(G.shape[0], t.secs))

			self.write_data('gen')
			atom = tables.Int8Atom()
			self.genotype = self.h5_gen_file.create_carray(self.h5_gen_file.root, 'genotype', atom,
														  (G.shape),
														  title='Genotype', filters=self.pytable_filters)
			with Timer() as t:
				self.genotype[:] = G

			print ('Time to write {} SNPs is {} s'.format(G.shape[0], t.secs))

			self.h5_gen_file.close()
			G=None
			gc.collect()



	def plink2hdf5(self,out, force=True):

		if force:
			try:
				os.mkdir(os.path.join(out,'genotype') )
				os.mkdir(os.path.join(out,'individuals') )
				os.mkdir(os.path.join(out,'probes') )
			except:
				print('Directories "genotype","probes","individuals" are already exist in {}...'.format(self.out))

		self.out=out
		self.write_data('ind')
		self.convert_individuals()
		self.h5_ind_file.close()

		self.reader.folder.processed=0
		self.write_data('pr')
		self.convert_probes()

		self.reader.folder.processed=0
		self.convert_genotypes()


	def _summary(self, head=10):
		pass #TODO (low) rewrite to get statistic not load to memory

		print('Number of Probes: %d' % None)
		print('Number of Individuals: %d' % None)
		print('The genotype matrix is of size %d by %d' % None)
config.py
MAPPER_CHUNK_SIZE=5000 #TODO (low) move to hase
CONVERTER_SPLIT_SIZE=25000 #TODO (low) move to hase
PYTHON_PATH=None
hdgwas/converter.py
class GenotypeVCF(object):

	def __init__(self, name, reader=None):
		self.reader=reader
		self.study_name=name
		self.split_size=None
		self.hdf5_iter=0
		self.pytable_filter=tables.Filters(complevel=9, complib='zlib')
		self.cluster=False

	def VCF2hdf5(self, out):

		FNULL = open(os.devnull, 'w')
		subprocess.call(['bash',os.path.join(os.environ['HASEDIR'],'tools','VCF2hdf5.sh'),
						 self.reader.folder.path, out , os.environ['HASEDIR'], self.study_name ], shell=False,stderr=subprocess.STDOUT,
						preexec_fn=lambda:signal.signal(signal.SIGPIPE, signal.SIG_DFL)
						)


		f=open(os.path.join( out,'vcf_convert.sh' ), 'w')
		probes=pd.HDFStore(os.path.join(out,'probes', self.study_name +'.h5'),'r')
		N_probes=probes.get_storer('probes').nrows
		print 'There are {} probes'.format(N_probes)
		chunk=np.vstack(((np.arange(0,N_probes,self.split_size)+1)[:-1],np.arange(0,N_probes,self.split_size)[1:]))
		N_jobs=chunk.shape[1]
		ch=[0,0]
		i_ch=-1
		for i_ch in range(chunk.shape[1]):
			ch=chunk[:,i_ch]
			#print ch
			l='bash {} {} {} {} {} {} {} {} \n'.format(
				os.path.join(os.environ['HASEDIR'],'tools','VCFGenotype2hdf5.sh'),
				self.reader.folder.path,
				out,
				os.environ['HASEDIR'],
				self.study_name,
				ch[0],
				ch[1],
				i_ch
					)
			f.write(l)
		if ch[1]!=N_probes:
			l='bash {} {} {} {} {} {} {} {} \n'.format(
				os.path.join(os.environ['HASEDIR'],'tools','VCFGenotype2hdf5.sh'),
				self.reader.folder.path,
				out,
				os.environ['HASEDIR'],
				self.study_name,
				ch[1]+1,
				N_probes,
				i_ch+1
					)
			f.write(l)
			N_jobs+=1
		f.close()
		if self.cluster:
			print 'Submit to cluster!'
			cmd="qsub -sync y -t 1-{} {} {}".format(N_jobs,os.path.join(os.environ['HASEDIR'],'tools','qsub_helper.sh'),os.path.join( out,'vcf_convert.sh' ))
			print cmd
			proc=subprocess.Popen(cmd, shell=True,stderr=subprocess.STDOUT,stdout=subprocess.PIPE).communicate()
		else:
			proc=subprocess.Popen(['bash',os.path.join( out,'vcf_convert.sh' ) ], shell=False,stderr=FNULL)
			print proc.communicate()

		shutil.move(os.path.join(out,'vcf_convert.sh'), os.path.join(out,'tmp_files','vcf_convert.sh') )
		shutil.move(os.path.join(out,'SUB_ID.txt'), os.path.join(out,'tmp_files','SUB_ID.txt') )
		shutil.move(os.path.join(out, 'snps_count.txt'), os.path.join(out, 'tmp_files', 'snps_count.txt'))

		shutil.move(os.path.join(out, 'files_order.txt'), os.path.join(out, 'tmp_files', 'files_order.txt'))
		shutil.move(os.path.join(out, 'info.txt'), os.path.join(out, 'tmp_files', 'info.txt'))
hdgwas/tools.py
class Mapper(object):

	def __init__(self):

		self.name=None
		self.genotype_names=[]
		self.dic=OrderedDict()
		self.n_study=0
		self.values=[] # array (N_study, N_keys), where N_keys = number of ID in reference table
		self.keys=None
		self.n_keys=None
		self.processed=0
		self._limit=None
		self.reference=False
		self.include=None
		self.exclude=None
		self.include_ind=np.array([])
		self.exclude_ind=np.array([])
		#self.hash=HashTable()
		self.cluster=None
		self.node=None
		self.chunk_pool=None
		self.chunk_size=None
		self.column_names=[]
		self.flip={} # keys - study names; values - array, with length equal to number of probes
		self.probes=None
		self.encoded={}


	def chunk_pop(self):
		if self.chunk_pool is None and self.cluster is None:
			raise ValueError('Cluster settings are not defined!')

		if self.chunk_pool is None:
			self.chunk_pool=[]
			if self.node is None:
				raise ValueError('cluster setting not defined! (number of nodes / this node number) ')
			self.node=[int(i) for i in self.node]
			N=self.n_keys/self.node[0]
			n=N/self.chunk_size
			if N<1:
				raise ValueError('Too many nodes! Change chunk size in mapper')
			elif N==1:
				self.chunk_pool=[[self.node[1]-1,self.node[1]]]
			else:
				if n==0:
					self.chunk_pool=[[N*(self.node[1]-1),N*(self.node[1]-1)+N]]
				else:
					self.chunk_pool=[]
					for i in range(n):
						self.chunk_pool.append(   [N*(self.node[1]-1)+ self.chunk_size*i, N*(self.node[1]-1) + self.chunk_size*(i+1)   ] )
					if (N-n*self.chunk_size)!=0:
						self.chunk_pool.append( [N*(self.node[1]-1) + self.chunk_size*(n), N*(self.node[1]-1) + self.chunk_size*(n)+ (N-n*self.chunk_size)  ]  )


			if self.node[1]==self.node[0]:
				if (self.n_keys-N*self.node[0])!=0:
					self.chunk_pool.append([self.chunk_pool[-1][1],self.n_keys])
			self.chunk_pool=self.chunk_pool[::-1]

			print self.chunk_pool

		if len(self.chunk_pool)!=0:
			ch=self.chunk_pool.pop()
			print ch
			return ch
		else:
			return None

	#@timing
	def fill(self, keys, name ,repeats=False, reference=False):#TODO (middle) remove
		self.reference=reference
		self.reference_name=name
		self.column_names.append(name)
		l=len(keys)
		uniq=np.unique(keys)
		u=len(uniq)
		if not repeats:
			if l!=u:
				raise ValueError('length of keys {} does not =  {} number of uniq values!!!'.format(l,u))
		else:
			l=u
			keys=uniq
		self.n_keys=l
		if isinstance(self._limit, type(None)):
			for i,j in enumerate(keys):
				self.dic[j]=[i]
		else:
			for i,j in enumerate(keys):
				if i==self._limit:
					break
				self.dic[j]=[i]

	#@timing
	def push(self, new_keys,name=None, new_id=True):#TODO (middle) remove
		if not self.reference and len(self.dic)==0:
			raise ValueError('You should fill mapper first with ref panel or your own rsids!')
		self.n_study+=1
		if name is not None:
			try:
				self.genotype_names.index(name)
				raise ValueError('Trying to push the same study to mapper')
			except:
				self.genotype_names.append(name)
				self.column_names.append(name)

		if isinstance(self._limit, type(None)):
			for i,j in enumerate(new_keys):
				if self.dic.get(j):
					self.dic[j]=self.dic[j] + [i]
				else:
					if new_id:
						print ('WARNING! You are pushing ID {} which is not present in reference panel!'.format(j))
						self.dic[j]=[-1]*(self.n_study+1)
						self.dic[j][self.n_study]=i
					else:
						continue
		else:
			for i,j in enumerate(new_keys):
				if i==self._limit:
					break
				if self.dic.get(j):
					self.dic[j]=self.dic[j] + [i]
				else:
					self.dic[j]=[-1]*(self.n_study+1)
					self.dic[j][self.n_study]=i

		for k in self.dic:
			if len(self.dic[k])<self.n_study+1:
				self.dic[k]=self.dic[k] + [-1]

	#@timing
	def load_flip(self,folder,encode=None):

		if folder is None:
			raise ValueError('Mapper is not defined!')
		if not os.path.isdir(folder):
			raise ValueError('{} is not a folder'.format(folder))

		if self.reference_name is None:
			raise ValueError('Reference name for mapper is not defined!')

		if len(glob.glob(folder + 'flip_*'))==0:
			raise ValueError('There is no flip mapper data in folder {}'.format(folder))

		for j,i in enumerate(self.genotype_names):
			if encode is not None:
				self.encoded[i]=encode[j]
				if encode[j]==0:
					self.flip[i]=np.load(os.path.join(folder, 'flip_'+self.reference_name+'_'+i+'.npy'))
				elif encode[j]==1:
					flip=np.load(os.path.join(folder, 'flip_'+self.reference_name+'_'+i+'.npy'))
					self.flip[i] = np.ones(flip.shape[0])
			else:
				self.flip[i] = np.load(os.path.join(folder, 'flip_' + self.reference_name + '_' + i + '.npy'))

	#@timing
	def load (self, folder):
		if folder is None:
			raise ValueError('Mapper is not defined!')
		if not os.path.isdir(folder):
			raise ValueError('{} is not a folder'.format(folder))

		if self.reference_name is None:
			raise ValueError('Reference name for mapper is not defined!')

		if len(glob.glob(folder + 'keys_*'))==0:
			raise ValueError('There is no mapper data in folder {}'.format(folder))

		keys=glob.glob(folder+ 'keys_*')
		if len(keys)>1:
			raise ValueError('There are more than one reference keys in folder {}'.format(folder))

		self.keys=np.load(os.path.join(keys[0]) )#TODO (middle) not safety to load only one file
		self.n_keys=self.keys.shape[0]

		values=glob.glob(folder+ 'values_*')
		if len(values)==0:
			raise ValueError('There is no mapper data in folder {}'.format(folder))

		if self.genotype_names is None:
			raise ValueError('Genotype names are not defined!')

		self.values=np.zeros(( self.n_keys , len(self.genotype_names)  ))
		for j,i in enumerate(self.genotype_names):
			self.values[:,j]=np.load(os.path.join(folder, 'values_'+self.reference_name+'_'+i+'.npy'))

		if self.n_keys!=self.values.shape[0]:
			raise ValueError('Number of indexes {} in mapper values is different from keys length {}!'.format(self.values.shape[0],self.n_keys ))

		self.n_study=self.values.shape[1]
		if self.n_keys<self.n_study:
			print ('WARNING!!! Normally should be more test values that studies! Check your saved data for Mapper')

		print ('You loaded values for {} studies and {} test values'.format(self.n_study, self.n_keys))

		if self.include is not None or self.exclude is not None:
			self.reference=Reference()
			self.reference.name=self.reference_name
			self.reference.load_index()

	#@timing
	def get(self,chunk_number=None):

		if isinstance(self.keys, type(None)) and isinstance(self.values, type(None)):
			raise ValueError('mapper data is not loaded!')

		if isinstance(self.chunk_size,type(None)):
			raise ValueError('chunk_size and should be defined')

		if self.processed==self.n_keys:
			return None, None

		if chunk_number is not None:
			start=chunk_number[0]
			finish=chunk_number[1]
		else:
			start=self.processed
			finish=self.processed+self.chunk_size if (self.processed+self.chunk_size)<self.n_keys else self.n_keys
			self.processed=finish

		with Timer() as t_q:
			if self.include is not None:
				if len(self.include_ind)==0:
					if 'ID' in self.include.columns:
						self.include_ind = np.in1d(self.keys, self.include.ID)
						self.include_ind = np.where(self.include_ind == True)[0]
						#self.query_include=self.reference.index.select('reference',where='ID=self.include.ID')
						#self.include_ind=self.query_include.index
					elif 'CHR' in self.include.columns and 'bp' in self.include.columns:
						self.query_include=self.reference.index.select('reference',where='CHR=self.include.CHR & bp=self.include.bp')
						self.include_ind=self.query_include.index

			if self.exclude is not None:
				if len(self.exclude_ind)==0:
						if 'ID' in self.exclude.columns:
							self.exclude_ind = np.in1d(self.keys, self.exclude.ID)
							self.exclude_ind = np.where(self.exclude_ind == True)[0]
							#self.query_exclude=self.reference.index.select('reference',where='ID=self.exclude.ID')
							#self.exclude_ind=self.query_exclude.index
						elif 'CHR' in self.exclude.columns and 'bp' in self.exclude.columns:
							self.query_exclude=self.reference.index.select('reference',where='CHR=self.exclude.CHR & bp=self.exclude.bp')
							self.exclude_ind=self.query_exclude.index

		print "Time to select SNPs {}s".format(t_q.secs)


		if self.exclude is not None or self.include is not None:
			self.include_ind=np.setxor1d(self.include_ind,self.exclude_ind)
			if len(self.include_ind)==0:
				print ('None of included ID found in genotype data!')
				return None, None
			else:
				ind = np.intersect1d(np.arange(start, finish), self.include_ind)
		else:
			ind = np.arange(start, finish)
			self.include_ind=np.arange(len(self.values))

		ind = ind.astype('int')
		indexes = self.values[ind, :]
		r = (indexes == -1).any(axis=1)
		ind = ind[~r]
		indexes = indexes[~r]
		keys = self.keys[ind]

		while len(ind)<self.chunk_size:
			if self.processed == self.n_keys:
				break
			if chunk_number is None:
				start = self.processed
				finish = self.processed + self.chunk_size if (self.processed + self.chunk_size) < self.n_keys else self.n_keys
				self.processed = finish
			else:
				ch = self.chunk_pop()
				if ch is not None:
					start = ch[0]
					finish = ch[1]
				else:
					break
			if len(np.append( ind, np.intersect1d(np.arange(start, finish), self.include_ind)  ))>1.5*self.chunk_size:

				if chunk_number is None:
					self.processed = self.processed - len(np.arange(start,finish))
					break
				else:
					print 'Added back chunk {}'.format(ch)
					self.chunk_pool.append(ch)
					break
			else:
				ind=np.append( ind, np.intersect1d(np.arange(start, finish), self.include_ind)  )
				ind=ind.astype('int')
				indexes=self.values[ind,:]
				r=(indexes==-1).any(axis=1)
				ind=ind[~r]
				indexes = indexes[~r]
				keys = self.keys[ind]

		if len(ind)==0:
			return None,None


		return [indexes[:,i].astype(np.int64) for i in range(self.n_study)], keys



	def get_all(self, name, nonempty=True):
		ind=self.genotype_names.index(name)
		indexes=self.values[:,ind]
		if nonempty:
			r=(self.values==-1).any(axis=1)
			if len(r)==self.values.shape[0]:
				raise ValueError('There is no common names between studies')
			indexes=indexes[~r]
		return indexes
hdgwas/hdregression.py
def A_inverse(a_covariates, a_test): #TODO (low) extend for any number of tests in model

	A_inv=[]
	n,m=a_covariates.shape
	k=n+1
	for i in xrange(a_test.shape[0]): #TODO (low) not in for loop
		inv=np.zeros(k*k).reshape(k,k)
		inv[ 0:k-1,0:k-1 ]=a_covariates
		inv[k-1,:]=a_test[i,:]
		inv[0:k,k-1]=a_test[i,0:k]
		try:
			A_inv.append(np.linalg.inv(inv))
		except:
			A_inv.append(np.zeros(k*k).reshape(k,k)) #TODO (high) test; check influence on results; warning;

	return np.array(A_inv)
hdgwas/data.py
class Reader(object):

	def __init__(self, name):
		self.name=name
		self.path=None
		self._data=None
		self.ext=['.npy', '.csv', '.txt','.h5', 'PLINK','.gz','VCF']
		self.pool=None
		self.folder=None
		self.processed=0
		self.kwargs=None
		self.permutation=False


	def start(self, path, **kwargs):

		self.kwargs=kwargs
		if path is None:
			raise ValueError('Not defined path for {}'.format(self.name))
		if os.path.isdir(path):

			self.folder=Folder(path)
			self.path=path

			self.format=self.folder.format

			if kwargs.get('vcf',0) and kwargs['vcf']:
				self.format='VCF'

			if self.name=='genotype' and self.format!='PLINK' and self.format!='.gz' and self.format!='VCF':
				self.format='.h5'

			if self.format not in self.ext:
				raise ValueError(' {} is not supported format (only {} supported)!'.format(self.format, self.ext))

			elif self.format=='.npy' and self.name!='partial':
				self.folder=NPFolder(path,protected=['info_dic.npy'] )
			#self.folder.protected=['info_dic.npy']

			elif self.format=='.npy' and self.name=='partial':
				self.folder=PDFolder(path,name=kwargs['study_name'])


			elif self.format in ['.csv', '.txt']:
				self.folder=CSVFolder(path)

			elif self.format=='.h5' and self.name=='genotype':
					self.folder=PLINKHDF5Folder(path,kwargs['study_name'])

			elif self.format=='PLINK':
				self.folder=PLINKFolder(path)

			elif self.format=='.gz':
				self.folder=MINIMACFolder(path)
				self.format="MINIMAC"

			elif self.format=='VCF':
				self.folder = VCFFolder(path)

			elif self.format=='.h5' and self.name!='genotype':
				raise ValueError('hdf5 format implemented only for converted from genotype data...Sorry')

			elif self.format!='.h5' and self.name=='genotype':
				raise ValueError('genotype data should be in hdf5 format')

			self._get=self.folder.get_next

		else:
			raise ValueError('{} is not a directory'.format(path))


	def get(self,*args,**kwargs):
		if self.format!='.h5' and self.name!='partial':
			raise ValueError()
		return self.folder.get(*args,**kwargs)


	def get_next(self,**kwargs):
		return self._get(**kwargs)
hdgwas/hdregression.py
def B4(phenotype,genotype):
	b4=np.tensordot(genotype, phenotype, axes=([1], [0]))
	return b4
hdgwas/data.py
class MetaParData(object):

	def __init__(self,pd,study_names,protocol=None):
		#@timing
		def _check(map,keys):
			values=np.array(map.dic.values() )
			result={}
			r=(values==-1).any(axis=1)
			if np.sum(r)==values.shape[0]:
				raise ValueError('There is no common names between studies')
			for i,k in enumerate(keys):
				result[k]=values[~r,i]
			return result

		self.name=None
		self.study_names=study_names
		self.phen_mapper=Mapper()
		self.cov_mapper=Mapper()
		self.covariates = OrderedDict()
		self.pd =OrderedDict()
		for i in pd:
			self.pd[i.folder.name]=i
		keys=[]
		if protocol is None:
			for i,k in enumerate(pd):
				self.covariates[k]=[n.split(self.study_names[i]+'_')[1] for n in k.folder._data.metadata['names']]
				if i==0:
					self.phen_mapper.fill(k.folder._data.metadata['phenotype'],k.folder.name, reference=False)
					self.cov_mapper.fill(self.covariates[k],k.folder.name, reference=False)
				else:
					self.phen_mapper.push(k.folder._data.metadata['phenotype'],name=k.folder.name,new_id=False)
					self.cov_mapper.push(self.covariates[k],name=k.folder.name, new_id=False)
				keys.append(k.folder.name)

			self.phen_order =_check(self.phen_mapper,keys)
			self.cov_order =_check(self.cov_mapper, keys)

		if protocol is not None:
			if not protocol.enable:
				protocol.parse()

	def check_pd(self, old, new):

		np.set_printoptions(precision=3, suppress=True)
		print ("*******PD CHECK*********")
		print ('A covariates...')
		print old[3] / old[3][0, 0]
		print new[3] / new[3][0, 0]
		print ('FREQ TESTs')
		N=np.random.randint(0, np.min( (old[0].shape[0],new[0].shape[0]) )  ,10)
		print np.array(old[0][:, 0] / old[3][0, 0]/ 2)[N]
		print np.array(new[0][:, 0] / new[3][0, 0] / 2)[N]
		print ('FREQ PHENO')
		M=np.random.randint(0, np.min( (old[1].shape[1],new[1].shape[1]) )  ,10)
		print np.array(old[1][0, :] / old[3][0, 0])[M]
		print np.array(new[1][0, :] / new[3][0, 0])[M]
		print ("****************")

	#@timing
	def check_maf(self, SNPs_index):
		maf=np.zeros( ( SNPs_index[0].shape[0], len(self.pd)) )

		for i,j in enumerate(self.pd):
			maf[:,i]=np.array(self.pd[j].folder._data.metadata['MAF'])[ SNPs_index[i].astype(np.int64) ]

		if (np.std(maf,axis=1)>0.1).any():
			raise ValueError('MAF is not consistent between PD data!')


	def get(self, SNPs_index=None, B4=False, regression_model=None,random_effect_intercept=False ):

		if self.pd is None:
			raise ValueError('Data not defined!')
		k=self.pd.keys()
		if SNPs_index is not None:
			if len(self.pd)!=len(SNPs_index):
				raise ValueError('There are not equal number od PD and SNPs indexes {}!={}'.format(len(self.pd), len(SNPs_index)))
			a_test, b_cov, C, a_cov = self.pd[k[0]].get(gen_order=SNPs_index[0], phen_order=self.phen_order[k[0]], cov_order=self.cov_order[k[0]] )
			if random_effect_intercept and len(self.pd)>1: #TODO (high)
				a_test_effect=a_test[:,0:1]
				b_cov_effect=b_cov[0:1,:]
				a_cov_effect=a_cov[0:1,:]
		else:
			raise ValueError('Indexes are not defined!')

		if B4:
			b4=self.pd[k[0]].folder._data.b4[SNPs_index[0],:]
			b4=b4[:,self.phen_order[k[0]]]

		#self.check_maf(SNPs_index)

		for i in range(1, len(self.pd)):
			a,b,c,a_c=self.pd[k[i]].get(gen_order=SNPs_index[i], phen_order=self.phen_order[k[i]], cov_order=self.cov_order[k[i]])
			self.check_pd([a_test, b_cov, C, a_cov], [a,b,c,a_c])
			a_test=a_test + a
			b_cov=b_cov + b
			C=C+c
			a_cov=a_cov + a_c
			if random_effect_intercept and i<(len(self.pd)-1) :
				a_test_effect=np.hstack((a_test_effect,a[:,0:1]))
				b_cov_effect=np.vstack((b_cov_effect,b[0:1,:]))
				a_cov_effect=np.vstack((a_cov_effect,a_c[0:1,:]))
			if B4:
				b4_tmp=self.pd[k[i]].folder._data.b4[SNPs_index[i], :	]
				b4=b4+b4_tmp[:,  self.phen_order[k[i]]]

		if random_effect_intercept and len(self.pd)>1:
				a_cov_I=np.zeros((len(self.pd)-1,len(self.pd)-1))
				np.fill_diagonal(a_cov_I,a_cov_effect[:,0])
				a_effect=np.hstack((a_cov_I,a_cov_effect))
				a_test=np.hstack((a_test_effect,a_test))
				b_cov=np.vstack((b_cov_effect,b_cov))
				a_cov=np.vstack((a_cov_effect,a_cov))
				a_cov=np.hstack((a_effect.T,a_cov))

		if B4:
			return a_test, b_cov, C, a_cov, b4
		else:
			return a_test, b_cov, C, a_cov


	def maf_pard(self,SNPs_index=None):

		samples=0
		maf=np.zeros( len(SNPs_index[0]) )

		for j,i in enumerate(self.pd):
			n=len(self.pd[i].folder._data.metadata['id'])
			samples+=n
			maf=maf+n*np.array(self.pd[i].folder._data.metadata['MAF'])[SNPs_index[j]]
		maf=maf/np.float(samples)
		return maf

	def get_n_id(self):
		return np.sum([len(self.pd[i].folder._data.metadata['id']) for i in self.pd  ])

	def get_phenotype_names(self):
		n={}
		for i in self.pd:
			n[i]=self.pd[i].folder._data.metadata['phenotype']
hdgwas/tools.py
class HaseAnalyser(Analyser):

	def __init__(self):
		self.name=None
		self.t_stat=None
		self.p_value=None
		self.betas=None
		self.SE=None
		self.DF=None
		self.n_studies=None
		self.probes_path={}
		self.results=OrderedDict()
		self.split_size=None
		self.threshold=None
		self.result_path=None
		self.result_index=1
		self.out=None
		self.MAF=None
		self.rsid=None
		self.reference=None
		self.cluster=False
		self.permutation=False
		self.rsid_dic={}
		self.result_folder=None
		self.result_dump_size=3
		self.file_number=None


	def summary(self):

		if isinstance(self.result_path,type(None)):
			raise ValueError('Please set result pathway to Analyser!')

		if isinstance(self.DF,type(None)):
			print ('DF is not defined. Forced to use z_score statistics!')

		if self.result_folder is None:
			self.result_folder=[ i for i in glob.glob( os.path.join(self.result_path, '*.npy') ) if 'RSID' not in i]

		self.results['RSID']=np.array([])
		self.results['p_value']=np.array([])
		self.results['t-stat']=np.array([])
		self.results['phenotype']=np.array([])
		self.results['SE']=np.array([])
		self.results['MAF']=np.array([])
		self.results['BETA'] = np.array([])

		files=[]

		if self.file_number is not None:
			try:
				d = np.load(os.path.join(self.result_path, '{}result.npy'.format(self.file_number))).item()
			except:
				print "Can't read {}".format(i)

			p_value = self.get_p_value(d['t-stat'], df=self.DF)
			self.results['t-stat'] = np.append(self.results['t-stat'], d['t-stat'].flatten())
			self.results['SE'] = np.append(self.results['SE'], d['SE'].flatten())
			self.results['RSID'] = np.append(self.results['RSID'], d['index'])
			self.results['phenotype'] = np.append(self.results['phenotype'], d['phenotype'])
			self.results['MAF'] = np.append(self.results['MAF'], d['MAF'])
			self.results['p_value'] = np.append(self.results['p_value'], p_value.flatten())
			self.results['BETA'] = np.append(self.results['BETA'], d['t-stat'].flatten() * d['SE'].flatten())

		else:

			for i in range(self.result_dump_size):
				try:
					files.append(self.result_folder.pop())
				except:
					break
			if len(files)!=0:
				for i in files:
					print i
					try:
						d=np.load(i).item()
					except:
						print "Can't read {}".format(i)
						continue
					p_value=self.get_p_value(d['t-stat'],df=self.DF)
					self.results['t-stat']=np.append(self.results['t-stat'],d['t-stat'].flatten())
					self.results['SE']=np.append(self.results['SE'],d['SE'].flatten())
					self.results['RSID']=np.append(self.results['RSID'],d['index'])
					self.results['phenotype']=np.append(self.results['phenotype'],d['phenotype'])
					self.results['MAF']=np.append(self.results['MAF'],d['MAF'])
					self.results['p_value']=np.append(self.results['p_value'],p_value.flatten())
					self.results['BETA'] = np.append(self.results['BETA'],  d['t-stat'].flatten() *  d['SE'].flatten() )
			else:
				self.results=None

	def get_p_value(self,t_stat,df=None):
		if df is None:
			return stats.norm.sf(np.abs(t_stat))*2
		else:
			return stats.t.sf(np.abs(t_stat),df)*2

	def save_result(self , phen_names):

		t_threshold=self.threshold
		save_path=self.out

		mask=([],[],[])

		if t_threshold:
			mask = np.where(np.abs(self.t_stat) > t_threshold)
		else:
			mask = np.where(np.abs(self.t_stat) > 0)

		if (len(mask[0]) != 0):
			print ('Saving results to {}'.format(save_path))
			t_save = self.t_stat[mask[0],mask[1],mask[2]]
			se=self.SE[mask[0],mask[1],mask[2]]
			result = {'phenotype': phen_names[mask[2]], 't-stat': t_save,'index':self.rsid[mask[0]],'SE':se, 'MAF':self.MAF[mask[0]]}

			if not self.cluster:
				np.save(os.path.join(save_path, str(self.result_index) + 'result.npy'), result)
			else:
				if self.permutation:
					result=self.t_stat
				self.rsid_dic[str(self.chunk[0])+'_' +str(self.chunk[1])]=self.rsid
				np.save(os.path.join(save_path,'node'+str(self.node)+'_'+ str(self.chunk[0])+'_' +str(self.chunk[1])+'_'+ str(self.result_index) + 'result.npy'),result)
		self.result_index+=1
hdgwas/regression.py
def haseregression(phen,gen,cov, mapper, Analyser, maf,intercept=True, interaction=None):

	g=tuple( [i.folder._data for i in gen ] )

	row_index, ids =  study_indexes(phenotype=phen.folder._data,
											   genotype=g,
											   covariates=cov.folder._data)

	if mapper is not None:
		SNP=[0,0,mapper.n_keys]
	else:
		SNP=[0,0,'unknown']

	covariates=cov.get_next(index=row_index[2])
	a_cov=A_covariates(covariates,intercept=intercept)

	while True:
		gc.collect()
		if mapper is not None:
			if mapper.cluster=='n':
				SNPs_index, keys=mapper.get()
			else:
				ch=mapper.chunk_pop()
				if ch is None:
					SNPs_index=None
					break
				SNPs_index, keys=mapper.get(chunk_number=ch)
			if isinstance(SNPs_index, type(None)):
				break
			Analyser.rsid=keys
		else:
			SNPs_index=None

		with Timer() as t:
			genotype=merge_genotype(gen, SNPs_index, mapper)
		print ('time to read and merge genotype {}s'.format(t.secs))
		gc.collect()
		if genotype is None:
			print 'All genotype processed!'
			break
		SNP[0]+=genotype.shape[0]
		genotype=genotype[:,row_index[0]]

		if mapper is None:
			Analyser.rsid=np.array(range(genotype.shape[0]))


		MAF=np.mean(genotype, axis=1)/2
		STD=np.std(genotype, axis=1)

		if maf!=0:

			filter=(MAF>maf) & (MAF<1-maf) & (MAF!=0.5)
			genotype=genotype[filter,:]
			Analyser.MAF=MAF[filter]
			Analyser.rsid=Analyser.rsid[filter]

			if genotype.shape[0]==0:
				print 'NO SNPs > MAF'
				continue

		else:
			Analyser.MAF=MAF

		SNP[1]+=genotype.shape[0]

		while True:
			phenotype=phen.get_next(index=row_index[1])

			if isinstance(phenotype, type(None)):
				phen.folder.processed=0
				print 'All phenotypes processed!'
				break

			if phen.permutation:
				np.random.shuffle(phenotype)

			b_cov=B_covariates(covariates,phenotype,intercept=intercept)

			C=C_matrix(phenotype)

			if interaction is not None:
				pass


			a_test=A_tests(covariates,genotype,intercept=intercept)
			a_inv=A_inverse(a_cov,a_test)

			N_con=a_inv.shape[1] - 1

			DF=(phenotype.shape[0] - a_inv.shape[1])

			b4=B4(phenotype,genotype)


			t_stat, SE=HASE(b4, a_inv, b_cov, C, N_con, DF)
			print('Read {}, processed {}, total {}'.format(SNP[0],SNP[1],SNP[2] ))
			Analyser.t_stat=t_stat
			Analyser.SE=SE
			if mapper is not None and mapper.cluster == 'y':
				Analyser.cluster=True
				Analyser.chunk=ch
				Analyser.node=mapper.node[1]
			if phen.permutation:
				Analyser.permutation=True
			Analyser.save_result( phen.folder._data.names[phen.folder._data.start:phen.folder._data.finish])
			t_stat=None
			Analyser.t_stat=None
			del b4
			del C
			del b_cov
			del a_inv
			del a_test
			del t_stat
			gc.collect()

	if Analyser.cluster:
		np.save(os.path.join(Analyser.out,str(Analyser.node)+'_node_RSID.npy'),Analyser.rsid_dic)
hdgwas/protocol.py
class Protocol: #TODO (mid) write class

	def __init__(self, path):
		self.name=None
		self.model=None
		self.covariates=None
		self.MAF=None
		self.QC=None
		self.genotype_format=None
		self.phenotype_format=None
		self.type=None #meta,single etc
		self.enable=False
		self.path=path
		if os.path.isfile(self.path):
			self.enable=True
			self.parse()

	def parse(self):
		print('Not implemented!')

	def regression_model(self):
		pass
hdgwas/tools.py
def check_converter(converted_folder, study_name):
	l_ind = os.listdir(  os.path.join(converted_folder, 'individuals')  )
	l_probes = os.listdir(os.path.join(converted_folder, 'probes'))
	l_genotype = os.listdir(os.path.join(converted_folder, 'genotype'))

	if os.path.isdir(os.path.join(converted_folder, 'tmp_files')):
		l_tmp=os.listdir(os.path.join(converted_folder, 'tmp_files'))
		if len(l_tmp)==0:
			print ('There is no tmp files!')
		else:
			try:
				df_tmp=pd.read_csv( os.path.join(converted_folder, 'tmp_files', 'info.txt') , header=None)
			except:
				df_tmp=None

	else:
		df_tmp=None

	if len(l_ind)!=1 or '.h5' not in l_ind[0]:
		print ('There should be 1 hdf5 file in individuals folder!')

	else:
		df_ind = pd.read_hdf(os.path.join(converted_folder, 'individuals',study_name + '.h5'),'individuals')
		if df_tmp is not None and int(df_tmp[0][0])!=df_ind.shape[0]:
			print ('Converted number of IDs {} != {} original number'.format( df_ind.shape[0] , df_tmp[0][0] ))
		else:
			print ('Number of individuals {} '.format(df_ind.shape[0]))
			print df_ind.head()

	if len(l_probes)!=2:
		print ('There should be 2 files in probes folder, you have {}!'.format(len(l_probes)))
		probes = pd.HDFStore(os.path.join(converted_folder, 'probes', study_name + '.h5'), 'r')
		N_probs = probes.get_storer('probes').nrows

	else:
		probes = pd.HDFStore(os.path.join(converted_folder, 'probes',study_name + '.h5'), 'r')
		N_probs=probes.get_storer('probes').nrows

	if df_tmp is not None and int(df_tmp[0][1])!=N_probs:
		print ("Converted number of variants {} diff from  original number {}".format(N_probs,df_tmp[0][1] ))

	else:
		print ('Converted number of variants {}'.format(N_probs))

		print probes.select('probes', start=0, stop=10)
		probes.close()

	if len(l_genotype)==0:
		print('There is no genotype data converted!')
	else:
		index = 0
		for i in l_genotype:
			n, m = h5py.File(os.path.join(converted_folder, 'genotype',i), 'r')['genotype'].shape
			index += n

		if index!=N_probs:
			print('Number of variants in genotype folder {} != {} number in probes file'.format(index, N_probs))

		else:
			print ('Number of variants in genotype folder {}'.format(index))

		print h5py.File(os.path.join(converted_folder, 'genotype',l_genotype[0]), 'r')['genotype'][...]
hdgwas/hdregression.py
def HASE(b4, A_inverse, b_cov, C, N_con, DF):

	with Timer() as t:

		B13=b_cov
		B4 = b4

		A1_B_constant = np.tensordot(A_inverse[:, :, 0:(N_con)], B13, axes=([2], [0]))

		A1_B_nonconstant = np.einsum('ijk,il->ijl', A_inverse[:, :, N_con:N_con+1], B4)

		A1_B_full = A1_B_constant + A1_B_nonconstant

		BT_A1B_const = np.einsum('ij,lji->li', B13.T, A1_B_full[:, 0:(N_con), :])

		BT_A1B_nonconst = np.einsum('ijk,ijk->ijk', B4[:, None, :], A1_B_full[:, (N_con):N_con+1, :])

		BT_A1B_full = BT_A1B_const[:, None, :] + BT_A1B_nonconst

		C_BTA1B = BT_A1B_full - C.reshape(1, -1)

		C_BTA1B = np.abs(C_BTA1B)

		a44_C_BTA1B = C_BTA1B * A_inverse[:, (N_con):N_con+1, (N_con):N_con+1]

		a44_C_BTA1B = np.sqrt( (a44_C_BTA1B) )

		t_stat = np.sqrt(DF) * np.divide(A1_B_full[:, (N_con):N_con+1, :], a44_C_BTA1B)

		SE =  a44_C_BTA1B/np.sqrt(DF)


	print "time to compute GWAS for {} phenotypes and {} SNPs .... {} sec".format(b4.shape[1],
																				  A_inverse.shape[0],
																				  t.secs)
	return t_stat, SE
hdgwas/tools.py
class Checker(object): #TODO (mid) finish or remove this class

	def __init__(self):
		self.mode=None
		self.name=None
		self.args=None

	def check(self,args, mode=None):

		if mode=='converting':
			return self.converting(args)
		elif mode=='encoding':
			self.encoding(args)
		elif mode=='single-meta':
			self.single_meta(args)
		elif mode=='meta-stage':
			self.meta_stage(args)
		elif mode=='regression':
			self.regression(args)


	def system_check(self,args):
		pass

	def converting(self,args):


		if len(args.study_name)>1:
			raise ValueError('Should be only one study name!')

		if len(args.genotype)>1:
			raise ValueError('Should be only one directory for genotype!')

		if isinstance(args.study_name, type(None)):
			raise ValueError('study name is not defined')

		args.genotype=args.genotype[0]
		if not os.path.isdir(args.genotype):
			raise ValueError('{} is not a directory!'.format(args.genotype))

	def encoding(self,args):

		if len(args.study_name)>1:
			raise ValueError('Should be only one study name!')

		if len(args.genotype)>1:
			raise ValueError('Should be only one directory for genotype!')

		if isinstance(args.study_name, type(None)):
			raise ValueError('study name is not defined')

		args.genotype=args.genotype[0]
		args.study_name=args.study_name[0]

		if not os.path.isdir(args.genotype) or not os.path.isdir(args.phenotype) or not os.path.isdir(args.out):
			raise ValueError('{},{} or {} is not a directory'.format(args.genotype, args.phenotype, args.out))

		if  not os.path.isdir(os.path.join(args.out, 'encode_genotype' )) or not os.path.isdir(os.path.join(args.out, 'encode_phenotype' )):
			raise ValueError('{} or {} not exist it is up to you to create these directory'.format(
				(os.path.join(args.out, 'encode_genotype' )),
				(os.path.join(args.out, 'encode_phenotype' ))
			)
			)


		id=np.array(pd.read_hdf(os.path.join(args.genotype,'individuals',args.study_name+'.h5'),'individuals').individual.tolist(), dtype=np.int)



	def single_meta(self,args):


		if len(args.study_name)>1:
			raise ValueError('Should be only one study name!')

		if len(args.genotype)>1:
			raise ValueError('Should be only one directory for genotype!')

		if isinstance(args.study_name, type(None)):
			raise ValueError('study name is not defined')

		args.genotype=args.genotype[0]
		args.study_name=args.study_name[0]

		if not os.path.isdir(args.genotype) or not os.path.isdir(args.phenotype) or not os.path.isdir(args.covariates) :
			raise ValueError('{},{} or {} is nor a directory'.format(os.path.isdir(args.genotype),os.path.isdir(args.phenotype), os.path.isdir(args.covariates )) )



	def meta_stage(self,args):

		if isinstance(args.mapper, type(None)):
			raise ValueError('You should define mapper folder --mapper, if it does not exist you should first run Mapper script')

		if isinstance(args.mapper_name, type(None)):
			raise ValueError('You should define mapper name --mapper_name, if it does not exist you should first run Mapper script')

		if len(args.genotype)<=1:
			raise ValueError('Should be only one directory for genotype!')

		g=[os.path.isdir(i) for i in args.genotype ]
		d=[os.path.isdir(i) for i in args.derivatives ]

		if np.sum(g)!=len(args.genotype) or not os.path.isdir(args.phenotype) or  np.sum(d)!=len(args.derivatives) :
			raise ValueError('{},{} or {} is nor a directory'.format(os.path.isdir(args.genotype),os.path.isdir(args.phenotype), os.path.isdir(args.derivatives )) )


	def regression(self,args):

		g=[os.path.isdir(i) for i in args.genotype ]

		if np.sum(g)!=len(args.genotype) or not os.path.isdir(args.phenotype[0]) or not os.path.isdir(args.covariates) :
			raise ValueError('{},{} or {} is not a directory'.format(os.path.isdir(args.genotype),os.path.isdir(args.phenotype), os.path.isdir(args.derivatives )) )
hdgwas/tools.py
class Reference(object):

	def __init__(self):
		self.name=None
		self.path_default=os.path.join(os.environ['HASEDIR'], 'data')
		self.path={
			'1000Gp1v3_ref':{'table':os.path.join(os.environ['HASEDIR'], 'data', '1000Gp1v3.ref.gz'),
							 'index':os.path.join(os.environ['HASEDIR'], 'data', '1000Gp1v3.ref_info.h5') }
				  }
		self.dataframe=None
		self.loaded=False
		self.chunk=10000
		self.read=0
		self.columns=['ID','allele1','allele2','CHR','bp']
		self.index=None



	def load(self):
		if self.name is not None:
			if self.path.get(self.name) is not None:
				try:
					self.dataframe=pd.read_csv(self.path[self.name]['table'], compression='gzip', sep=' ',chunksize=self.chunk)
				except:
					self.dataframe=pd.read_csv(self.path[self.name]['table'], sep=' ',chunksize=self.chunk)
				self.loaded=True
			else:
				if os.path.isfile(os.path.join(self.path_default,self.name)):
					try:
						self.dataframe=pd.read_csv(os.path.join(self.path_default,self.name), compression='gzip', sep=' ',chunksize=self.chunk)
					except:
						self.dataframe=pd.read_csv(os.path.join(self.path_default,self.name), sep=' ',chunksize=self.chunk)
					self.loaded=True
					self.name=os.path.basename(self.name)
				else:
					raise ValueError('Unknown reference {}!'.format((self.name)))
		else:
			raise ValueError('Reference name is not define!')


	def load_index(self):
		try:
			self.index=pd.HDFStore( os.path.join( self.path[self.name]['index'] ), 'r' )
		except:
			if os.path.isfile(os.path.join(self.path_default,self.name)):
				self.index=pd.HDFStore(os.path.join(self.path_default,self.name), 'r' )
			else:
				raise ValueError('There is {} no index file {}'.format(self.path_default, self.name))


	def next(self):
		df=self.dataframe.get_chunk()
		self.read+=df.shape[0]
		return df
hdgwas/tools.py
class Timer(object):
	def __init__(self, verbose=False):
		self.verbose = verbose

	def __enter__(self):
		self.start = time.time()
		return self

	def __exit__(self, *args):
		self.end = time.time()
		self.secs = self.end - self.start
		self.msecs = self.secs * 1000  # millisecs
		if self.verbose:
			print 'elapsed time: %f ms' % self.msecs
hdgwas/tools.py
def check_np():

	for i in ['atlas_blas_info', 'openblas_info', 'lapack_opt_info',
			  'atlas_info', 'lapack_mkl_info', 'blas_mkl_info',
			  'atlas_blas_info', 'mkl_info']:
		try:
			if len(getattr(np.__config__, i)) >0:
				print('Numpy linked to {}'.format(i))
				return 0
		except:
			continue

	raise ValueError('NO BLAS/LAPACK/MKL found!'
					 'To restart with slow speed use -np False')
hdgwas/data.py
class MetaPhenotype(object):

	def __init__(self, phen, protocol=None, include=None, exclude=None):
		#@timing
		def _check(map,keys,index=None):
			values=np.array(map.dic.values() )
			result={}
			r=(values==-1).any(axis=1)
			if index is not None:
				r[np.where(index==False)[0] ]=True
			if np.sum(r)==values.shape[0]:
				raise ValueError('There is no common names between studies')
			for i,k in enumerate(keys):
				result[k]=values[~r,i]
			return result, np.array(map.dic.keys())[~r]
		self.chunk_size=10000
		self.exclude=None
		self.include=None
		self.name=None
		self.mapper=Mapper()
		self.keys=[]
		self.keep_index=None
		if include is not None:
			self.include = pd.DataFrame.from_csv(include, index_col=None)
			print 'Include:'
			print self.include.head()
			if 'ID' not in self.include.columns:
				raise ValueError('{} table does not have ID column for phenotypes'.format(include))
		if exclude is not None:
			self.exclude = pd.DataFrame.from_csv(exclude, index_col=None)
			print 'Exclude:'
			print self.exclude.head()
			if 'ID' not in self.exclude.columns:
				raise ValueError('{} table does not have ID column for phenotypes'.format(exclude))
		if protocol is None:
			for i,k in enumerate(phen):
				phen_names=[]
				if i==0:
					for j in k.folder.files:
						if j!='info_dic.npy':
							phen_names=phen_names+list(k.folder.data_info[j])
					self.mapper.fill(phen_names,i, reference=False)
				else:
					for j in k.folder.files:
						if j!='info_dic.npy':
							phen_names=phen_names+list(k.folder.data_info[j])
					self.mapper.push(phen_names,name=i, new_id=False)

				self.keys.append(i)
			if self.exclude is not None or self.include is not None:
				phen_names=pd.Series(self.mapper.dic.keys())
				phen_names=phen_names[phen_names.isin(self.include.ID)] if self.include is not None else phen_names
				phen_names = phen_names[~phen_names.isin(self.exclude.ID)] if self.exclude is not None else phen_names
				self.keep_index=pd.Series(self.mapper.dic.keys()).isin(phen_names)
			print np.sum(self.keep_index)
			self.order, self.phen_names =_check(self.mapper,self.keys,index=self.keep_index)
			self.n_phenotypes=len(self.order[self.keys[0]])
			print ('Loaded {} common phenotypes for meta-analysis'.format( self.n_phenotypes  ))
			self.processed=0
		else:
			if not protocol.enable:
				protocol.parse()

		self.pool={i:PhenPool(j.folder) for i,j in enumerate(phen) }


	def get(self):
		if self.processed==self.n_phenotypes:
			return None,None
		else:
			start=self.processed
			finish=self.processed+self.chunk_size if (self.processed+self.chunk_size)<=self.n_phenotypes else self.n_phenotypes
			self.processed=finish
			phenotype = np.zeros( (   np.sum([ len(self.pool[i].folder._data.id) for i in self.pool ] ) , len(range(start,finish))      )  )
		for i,j in enumerate(self.keys):

			if i==0:
				a,b=0,self.pool[j].folder._data.id.shape[0]
				phenotype[a:b,:]=self.pool[j].get_chunk(self.order[j][start:finish])
				a=b
				N = np.random.randint(0, phenotype.shape[1], 10)
				print phenotype.mean(axis=0)[N]
			else:
				ph_tmp=self.pool[j].get_chunk(self.order[j][start:finish])
				print self.pool[j].folder.path
				print ph_tmp.mean(axis=0)[N]
				b+=ph_tmp.shape[0]
				phenotype[a:b,:]=ph_tmp
				a=b


		return phenotype, self.phen_names[start:finish]
hdgwas/hdregression.py
def A_covariates(covariates, intercept=True):

	'''
	:param covariates: (n_subjects, n_covariates) - only constant covariates should be included (age, sex, ICV etc)
	:param intercept: default True, add intercept to model
	:return: matrix (n_cavariates, n_covariates), constant part for the rest of the study
	'''

	S,N=covariates.shape
	if intercept:
		I=np.ones(S).reshape(S,1)
		covariates=np.hstack((I,covariates))
	a_cov=np.dot(covariates.T,covariates)
	return a_cov
hdgwas/fake.py
class Encoder(object):

	def __init__(self, out):
		self.name='Encoder'
		self.N_matrix=None
		self.F=None
		self.F_inv=None
		self.encoder_chunks={}
		self.hdf5_iter=0
		self.npy_iter=0
		self.pytable_filters = tables.Filters(complevel=9, complib='zlib')
		self.out=out
		self.metadata=None
		self.study_name=None
		self.phen_info_dic={}
		self.phen_info_dic['id']=None
		try:
			print ('Creating directories...')
			os.mkdir(os.path.join(self.out,'encode_genotype') )
			os.mkdir(os.path.join(self.out,'encode_phenotype') )
			os.mkdir(os.path.join(self.out,'encode_individuals'))

		except:
			print('Directories "encode_genotype","encode_phenotype","encode_individuals" are already exist in {}...'.format(self.out))


	def matrix(self,N, save=False):

		self.F=np.random.randint(1,10,N*N).reshape(N,N)
		self.F_inv=np.linalg.inv(self.F)
		self.F=np.array(self.F,dtype=np.float)
		if save:
			print ('Saving decoders Matrices to...{}'.format(self.out))
			np.save(os.path.join(self.out,'F.npy'),self.F)
			np.save(os.path.join(self.out,'F_inv.npy'),self.F_inv)



	def encode(self,data, data_type=None):

		if isinstance(self.F,type(None)):
			raise ValueError('Encode matrix is not define')

		x,y=data.shape
		#TODO (mid) add to encode_chunk info about data size and/or names indexes
		#TODO (mid) check the protocol if exist... all study on single stage

		a,b=self.F.shape

		if isinstance(data_type,type(None)):
			raise ValueError('data_type is None')
		elif data_type=="genotype":
			print ('Decoding genotype')
			return np.dot(data,self.F)
		elif data_type=='phenotype':
			print ('Decoding phenotype')
			return np.dot(self.F_inv,data)


	def save_npy(self,data, save_path=None, info=None, index=None):
		#only for phenotype
		if isinstance(save_path,type(None)) or  not os.path.isdir(save_path):
			raise ValueError('There is no such path or directory {}'.format(save_path))
		np.save(os.path.join(save_path,str(self.npy_iter)+'_'+self.study_name+ '.npy'),data )
		if self.phen_info_dic['id'] is None:
			self.phen_info_dic['id']=np.array(info._data.id)[index]
		self.phen_info_dic[str(self.npy_iter)+'_'+self.study_name+ '.npy']=info._data.names[info._data.start:info._data.finish]
		self.npy_iter+=1

	def save_csv(self,data,save_path=None, info=None, index=None):
		if isinstance(save_path,type(None)) or  not os.path.isdir(save_path):
			raise ValueError('There is no such path or directory {}'.format(save_path))
		df=pd.DataFrame(data)
		df.columns=info._data.names[info._data.start:info._data.finish]
		df.insert(0,'id',np.array(info._data.id)[index])
		df.to_csv(os.path.join(save_path,str(self.npy_iter)+'_'+self.study_name+ '.csv'), sep='\t', index=False)
		self.npy_iter+=1



	def save_hdf5(self,data, save_path=None,info=None, index=None):
		#only for genetics data
		print ('Saving data to ... {}'.format(os.path.join(save_path,str(self.hdf5_iter)+'_'+self.study_name+'_encoded.h5')))

		if not os.path.isfile(os.path.join(self.out,'encode_individuals',self.study_name + '.h5'  )):
			chunk = pd.DataFrame.from_dict({"individual": info._data.id[index] })
			chunk.to_hdf(os.path.join(self.out,'encode_individuals',self.study_name + '.h5' ), key='individuals', format='table', min_itemsize=25, complib='zlib', complevel=9)

		if isinstance(save_path,type(None)) or  not os.path.isdir(save_path):
			raise ValueError('There is no such path or directory {}'.format(save_path))

		self.h5_gen_file = tables.open_file(
			os.path.join(save_path,str(self.hdf5_iter)+'_'+self.study_name +'.h5'), 'w', title='encode_genotype')
		self.hdf5_iter+=1

		atom = tables.Float64Atom()
		self.genotype = self.h5_gen_file.create_carray(self.h5_gen_file.root, 'genotype', atom,
													  (data.shape),
													  title='Genotype', filters=self.pytable_filters)
		self.genotype[:] = data
		self.h5_gen_file.close()
		gc.collect()
hdgwas/pard.py
def partial_derivatives(save_path=None,COV=None,PHEN=None, GEN=None,
                        MAP=None, MAF=None, R2=None, B4_flag=False, study_name=None,intercept=True):

    row_index, ids =  study_indexes(phenotype=PHEN.folder._data,genotype=GEN.folder._data,covariates=COV.folder._data)

    metadata={}



    #TODO (mid) add parameter to compute PD only for new phenotypes or cov
    metadata['id']=ids
    metadata['MAF']=[]
    metadata['filter']=[]
    metadata['names']=[] #TODO (low) change to cov_names
    metadata['phenotype']=[]
    b_cov=[]
    C=[]
    a_test=[]
    b4=[]

    covariates=COV.get_next(index=row_index[2])

    if MAP.cluster == 'n' or MAP.node[1] == 1:
        if intercept:
            metadata['names'].append(study_name+ '_intercept')
        metadata['names']=metadata['names']+[ study_name+ '_' + i for i in COV.folder._data.get_names() ]

        a_cov=A_covariates(covariates,intercept=intercept)
        np.save(os.path.join(save_path,study_name+'_a_cov.npy'),a_cov)


        with Timer() as t_phen:

            while True:

                phenotype=PHEN.get_next(index=row_index[1])
                if isinstance(phenotype, type(None)):
                    b_cov=np.concatenate(b_cov, axis=1)
                    C=np.concatenate(C, axis=0)
                    np.save(os.path.join(save_path,study_name+'_b_cov.npy'),b_cov)
                    np.save(os.path.join(save_path,study_name+'_C.npy'),C)
                    break

                metadata['phenotype']=metadata['phenotype']+ list(PHEN.folder._data.get_names())
                b_cov.append(B_covariates(covariates,phenotype,intercept=intercept))
                C.append(C_matrix(phenotype))

        print ('Time to PD phenotype {} is {} s'.format(np.array(C).shape, t_phen.secs))

    if MAP.cluster == 'y':
        f_max=np.max([ int(f.split('_')[0]) for f in GEN.folder.files  ])
        files2read=[ '{}_{}.h5'.format(i,study_name) for i in np.array_split(range(f_max+1),MAP.node[0])[MAP.node[1] -1  ]  ][::-1]
        filesdone=[]
        for i in range(MAP.node[1] -1):
            filesdone=filesdone + [ '{}_{}.h5'.format(i,study_name) for i in np.array_split(range(f_max+1),MAP.node[0])[ i  ]  ]

        N_snps_read=0
        for f in filesdone:
            file = os.path.join(GEN.folder.path, 'genotype', f)
            N_snps_read+=GEN.folder.get_info(file)['shape'][0]
    else:
        N_snps_read=0
    while True:
        with Timer() as t_gen:
            if MAP.cluster == 'y':
                if len(files2read)!=0:
                    file = os.path.join(GEN.folder.path, 'genotype',files2read.pop() )
                    genotype=GEN.folder.read(file)
                else:
                    genotype=None
            else:
                genotype=GEN.get_next()
            if isinstance(genotype, type(None)):
                if MAP.cluster == 'y':

                    np.save(os.path.join(save_path,'node_{}_'.format(MAP.node[1]) + study_name + '_a_test.npy'), np.concatenate(a_test).astype(np.float64))
                    np.save(os.path.join(save_path,'node_{}_'.format(MAP.node[1]) +  study_name + '_metadata.npy'), metadata)
                    if B4_flag:
                        b4 = np.concatenate(b4, axis=0)
                        np.save(os.path.join(save_path, 'node_{}_'.format(MAP.node[1]) + study_name + '_b4.npy'), b4.astype(np.float64))
                    if MAP.node[1]==MAP.node[0]:
                        merge_PD(save_path, MAP.node[0],study_name)

                else:
                    np.save(os.path.join(save_path,study_name+'_a_test.npy'), np.concatenate(a_test) )
                    np.save(os.path.join(save_path,study_name+'_metadata.npy'),metadata)
                    if B4_flag:
                        b4=np.concatenate(b4, axis=0)
                        np.save(os.path.join(save_path, study_name + '_b4.npy'), b4)
                break
            flip = MAP.flip[GEN.folder.name][N_snps_read:N_snps_read + genotype.shape[0]]
            N_snps_read += genotype.shape[0]
            flip_index=(flip==-1)
            genotype=np.apply_along_axis(lambda x: flip*(x-2*flip_index) ,0,genotype)
            genotype=genotype[:,row_index[0]]
            maf=np.mean(genotype, axis=1)/2
            metadata['MAF']=metadata['MAF']+list(maf)

            #TODO (low) add interaction
            a_test.append(A_tests(covariates,genotype,intercept=intercept))

            if B4_flag:
                #works only when all phenotypes in one chunk, if not, do not use this option!
                #it would use to much disk space anyway
                if len([f for f in PHEN.folder.files if f!='info_dic.npy' ])>1:
                    print 'pd_full flag disabled!'
                    B4_flag=False
                    continue
                PHEN.folder.processed=0
                phenotype=PHEN.get_next(index=row_index[1])
                b4.append(B4(phenotype,genotype))

        print ('Time to PD genotype {} is {} s'.format(genotype.shape, t_gen.secs))
hdgwas/converter.py
class GenotypeMINIMAC(object):

	def __init__(self, name, reader=None):
		self.reader=reader
		self.study_name=name
		self.split_size=None
		self.hdf5_iter=0
		self.pytable_filter=tables.Filters(complevel=9, complib='zlib')
		self.cluster=False

	def save_hdf5_chunk(self,data,out,name):
		print 'Saving chunk...{}'.format(os.path.join(out,'genotype',str(self.hdf5_iter)+'_'+name+'.h5'))
		h5_gen_file = tables.open_file(
			os.path.join(out,'genotype',str(self.hdf5_iter)+'_'+name+'.h5'), 'w', title=name)

		atom = tables.Float16Atom()  # TODO (low) check data format
		genotype = h5_gen_file.create_carray(h5_gen_file.root, 'genotype', atom,
											(data.shape),
											title='Genotype',
											filters=self.pytable_filter)
		genotype[:] = data
		h5_gen_file.close()
		genotype=None
		data=None
		gc.collect()
		self.hdf5_iter+=1

	def MACH2hdf5(self, out, id=False,remove_id=False):
		FNULL = open(os.devnull, 'w')
		subprocess.call(['bash',os.path.join(os.environ['HASEDIR'],'tools','minimac2hdf5.sh'),
						 self.reader.folder.path, out , os.environ['HASEDIR'], self.study_name ], shell=False,stderr=subprocess.STDOUT)

		if id:
			if self.cluster:
				ind=pd.read_hdf(os.path.join(out,'individuals',self.study_name+'.h5'),'individuals').individual
				N=ind.shape[0]
				print 'Submit to cluster!'
				cmd="qsub -sync y -t 1-{} {} {}".format(N,os.path.join(os.environ['HASEDIR'],'tools','qsub_helper.sh'),os.path.join( out,'id_convert.sh' ))
				print cmd
				proc=subprocess.Popen(cmd, shell=True,stderr=subprocess.STDOUT,stdout=subprocess.PIPE).communicate()
			else:
				proc=subprocess.Popen(['bash',os.path.join( out,'id_convert.sh' ) ], shell=False,stderr=subprocess.STDOUT)
				print proc.communicate()
			self.folder=MINIMACHDF5Folder(out,self.study_name)
			self.folder.pool.split_size=self.split_size
			self.folder.pool.chunk_size=self.split_size
			print('Start to convert id files to chunk files...')
			while True:
				data=self.folder.get_next()
				if data is None:
					break
				self.save_hdf5_chunk(data,out,self.study_name)
				gc.collect()

			print('Finished')
			if remove_id:
				self.folder.pool.remove(type='all')
			else:
				try:
					os.mkdir(os.path.join(out,'id_genotype') )
				except:
					print 'id_genotype folder already exist'

				self.folder.pool.move(os.path.join(out,'id_genotype'),type='all')

			shutil.move(os.path.join(out,'id_convert.sh'), os.path.join(out,'tmp_files','id_convert.sh') )
			shutil.move(os.path.join(out,'SUB_FAM.txt'), os.path.join(out,'tmp_files','SUB_FAM.txt') )
			shutil.move(os.path.join(out,'SUB_ID.txt'), os.path.join(out,'tmp_files','SUB_ID.txt') )

		else:
			f=open(os.path.join( out,'minimac_convert.sh' ), 'w')
			probes=pd.HDFStore(os.path.join(out,'probes', self.study_name +'.h5'),'r')
			N_probes=probes.get_storer('probes').nrows
			print 'There are {} probes'.format(N_probes)
			chunk=np.vstack(((np.arange(0,N_probes,self.split_size)+1)[:-1],np.arange(0,N_probes,self.split_size)[1:]))
			N_jobs=chunk.shape[1]
			ch=[0,0]
			i_ch = -1
			for i_ch in range(chunk.shape[1]):
				ch=chunk[:,i_ch]
				#print ch
				l='bash {} {} {} {} {} {} {} {} \n'.format(
					os.path.join(os.environ['HASEDIR'],'tools','minimacGenotype2hdf5.sh'),
					self.reader.folder.path,
					out,
					os.environ['HASEDIR'],
					self.study_name,
					ch[0],
					ch[1],
					i_ch
						)
				f.write(l)
			if ch[1]!=N_probes:
				l='bash {} {} {} {} {} {} {} {} \n'.format(
					os.path.join(os.environ['HASEDIR'],'tools','minimacGenotype2hdf5.sh'),
					self.reader.folder.path,
					out,
					os.environ['HASEDIR'],
					self.study_name,
					ch[1]+1,
					N_probes,
					i_ch+1
						)
				f.write(l)
				N_jobs+=1
			f.close()
			if self.cluster:
				print 'Submit to cluster!'
				cmd="qsub -sync y -t 1-{} {} {}".format(N_jobs,os.path.join(os.environ['HASEDIR'],'tools','qsub_helper.sh'),os.path.join( out,'minimac_convert.sh' ))
				print cmd
				proc=subprocess.Popen(cmd, shell=True,stderr=subprocess.STDOUT,stdout=subprocess.PIPE).communicate()
			else:
				proc=subprocess.Popen(['bash',os.path.join( out,'minimac_convert.sh' ) ], shell=False,stderr=FNULL)
				print proc.communicate()

			shutil.move(os.path.join(out,'minimac_convert.sh'), os.path.join(out,'tmp_files','minimac_convert.sh') )
			shutil.move(os.path.join(out,'id_convert.sh'), os.path.join(out,'tmp_files','id_convert.sh') )
			shutil.move(os.path.join(out,'SUB_FAM.txt'), os.path.join(out,'tmp_files','SUB_FAM.txt') )
			shutil.move(os.path.join(out,'SUB_ID.txt'), os.path.join(out,'tmp_files','SUB_ID.txt') )

		shutil.move(os.path.join(out,'files_order.txt'),os.path.join(out,'tmp_files','files_order.txt'))
		shutil.move(os.path.join(out, 'info.txt'), os.path.join(out, 'tmp_files', 'info.txt'))

	def summary(self):
		pass
hdgwas/hdregression.py
def A_tests(covariates, genotype, intercept=True): #TODO (low) extend for any number of tests in model
	'''
	:param covariates: (n_subjects, n_covariates) - only constant covariates should be included (age, sex, ICV etc)
	:param genotype: (n_tests, n_subjects) - test could be any kind of quantitative covariance
	:return: (1,n_covariates + intercept)
	'''

	if  intercept:
		fst=np.sum(genotype, axis=1).reshape(-1,1)
		sec=np.dot(genotype, covariates)
		tr=np.sum(np.power(genotype,2), axis=1).reshape(-1,1)
		return np.hstack((fst, sec, tr))

	else:
		sec=np.dot(genotype, covariates)
		tr=np.sum(np.power(genotype,2), axis=1).reshape(-1,1)
		return np.hstack(( sec, tr))
hdgwas/tools.py
def timing(f):
	def wrap(*args,**kwargs):
		time1 = time.time()
		ret = f(*args,**kwargs)
		time2 = time.time()
		print '%s function took %0.3f s' % (f.func_name, (time2 - time1) )
		return ret

	return wrap
hdgwas/tools.py
def study_indexes( args=None, genotype=None,phenotype=None,covariates=None):

	def _get_id(notype):
		if isinstance(notype,type(None)):
			id_p=np.array([],dtype=np.str)
		else:
			if isinstance(notype,tuple):
				id_p=np.array([],dtype=np.str)
				for i in notype:
					if isinstance(i,dict):
						id_p=np.append(id_p,i['id'])
					else:
						id_p=np.append(id_p,i.get_id())
			else:
				id_p=notype.get_id()
		return id_p.astype(np.str)

	id_c=_get_id(covariates)
	id_p=_get_id(phenotype)
	id_g=_get_id(genotype)

	index_g=np.array([])
	index_p=np.array([])
	index_c=np.array([])

	if args is not None:
		if not isinstance(args.ind_id_inc,type(None)) or not isinstance(args.ind_id_exc,type(None)):
			pass
		if not isinstance(args.cov_name_inc,type(None)) or not isinstance(args.cov_name_exc,type(None)):
			pass

	if len(id_g)!=0 and len(id_p)!=0 and len(id_c)!=0:
		common_id=(np.intersect1d(np.intersect1d(id_g,id_p),id_c))

		index_g=np.array([np.where(id_g==i)[0][0] for i in common_id])
		index_p=np.array([np.where(id_p==i)[0][0] for i in common_id])
		index_c=np.array([np.where(id_c==i)[0][0] for i in common_id])

	elif len(id_g)==0:
		common_id=(np.intersect1d(id_p,id_c))
		index_p=np.array([np.where(id_p==i)[0][0] for i in common_id])
		index_c=np.array([np.where(id_c==i)[0][0] for i in common_id])

	elif len(id_c)==0:
		common_id=(np.intersect1d(id_p,id_g))
		index_g=np.array([np.where(id_g==i)[0][0] for i in common_id])
		index_p=np.array([np.where(id_p==i)[0][0] for i in common_id])

	else:
		common_id=(np.intersect1d(id_g,id_c))
		index_g=np.array([np.where(id_g==i)[0][0] for i in common_id])
		index_c=np.array([np.where(id_c==i)[0][0] for i in common_id])

	print ('There are {} common ids'.format(len(common_id)))
	np.savetxt(os.path.join(os.environ['HASEOUT'],'study_common_id.txt'),common_id, fmt='%s')
	np.savetxt(os.path.join(os.environ['HASEOUT'], 'gen_id.txt'), index_g, fmt='%s')
	np.savetxt(os.path.join(os.environ['HASEOUT'], 'phen_id.txt'), index_p, fmt='%s')
	np.savetxt(os.path.join(os.environ['HASEOUT'], 'cov_id.txt'), index_c, fmt='%s')
	if len(common_id)==0:
		exit(0)

	return [index_g,index_p,index_c], np.array(common_id)
hdgwas/tools.py
def merge_genotype(genotype, SNPs_index , mapper, flip_flag=True):

	if SNPs_index is None:
		gen=genotype[0].get_next()
		if gen is not None:
			for i in range(1, len(genotype)):
				gen=np.hstack( (gen,genotype[i].get_next() ) )
		return gen
	else:
		if len(genotype)!=len(SNPs_index):
			raise ValueError('There are not equal number of genotypes and SNPs indexes {}!={}'.format(len(genotype), len(SNPs_index)))
		gen=np.zeros( ( len(SNPs_index[0]), np.sum( [i.folder._data.id.shape[0] for i in genotype]  )  )  )
		a,b=0,genotype[0].folder._data.id.shape[0]
		gen[:,a:b]=genotype[0].get(SNPs_index[0],impute=mapper.encoded.get(genotype[0].folder.name,0) )
		a=b
		print gen.shape
		if flip_flag:
			if not mapper.encoded.get(genotype[0].folder.name, 0):
				flip=mapper.flip[genotype[0].folder.name][SNPs_index[0]]
				flip_index=(flip==-1)
				gen=np.apply_along_axis(lambda x: flip*(x-2*flip_index) ,0,gen)
		for i in range(1, len(genotype)):
			g=genotype[i].get(SNPs_index[i],impute=mapper.encoded.get(genotype[i].folder.name,0))
			print g.shape
			b+=g.shape[1]
			if flip_flag:
				if not mapper.encoded.get(genotype[i].folder.name,0):
					flip=mapper.flip[genotype[i].folder.name][SNPs_index[i]]
					flip_index=(flip==-1)
					g=np.apply_along_axis(lambda x: flip*(x-2*flip_index) ,0,g)

			gen[:,a:b]=g
			a=b

		return gen


Question: import sys
import os
import numpy as np
import h5py
import tables
import argparse
import gc
import pandas as pd
import time
from config import MAPPER_CHUNK_SIZE, basedir,CONVERTER_SPLIT_SIZE, PYTHON_PATH
from  hdgwas.tools import Timer, Checker, study_indexes, Mapper,HaseAnalyser, merge_genotype, Reference, timing, check_np, check_converter
from hdgwas.converter import  GenotypePLINK, GenotypeMINIMAC, GenotypeVCF
from hdgwas.data import Reader, MetaParData, MetaPhenotype
from hdgwas.fake import Encoder
from hdgwas.hdregression import HASE, A_covariates, A_tests, B_covariates, C_matrix, A_inverse,B4
from hdgwas.pard import partial_derivatives
from hdgwas.regression import haseregression
from hdgwas.protocol import Protocol
HEAD += "*********************************************************************\n"




if __name__=='__main__':

	start = time.time()

	parser = argparse.ArgumentParser(description='Script to use HASE in command line')

	#MAIN SETTINGS
	parser.add_argument("-thr",type=float,
						help="predefined threshold for t-value, if not defined save everything")

	parser.add_argument("-o", "--out", type=str, required=True, help="path to save result folder")


	parser.add_argument("-mode", required=True,type=str,
						choices=['regression','converting','single-meta', 'meta-stage','encoding'],
						help='Main argument, specify type of analysis'
							 '*****************'
							 'converting - before start to use any kind of analysis you need to convert data, it could take some time, '
							 'you need to do it ones and then everything will be super fast'
							 '*****************'
							 'single-meta - if you are the part of meta-analysis and will send data to main center, use this mode'
							 '*****************'
							 'encoding - if you are doing HD association, before send data to main center create fake data'
							 '*****************'
							 'meta-stage - if you are the main center of meta-analysis and already got precomputed data from all sites, use it'
							 '*****************'
							 'regression - if you want run everything yourself and do not use precomputed data use it'

							 'If you want to run single site full association analysis run (1) single-meta mode and then (2) meta-stage'

							 '*****************'
						)

	parser.add_argument("-g", "--genotype",nargs='+', type=str, help="path/paths to genotype data folder")
	parser.add_argument("-ph", "--phenotype", nargs='+',type=str, help="path to phenotype data folder")
	parser.add_argument("-cov", "--covariates", type=str, help="path to covariates data folder")


	parser.add_argument( "-derivatives", nargs='+', type=str, help="path to derivatives data folder")

	parser.add_argument("-protocol", type=str, help="path to study protocol file")

	parser.add_argument('-study_name', type=str, required=True,nargs='+', help=' Name for saved genotype data, without ext')

	parser.add_argument('-mapper', type=str, help='Mapper data folder')

	parser.add_argument('-ref_name', type=str, default='1000Gp1v3_ref', help='Reference panel name')

	#ADDITIONAL SETTINGS
	parser.add_argument("-snp_id_inc", type=str, help="path to file with SNPs id to include to analysis")
	parser.add_argument("-snp_id_exc", type=str, help="path to file with SNPs id to exclude from analysis")
	parser.add_argument("-ph_id_inc", type=str, help="path to file with phenotype id to exclude from analysis")
	parser.add_argument("-ph_id_exc", type=str, help="path to file with phenotype id to exclude from analysis")

	#parser.add_argument("-ind_id_inc", type=str, help="path to file with individuals id to include to analysis") #TODO (low)
	#parser.add_argument("-ind_id_exc", type=str, help="path to file with individuals id to exclude from analysis")#TODO (low)
	#parser.add_argument("-cov_name_inc", type=str, help="path to file with covariates names to include to analysis")#TODO (low)
	#parser.add_argument("-cov_name_exc", type=str, help="path to file with covariates names to exclude from analysis")#TODO (low)

	parser.add_argument('-intercept', type=str, default='y', choices=['y','n'], help='include intercept to regression, default yes')

	parser.add_argument('-maf', type=float, default=0.0, help='MAF for genetics data')

	parser.add_argument('-encoded', nargs='+', type=int, help='Value per every study, 1 - if encoded, 0 - if not')
	###

	#FLAGS
	parser.add_argument('-hdf5', action='store_true',default=True, help='flag for genotype data format')
	parser.add_argument('-id', action='store_true', default=False, help='Flag to convert minimac data to genotype per subject files first (default False)')
	parser.add_argument('-pd_full', action='store_true', default=False, help='For not HD association study')
	parser.add_argument('-effect_intercept', action='store_true', default=False, help='Flag for add study effect to PD regression model')
	parser.add_argument('-permute_ph', action='store_true', default=False, help='Flag for phenotype permutation')
	parser.add_argument('-vcf', action='store_true', default=False, help='Flag for VCF data to convert')
	parser.add_argument('-np', action='store_true', default=True, help='Check BLAS/LAPACK/MKL')

	#TODO (low) save genotype after MAF
	###

	###CLUSTER SETTING
	parser.add_argument('-cluster', type=str, default='n', choices=['y','n'], help=' Is it parallel cluster job, default no')
	parser.add_argument('-node', nargs='+',type=int,help='number of nodes / this node number, example: 10 2 ')
	###

	#ADVANCED SETTINGS
	parser.add_argument('-interaction', type=str, help='path to file with data for genotype interaction test') #TODO (low)
	parser.add_argument('-mapper_chunk', type=int, help='Change mapper chunk size from config file')
	###
	args = parser.parse_args()
	if not args.thr:
		print ('WARNING!!! You did not set threshold for t-value, all results will be saved')
	if args.mapper_chunk:
		MAPPER_CHUNK_SIZE=args.mapper_chunk
	ARG_CHECKER=Checker()
	print args
	os.environ['HASEOUT']=args.out

	if args.cluster=='y':
		if args.node is not None:
			if args.node[1]>args.node[0]:
				raise ValueError('Node # {} > {} total number of nodes'.format(args.node[1], args.node[0] ))



	if not os.path.isdir(args.out):
		print "Creating output folder {}".format(args.out)
		os.mkdir(args.out)

	if args.np:
		check_np()

	################################### CONVERTING ##############################
	if args.mode=='converting':

		#ARG_CHECKER.check(args,mode='converting')


Answer: 		R=Reader('genotype')