# --------------------------------------------------------
# Tensorflow Faster R-CNN
# Licensed under The MIT License [see LICENSE for details]
# Written by Jiasen Lu, Jianwei Yang, based on code from Ross Girshick
# --------------------------------------------------------
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import _init_paths
import os
import sys
import numpy as np
import argparse
import pprint
import pdb
import time
import cv2
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim

import xml.etree.ElementTree as ET
import torchvision.transforms as transforms
import torchvision.datasets as dset
from scipy.misc import imread
from roi_data_layer.roidb import combined_roidb
from roi_data_layer.roibatchLoader import roibatchLoader
from model.utils.config import cfg, cfg_from_file, cfg_from_list, get_output_dir
from model.rpn.bbox_transform import clip_boxes
from model.nms.nms_wrapper import nms
from model.rpn.bbox_transform import bbox_transform_inv
from model.utils.net_utils import save_net, load_net, vis_detections
from model.utils.blob import im_list_to_blob
from model.faster_rcnn.vgg16 import vgg16
from model.faster_rcnn.resnet import resnet
from model.faster_rcnn.prefood_res50 import PreResNet50
from datasets.food_category import get_categories
from datasets.id2name import id2eng, id2chn

try:
    xrange          # Python 2
except NameError:
    xrange = range  # Python 3

beehoonid2name = {'1': 'bee hoon', '2': 'fried noodles', '3': 'kway teow', '4': 'kway teow, yellow noodles mix', '5': 'rice', '51': 'fried rice', '7': 'hokkien mee', '8': 'maggie noodle', '9': 'Glutinous rice', '10': 'beehoon and noodle mix', '110': 'stir fry mee tai mak', '11': 'fried egg', '12': 'scrambled egg', '13': 'cabbage', '131': 'hairy gourd with egg', '14': 'french bean/long bean', '141': 'broccoli', '142': 'celery', '143': 'beansprout', '15': 'deep fried beancurd skin', '16':
                  'fried beancurd/taukwa', '17': 'taupok', '171': 'braised taupok', '18': 'Acar', '181': 'Stir fried eggplant', '19': 'cucumber', '21': 'luncheon meat', '22': 'hashbrown', '23': 'ngoh hiang', '24': 'begedil', '25': 'spring roll', '31': 'otah', '32': 'fish ball/sotong ball', '33': 'white, yellow fish fillet', '331': 'orange, red fish fillet', '34': 'fish cake', '341': 'ngoh hiang fish cake', '35': 'kuning fish (fried small fish)', '351': 'fried fish steak', '36': 'siew mai',
                  '41': 'hotdog/taiwan sausage', '42': 'seaweed chicken', '43': 'chicken nugget', '44': 'fried chicken / chicken wings', '441': 'fried chicken chopped up', '45': 'fried chicken cutlet (not ground meat)', '55': 'curry mixed veg', '551': 'curry chicken and potato', '61': 'ikan bilis', '62': 'chilli paste', '63': 'green chilli', '64': 'peanut', '65': 'Sweet Sauce', '66': 'red chilli chopped', '71': 'deep fried fish', '91': 'Butter cereal chicken', '92': 'fried wanton/ dumpling', '93':
                  'Vegetarian meat', '94': 'Fried onions', '95': 'Crabstick'}

#id2chn = beehoonid2name

def parse_rec(filename):
    """ Parse a PASCAL VOC xml file """
    tree = ET.parse(filename)
    objects = []
    for obj in tree.findall('object'):
        obj_struct = {}
        obj_struct['name'] = obj.find('name').text
        obj_struct['pose'] = obj.find('pose').text
        obj_struct['truncated'] = int(obj.find('truncated').text)
        obj_struct['difficult'] = int(obj.find('difficult').text)
        bbox = obj.find('bndbox')
        obj_struct['bbox'] = [int(bbox.find('xmin').text),
                              int(bbox.find('ymin').text),
                              int(bbox.find('xmax').text),
                              int(bbox.find('ymax').text)]
        objects.append(obj_struct)

    return objects

def parse_args():
    """
    Parse input arguments
    """
    parser = argparse.ArgumentParser(description='Train a Fast R-CNN network')
    parser.add_argument('--dataset', dest='dataset',
                        help='training dataset',
                        default='pascal_voc', type=str)
    parser.add_argument('--cfg', dest='cfg_file',
                        help='optional config file',
                        default='cfgs/vgg16.yml', type=str)
    parser.add_argument('--net', dest='net',
                        help='vgg16, res50, res101, res152',
                        default='res101', type=str)
    parser.add_argument('--set', dest='set_cfgs',
                        help='set config keys', default=None,
                        nargs=argparse.REMAINDER)
    parser.add_argument('--load_dir', dest='load_dir',
                        help='directory to load models',
                        default="/srv/share/jyang375/models")
    parser.add_argument('--image_dir', dest='image_dir',
                        help='directory to load images for demo',
                        default="images")
    parser.add_argument('--cuda', dest='cuda',
                        help='whether use CUDA',
                        action='store_true')
    parser.add_argument('--mGPUs', dest='mGPUs',
                        help='whether use multiple GPUs',
                        action='store_true')
    parser.add_argument('--cag', dest='class_agnostic',
                        help='whether perform class_agnostic bbox regression',
                        action='store_true')
    parser.add_argument('--parallel_type', dest='parallel_type',
                        help='which part of model to parallel, 0: all, 1: model before roi pooling',
                        default=0, type=int)
    parser.add_argument('--checksession', dest='checksession',
                        help='checksession to load model',
                        default=1, type=int)
    parser.add_argument('--checkepoch', dest='checkepoch',
                        help='checkepoch to load network',
                        default=1, type=int)
    parser.add_argument('--checkpoint', dest='checkpoint',
                        help='checkpoint to load network',
                        default=10021, type=int)
    parser.add_argument('--bs', dest='batch_size',
                        help='batch_size',
                        default=1, type=int)
    parser.add_argument('--vis', dest='vis',
                        help='visualization mode',
                        action='store_true')
    parser.add_argument('--webcam_num', dest='webcam_num',
                        help='webcam ID number',
                        default=-1, type=int)

    args = parser.parse_args()
    return args

lr = cfg.TRAIN.LEARNING_RATE
momentum = cfg.TRAIN.MOMENTUM
weight_decay = cfg.TRAIN.WEIGHT_DECAY

def _get_image_blob(im):
    """Converts an image into a network input.
    Arguments:
      im (ndarray): a color image in BGR order
    Returns:
      blob (ndarray): a data blob holding an image pyramid
      im_scale_factors (list): list of image scales (relative to im) used
        in the image pyramid
    """
    im_orig = im.astype(np.float32, copy=True)
    im_orig -= cfg.PIXEL_MEANS

    im_shape = im_orig.shape
    im_size_min = np.min(im_shape[0:2])
    im_size_max = np.max(im_shape[0:2])

    processed_ims = []
    im_scale_factors = []

    for target_size in cfg.TEST.SCALES:
        im_scale = float(target_size) / float(im_size_min)
        # Prevent the biggest axis from being more than MAX_SIZE
        if np.round(im_scale * im_size_max) > cfg.TEST.MAX_SIZE:
            im_scale = float(cfg.TEST.MAX_SIZE) / float(im_size_max)
        im = cv2.resize(im_orig, None, None, fx=im_scale, fy=im_scale,
                        interpolation=cv2.INTER_LINEAR)
        im_scale_factors.append(im_scale)
        processed_ims.append(im)

    # Create a blob to hold the input images
    blob = im_list_to_blob(processed_ims)

    return blob, np.array(im_scale_factors)

if __name__ == '__main__':

    args = parse_args()

    print('Called with args:')
    print(args)

    if args.cfg_file is not None:
        cfg_from_file(args.cfg_file)
    if args.set_cfgs is not None:
        cfg_from_list(args.set_cfgs)

    cfg.USE_GPU_NMS = args.cuda

    print('Using config:')
    pprint.pprint(cfg)
    np.random.seed(cfg.RNG_SEED)

    # train set
    # -- Note: Use validation set and disable the flipped to enable faster loading.

    input_dir = args.load_dir + "/" + args.net + "/" + args.dataset
    if not os.path.exists(input_dir):
        raise Exception(
            'There is no input directory for loading network from ' + input_dir)
    load_name = os.path.join(input_dir,
                             'faster_rcnn_{}_{}_{}.pth'.format(args.checksession, args.checkepoch, args.checkpoint))

    #pascal_classes = np.asarray(get_categories("EconomicBeeHoon_train"))
    pascal_classes = get_categories('All_train_mt10')

    # initilize the network here.
    if args.net == 'vgg16':
        fasterRCNN = vgg16(pascal_classes, pretrained=False,
                           class_agnostic=args.class_agnostic)
    elif args.net == 'res101':
        fasterRCNN = resnet(pascal_classes, 101, pretrained=False,
                            class_agnostic=args.class_agnostic)
    elif args.net == 'res50':
        fasterRCNN = resnet(pascal_classes, 50, pretrained=False,
                            class_agnostic=args.class_agnostic)
    elif args.net == 'res152':
        fasterRCNN = resnet(pascal_classes, 152, pretrained=False,
                            class_agnostic=args.class_agnostic)
    elif args.net == 'foodres50':
        fasterRCNN = PreResNet50(pascal_classes, pretrained=False,
                                 class_agnostic=args.class_agnostic)
    else:
        print("network is not defined")
        pdb.set_trace()

    fasterRCNN.create_architecture()

    print("load checkpoint %s" % (load_name))
    if args.cuda > 0:
        checkpoint = torch.load(load_name)
    else:
        checkpoint = torch.load(
            load_name, map_location=(lambda storage, loc: storage))
    fasterRCNN.load_state_dict(checkpoint['model'])
    if 'pooling_mode' in checkpoint.keys():
        cfg.POOLING_MODE = checkpoint['pooling_mode']

    print('load model successfully!')

    # pdb.set_trace()

    print("load checkpoint %s" % (load_name))

    # initilize the tensor holder here.
    im_data = torch.FloatTensor(1)
    im_info = torch.FloatTensor(1)
    num_boxes = torch.LongTensor(1)
    gt_boxes = torch.FloatTensor(1)

    # ship to cuda
    if args.cuda > 0:
        im_data = im_data.cuda()
        im_info = im_info.cuda()
        num_boxes = num_boxes.cuda()
        gt_boxes = gt_boxes.cuda()

    # make variable
    im_data = Variable(im_data, volatile=True)
    im_info = Variable(im_info, volatile=True)
    num_boxes = Variable(num_boxes, volatile=True)
    gt_boxes = Variable(gt_boxes, volatile=True)

    if args.cuda > 0:
        cfg.CUDA = True

    if args.cuda > 0:
        fasterRCNN.cuda()

    fasterRCNN.eval()

    start = time.time()
    max_per_image = 100
    thresh = 0.05
    vis = True

    webcam_num = args.webcam_num
    # Set up webcam or get image directories
    if webcam_num >= 0:
        cap = cv2.VideoCapture(webcam_num)
        num_images = 0
    else:
        imglist = os.listdir(args.image_dir)
        num_images = len(imglist)

    print('Loaded Photo: {} images.'.format(num_images))

    while (num_images >= 0):
        total_tic = time.time()
        if webcam_num < 0:
            num_images -= 1

        # Get image from the webcam
        if webcam_num >= 0:
            if not cap.isOpened():
                raise RuntimeError(
                    "Webcam could not open. Please check connection.")
            ret, frame = cap.read()
            im_in = np.array(frame)
        # Load the demo image
        else:
            im_file = os.path.join(args.image_dir, imglist[num_images])
            # im = cv2.imread(im_file)
            im_in = np.array(imread(im_file))
        if len(im_in.shape) == 2:
            im_in = im_in[:, :, np.newaxis]
            im_in = np.concatenate((im_in, im_in, im_in), axis=2)
        # rgb -> bgr
        im = im_in[:, :, ::-1]

        blobs, im_scales = _get_image_blob(im)
        assert len(im_scales) == 1, "Only single-image batch implemented"
        im_blob = blobs
        im_info_np = np.array(
            [[im_blob.shape[1], im_blob.shape[2], im_scales[0]]], dtype=np.float32)

        im_data_pt = torch.from_numpy(im_blob)
        im_data_pt = im_data_pt.permute(0, 3, 1, 2)
        im_info_pt = torch.from_numpy(im_info_np)

        im_data.data.resize_(im_data_pt.size()).copy_(im_data_pt)
        im_info.data.resize_(im_info_pt.size()).copy_(im_info_pt)
        gt_boxes.data.resize_(1, 1, 5).zero_()
        num_boxes.data.resize_(1).zero_()

        # pdb.set_trace()
        det_tic = time.time()

        rois, cls_prob, bbox_pred, \
            rpn_loss_cls, rpn_loss_box, \
            RCNN_loss_cls, RCNN_loss_bbox, \
            rois_label = fasterRCNN(im_data, im_info, gt_boxes, num_boxes)

        scores = cls_prob.data
        boxes = rois.data[:, :, 1:5]

        if cfg.TEST.BBOX_REG:
                # Apply bounding-box regression deltas
            box_deltas = bbox_pred.data
            if cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED:
                # Optionally normalize targets by a precomputed mean and stdev
                if args.class_agnostic:
                    if args.cuda > 0:
                        box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda() \
                            + torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda()
                    else:
                        box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_STDS) \
                            + torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_MEANS)

                    box_deltas = box_deltas.view(1, -1, 4)
                else:
                    if args.cuda > 0:
                        box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_STDS).cuda() \
                            + torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_MEANS).cuda()
                    else:
                        box_deltas = box_deltas.view(-1, 4) * torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_STDS) \
                            + torch.FloatTensor(cfg.TRAIN.BBOX_NORMALIZE_MEANS)
                    box_deltas = box_deltas.view(
                        1, -1, 4 * len(pascal_classes))

            pred_boxes = bbox_transform_inv(boxes, box_deltas, 1)
            pred_boxes = clip_boxes(pred_boxes, im_info.data, 1)
        else:
            # Simply repeat the boxes, once for each class
            pred_boxes = np.tile(boxes, (1, scores.shape[1]))

        pred_boxes /= im_scales[0]

        scores = scores.squeeze()
        pred_boxes = pred_boxes.squeeze()
        det_toc = time.time()
        detect_time = det_toc - det_tic
        misc_tic = time.time()

        # get gt
        # 1. read xml

        if vis:
            im2show = np.copy(im)
        for j in xrange(1, len(pascal_classes)):
            inds = torch.nonzero(scores[:, j] > thresh).view(-1)
            # if there is det
            if inds.numel() > 0:
                cls_scores = scores[:, j][inds]
                _, order = torch.sort(cls_scores, 0, True)
                if args.class_agnostic:
                    cls_boxes = pred_boxes[inds, :]
                else:
                    cls_boxes = pred_boxes[inds][:, j * 4:(j + 1) * 4]

                cls_dets = torch.cat((cls_boxes, cls_scores.unsqueeze(1)), 1)
                # cls_dets = torch.cat((cls_boxes, cls_scores), 1)
                cls_dets = cls_dets[order]
                keep = nms(cls_dets, cfg.TEST.NMS,
                           force_cpu=not cfg.USE_GPU_NMS)
                cls_dets = cls_dets[keep.view(-1).long()]
                if vis:
                    im2show = vis_detections(
                        im2show, id2eng[pascal_classes[j]], cls_dets.cpu().numpy(), 0.5)

        misc_toc = time.time()
        nms_time = misc_toc - misc_tic

        if webcam_num == -1:
            sys.stdout.write('im_detect: {:d}/{:d} {:.3f}s {:.3f}s   \r'
                             .format(num_images + 1, len(imglist), detect_time, nms_time))
            sys.stdout.flush()

        if vis and webcam_num == -1:
            # cv2.imshow('test', im2show)
            # cv2.waitKey(0)
            result_path = os.path.join(
                args.image_dir, imglist[num_images][:-4] + "_det.jpg")
            cv2.imwrite(result_path, im2show)
        else:
            #im2showRGB = cv2.cvtColor(im2show, cv2.COLOR_BGR2RGB)
            im2showRGB = im2show
            cv2.namedWindow("frame", 0)
            cv2.resizeWindow("frame", 800, 800)
            cv2.imshow("frame", im2showRGB)
            total_toc = time.time()
            total_time = total_toc - total_tic
            frame_rate = 1 / total_time
            print('Frame rate:', frame_rate)
            if cv2.waitKey(5000) & 0xFF == ord('q'):
                break
    if webcam_num >= 0:
        cap.release()
        cv2.destroyAllWindows()


__author__ = "Timothy Tickle"
__copyright__ = "Copyright 2015"
__credits__ = [ "Timothy Tickle", "Brian Haas" ]
__license__ = "MIT"
__maintainer__ = "Timothy Tickle"
__email__ = "ttickle@broadinstitute.org"
__status__ = "Development"

import Commandline
import os
import ParentPipelineTester
import unittest

class ScriptTester( ParentPipelineTester.ParentPipelineTester ):
    """
    Testing for scripts, starting at command line.
    """
    str_script_dir = "src"
    str_test_data_dir = "test_data"
    str_test_data_dir_working = os.path.join( str_test_data_dir, "active_testing_script_tester" )

# combine_vchk.py
    def test_combine_vchk_for_1_file( self ):
        """
        Test combining VCHK files when only one is given
        """
        # Create test environment
        str_combine_script = os.path.join( self.str_script_dir, "combine_vchk.py" )
        str_vchk_dir = os.path.join( self.str_test_data_dir, "test_combine_vchk_for_1_file" )
        str_output_dir = os.path.join( self.str_test_data_dir_working, "test_combine_vchk_for_1_file" )
        str_substitutions_dis_json = os.path.join( self.str_test_data_dir, "test_combine_vchk_1_distributions_substitutions_ANSWER.json" )
        str_substitutions_dis_json_test = os.path.join( str_output_dir, "Distributions_substitutions.json" )
        str_substitutions_dis_pdf_test = os.path.join( str_output_dir, "Distributions_substitutions.pdf" )
        str_substitions_total_json = os.path.join( self.str_test_data_dir, "test_combine_vchk_for_1_total_substitutions_ANSWER.json" )
        str_substitions_total_json_test = os.path.join( str_output_dir, "Total_substitutions.json" )
        str_substitions_total_pdf_test = os.path.join( str_output_dir, "Total_substitutions.pdf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( str_output_dir )
        # Call Example script
        str_command = " ".join( [ str_combine_script, "--input_dir", str_vchk_dir, "--output_dir", str_output_dir ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_substitutions_dis_json, str_substitutions_dis_json_test )
        f_success = f_success and self.func_are_files_equivalent( str_substitions_total_json, str_substitions_total_json_test )
        # Destroy environment
        self.func_remove_files( [ str_substitutions_dis_json_test, str_substitions_total_json_test ] )
        self.func_remove_files( [ str_substitutions_dis_pdf_test, str_substitions_total_pdf_test ] )
        self.func_remove_dirs( [ str_output_dir ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_combine_vchk_for_2_file( self ):
        """
        Test combining VCHK files when two are given
        """
        # Create test environment
        str_combine_script = os.path.join( self.str_script_dir, "combine_vchk.py" )
        str_vchk_dir = os.path.join( self.str_test_data_dir, "test_combine_vchk_for_2_file" )
        str_output_dir = os.path.join( self.str_test_data_dir_working, "test_combine_vchk_for_2_file" )
        str_substitutions_dis_json = os.path.join( self.str_test_data_dir, "test_combine_vchk_2_distributions_substitutions_ANSWER.json" )
        str_substitutions_dis_json_test = os.path.join( str_output_dir, "Distributions_substitutions.json" )
        str_substitutions_dis_pdf_test = os.path.join( str_output_dir, "Distributions_substitutions.pdf" )
        str_substitions_total_json = os.path.join( self.str_test_data_dir, "test_combine_vchk_for_2_total_substitutions_ANSWER.json" )
        str_substitions_total_json_test = os.path.join( str_output_dir, "Total_substitutions.json" )
        str_substitions_total_pdf_test = os.path.join( str_output_dir, "Total_substitutions.pdf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( str_output_dir )
        # Call Example script
        str_command = " ".join( [ str_combine_script, "--input_dir", str_vchk_dir, "--output_dir", str_output_dir ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_substitutions_dis_json, str_substitutions_dis_json_test )
        f_success = f_success and self.func_are_files_equivalent( str_substitions_total_json, str_substitions_total_json_test )
        # Destroy environment
        self.func_remove_files( [ str_substitutions_dis_json_test, str_substitions_total_json_test ] )
        self.func_remove_files( [ str_substitutions_dis_pdf_test, str_substitions_total_pdf_test ] )
        self.func_remove_dirs( [ str_output_dir ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_combine_vchk_for_3_file( self ):
        """
        Test combining VCHK files when three are given
        """
        # Create test environment
        str_combine_script = os.path.join( self.str_script_dir, "combine_vchk.py" )
        str_vchk_dir = os.path.join( self.str_test_data_dir, "test_combine_vchk_for_3_file" )
        str_output_dir = os.path.join( self.str_test_data_dir_working, "test_combine_vchk_for_3_file" )
        str_substitutions_dis_json = os.path.join( self.str_test_data_dir, "test_combine_vchk_3_distributions_substitutions_ANSWER.json" )
        str_substitutions_dis_json_test = os.path.join( str_output_dir, "Distributions_substitutions.json" )
        str_substitutions_dis_pdf_test = os.path.join( str_output_dir, "Distributions_substitutions.pdf" )
        str_substitions_total_json = os.path.join( self.str_test_data_dir, "test_combine_vchk_for_3_total_substitutions_ANSWER.json" )
        str_substitions_total_json_test = os.path.join( str_output_dir, "Total_substitutions.json" )
        str_substitions_total_pdf_test = os.path.join( str_output_dir, "Total_substitutions.pdf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( str_output_dir )
        # Call Example script
        str_command = " ".join( [ str_combine_script, "--input_dir", str_vchk_dir, "--output_dir", str_output_dir ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_substitutions_dis_json, str_substitutions_dis_json_test )
        f_success = f_success and self.func_are_files_equivalent( str_substitions_total_json, str_substitions_total_json_test )
        # Destroy environment
        self.func_remove_files( [ str_substitutions_dis_json_test, str_substitions_total_json_test ] )
        self.func_remove_files( [ str_substitutions_dis_pdf_test, str_substitions_total_pdf_test ] )
        self.func_remove_dirs( [ str_output_dir ] )
        # Evaluate
        self.func_test_true( f_success )

# comfirm_maf_mutations.py
# filter_snps_rna_editing.py
    def test_filter_snps_rna_editing_no_resources( self ):
        """
        Test filter_snps_rna_editing with data no resources for filtering
        """
        # Create test environment
        str_filter_script = os.path.join( self.str_script_dir, "filter_snps_rna_editing.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_no_resources.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_no_resources_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_snps_rna_editing_no_resources.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        #self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_snps_rna_editing_with_darned( self ):
        """
        Test filter_snps_rna_editing with data darned resources for filtering
        """
        # Create test environment
        str_filter_script = os.path.join( self.str_script_dir, "filter_snps_rna_editing.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_darned.vcf" )
        str_darned_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_darned.tab" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_darned_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_snps_rna_editing_darned.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_script, "--darned", str_darned_file, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        #self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_snps_rna_editing_with_radar( self ):
        """
        Test filter_snps_rna_editing with data radar resources for filtering
        """
        # Create test environment
        str_filter_script = os.path.join( self.str_script_dir, "filter_snps_rna_editing.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_radar.vcf" )
        str_radar_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_radar.tab" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_radar_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_snps_rna_editing_radar.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_script, "--radar", str_radar_file, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        #self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_snps_rna_editing_with_darned_radar( self ):
        """
        Test filter_snps_rna_editing with darned and radar data resources for filtering
        """
        # Create test environment
        str_filter_script = os.path.join( self.str_script_dir, "filter_snps_rna_editing.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_radar_darned.vcf" )
        str_radar_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_radar.tab" )
        str_darned_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_darned.tab" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_snps_rna_editing_radar_darned_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_snps_rna_editing_radar_darned.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_script, "--radar", str_radar_file, "--darned", str_darned_file, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        #self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

# filter_variant_clusters.py
    def test_filter_clusters_for_no_filtering( self ):
        """
        Test filter_variant_cluster with data that does not need filtering
        """
        # Create test environment
        str_filter_script = os.path.join( self.str_script_dir, "filter_variant_clusters.py" )
        str_output_dir = os.path.join( self.str_test_data_dir_working, "test_filter_clusters" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_clusters.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_clusters_for_no_filtering_ANSWER.vcf" )
        str_result_file = os.path.join( str_output_dir, "test_filter_clusters_for_no_filtering.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        self.func_make_dummy_dir( str_output_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_script, "--window", "35", "--cluster", "34", str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_test_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        self.func_remove_dirs( [ str_output_dir ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_clusters_for_all_filtering( self ):
        """
        Test filter_variant_cluster with data that will be completely filtered
        """
        # Create test environment
        str_filter_script = os.path.join( self.str_script_dir, "filter_variant_clusters.py" )
        str_output_dir = os.path.join( self.str_test_data_dir_working, "test_filter_clusters" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_clusters.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_clusters_for_all_filtering_ANSWER.vcf" )
        str_result_file = os.path.join( str_output_dir, "test_filter_clusters_for_all_filtering.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        self.func_make_dummy_dir( str_output_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_script, "--window", "1", "--cluster", "1", str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        self.func_remove_dirs( [ str_output_dir ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_clusters_for_mild_filtering( self ):
        """
        Test filter_variant_cluster with mild filtering
        """
        # Create test environment
        str_filter_script = os.path.join( self.str_script_dir, "filter_variant_clusters.py" )
        str_output_dir = os.path.join( self.str_test_data_dir_working, "test_filter_clusters" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_clusters.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_clusters_for_mild_filtering_ANSWER.vcf" )
        str_result_file = os.path.join( str_output_dir, "test_filter_clusters_for_mild_filtering.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        self.func_make_dummy_dir( str_output_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_script, "--window", "35", "--cluster", "2", str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        self.func_remove_dirs( [ str_output_dir ] )
        # Evaluate
        self.func_test_true( f_success )

# filter_vcf_for_cancer.py
    def test_filter_vcf_for_cancer_for_COMMON_filtering( self ):
        """
        Test filter_vcf_for_cancer for COMMON features.
        """
        # Create test environment
        str_filter_cancer_script = os.path.join( self.str_script_dir, "filter_vcf_for_cancer.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_COMMON.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_COMMON_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_for_cancer_COMMON.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_cancer_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_vcf_for_cancer_for_DP_filtering( self ):
        """
        Test filter_vcf_for_cancer for DP features.
        """
        # Create test environment
        str_filter_cancer_script = os.path.join( self.str_script_dir, "filter_vcf_for_cancer.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_DP.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_DP_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_for_cancer_DP.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_cancer_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_vcf_for_cancer_for_SAO_filtering( self ):
        """
        Test filter_vcf_for_cancer for SAO features.
        """
        # Create test environment
        str_filter_cancer_script = os.path.join( self.str_script_dir, "filter_vcf_for_cancer.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_SAO.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_SAO_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_for_cancer_SAO.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_cancer_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_vcf_for_cancer_for_FATHMM_filtering( self ):
        """
        Test filter_vcf_for_cancer for FATHMM features.
        """
        # Create test environment
        str_filter_cancer_script = os.path.join( self.str_script_dir, "filter_vcf_for_cancer.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_FATHMM.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_FATHMM_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_for_cancer_FATHMM.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_cancer_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_filter_vcf_for_cancer_for_ALL_filtering( self ):
        """
        Test filter_vcf_for_cancer for ALL possible features.
        """
        # Create test environment
        str_filter_cancer_script = os.path.join( self.str_script_dir, "filter_vcf_for_cancer.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_ALL.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_filter_vcf_for_cancer_ALL_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_filter_for_cancer_ALL.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_filter_cancer_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

# groom_cravat_annotation.py
    def test_groom_cravat_annotation_for_coding_variants_tab( self ):
        """
        Test groom_cravat_annotation for a coding variants tab file.
        """
        # Create test environment
        str_groom_script = os.path.join( self.str_script_dir, "groom_cravat_annotation.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_groom_cravat_annotations_coding.tab" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_groom_cravat_annotations_coding_ANSWER.tab" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_groom_cravat_annotations_coding.tab" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_groom_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_groom_cravat_annotation_for_noncoding_variants_tab( self ):
        """
        Test groom_cravat_annotation for a noncoding variants tab file.
        """
        # Create test environment
        str_groom_script = os.path.join( self.str_script_dir, "groom_cravat_annotation.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_groom_cravat_annotations_noncoding.tab" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_groom_cravat_annotations_noncoding_ANSWER.tab" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_groom_cravat_annotations_noncoding.tab" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_groom_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

# groom_vcf_gatk.py
    def test_groom_vcf_gatk_for_good_vcf( self ):
        """
        Test filter_groom_vcf_gatk for a vcf not needed to be filtered
        """
        # Create test environment
        str_groom_script = os.path.join( self.str_script_dir, "groom_vcf.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_groom_vcf_gatk.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_groom_vcf_gatk_for_good_vcf_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_groom_vcf_gatk_for_good_vcf.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_groom_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_groom_vcf_gatk_for_remove_spaces_vcf( self ):
        """
        Test filter_groom_vcf_gatk for a vcf which needs spaces removed.
        """
        # Create test environment
        str_groom_script = os.path.join( self.str_script_dir, "groom_vcf.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_groom_vcf_gatk_spaces.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_groom_vcf_gatk_for_remove_spaces_vcf_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_groom_vcf_gatk_for_remove_spaces_vcf.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_groom_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_groom_vcf_gatk_for_42features_vcf( self ):
        """
        Test filter_groom_vcf_gatk for a vcf with VCF 4.2 features
        """
        # Create test environment
        str_groom_script = os.path.join( self.str_script_dir, "groom_vcf.py" )
        str_test_file = os.path.join( self.str_test_data_dir, "test_groom_vcf_gatk_42features.vcf" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_groom_vcf_gatk_for_42features_vcf_ANSWER.vcf" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_groom_vcf_gatk_for_42features_vcf.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        self.func_make_dummy_dir( self.str_test_data_dir )
        # Call Example script
        str_command = " ".join( [ str_groom_script, str_test_file, str_result_file ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file ] )
        # Evaluate
        self.func_test_true( f_success )

# reduce_vcf_to_snps.py
    def test_reduce_vcf_to_snp_for_small_file_no_filter( self ):
        """
        Test reducing the vcf file to snps for a file that is small (less than 100) and should not be filtered.
        """
        # Create test environment
        str_filtered_vcf_script = os.path.join( self.str_script_dir, "reduce_vcf_to_snps.py" )
        str_filtered_vcf_test_file = os.path.join( self.str_test_data_dir, "test_reduce_vcf_to_snp_for_small_file_no_filter.vcf" )
        str_filtered_vcf = os.path.join( self.str_test_data_dir_working, "test_reduce_vcf_to_snp_for_small_file_no_filter_RESULT.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call Example script
        str_command = " ".join( [ str_filtered_vcf_script, "--reference", str_filtered_vcf_test_file, str_filtered_vcf ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_filtered_vcf, str_filtered_vcf_test_file )
        # Destroy environment
        self.func_remove_files( [ str_filtered_vcf ] )
        # Evaluate
        self.func_test_true( f_success )

    def test_reduce_vcf_to_snp_for_small_file_filter_reference( self ):
        """
        Test reducing the vcf file to snps for a file that is small (less than 100) and should be filtered.
        When filtering as a reference, it will not have PASS info so this is ignored.
        """

        # Create test environment
        str_filtered_vcf_script = os.path.join( self.str_script_dir, "reduce_vcf_to_snps.py" )
        str_filtered_vcf_test_file = os.path.join( self.str_test_data_dir, "test_reduce_vcf_to_snp_for_small_file_filter.vcf" )
        str_filtered_vcf_answer = os.path.join( self.str_test_data_dir, "test_reduce_vcf_to_snp_for_small_file_filter_reference_ANSWER.vcf" )
        str_filtered_vcf_result = os.path.join( self.str_test_data_dir_working, "test_reduce_vcf_to_snp_for_small_file_filter_reference_RESULT.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )

        # Call Example script
        str_command = " ".join( [ str_filtered_vcf_script, "--reference", str_filtered_vcf_test_file, str_filtered_vcf_result ] )
        Commandline.Commandline().func_CMD( str_command )

        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_filtered_vcf_answer, str_filtered_vcf_result )

        # Destroy environment
        self.func_remove_files( [ str_filtered_vcf_result ] )

        # Evaluate
        self.func_test_true( f_success )

    def test_reduce_vcf_to_snp_for_small_file_filter( self ):
        """
        Test reducing the vcf file to snps for a file that is small (less than 100) and should be filtered.
        """

        # Create test environment
        str_filtered_vcf_script = os.path.join( self.str_script_dir, "reduce_vcf_to_snps.py" )
        str_filtered_vcf_test_file = os.path.join( self.str_test_data_dir, "test_reduce_vcf_to_snp_for_small_file_filter.vcf" )
        str_filtered_vcf_answer = os.path.join( self.str_test_data_dir, "test_reduce_vcf_to_snp_for_small_file_filter_ANSWER.vcf" )
        str_filtered_vcf_result = os.path.join( self.str_test_data_dir_working, "test_reduce_vcf_to_snp_for_small_file_filter_RESULT.vcf" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )

        # Call Example script
        str_command = " ".join( [ str_filtered_vcf_script, str_filtered_vcf_test_file, str_filtered_vcf_result ] )
        Commandline.Commandline().func_CMD( str_command )

        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_filtered_vcf_answer, str_filtered_vcf_result )

        # Destroy environment
        self.func_remove_files( [ str_filtered_vcf_result ] )

        # Evaluate
        self.func_test_true( f_success )

# vcfs_to_snp_calls_tab.py
    def test_vcfs_to_snp_calls_tab_filter_maf_vcf( self ):
        """
        Test vcfs_to_snp_calls_tab.py with filtering. Inputs are maf and vcf files.
        """
        # Create test environment
        str_snp_calls_script = os.path.join( self.str_script_dir, "vcfs_to_snp_calls_tab.py" )
        str_snp_calls_input_file_1 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_filter.maf" )
        str_maf_tumor_key = "test"
        str_snp_calls_input_file_2 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_1_filter.vcf" )
        str_snp_calls_input_depth_1 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_1_filter.depth" )
        str_snp_calls_input_depth_2 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_1_filter.depth" )
        str_snp_calls_answer = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_filter_maf_vcf_1_ANSWER_sorted.tab" )
        str_snp_calls_result = os.path.join( self.str_test_data_dir_working, "vcfs_to_snp_calls_tab_filter_maf_vcf_1_RESULT.tab" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call Example script
        str_command = " ".join( [ str_snp_calls_script, "--maf_reference", str_snp_calls_input_file_1, "--tumor", str_maf_tumor_key,
                                  "--vcf", str_snp_calls_input_file_2, "--count_reference", str_snp_calls_input_depth_1,
                                  "--count", str_snp_calls_input_depth_2, str_snp_calls_result ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        lstr_answer_lines = None
        with open( str_snp_calls_answer, "r" ) as hndl_answer:
            lstr_answer_lines = [ str_line for str_line in hndl_answer.read().split("\n") if str_line ]
        lstr_answer_lines.sort()
        lstr_result_lines = None
        with open( str_snp_calls_result, "r" ) as hndl_result:
            lstr_result_lines = [ str_line for str_line in hndl_result.read().split("\n") if str_line ]
        lstr_result_lines.sort()
        # Destroy environment
        self.func_remove_files( [ str_snp_calls_result ] )
        # Evaluate
        self.func_test_equals( "\n".join( lstr_answer_lines), "\n".join( lstr_result_lines ) )

    def test_vcfs_to_snp_calls_tab_filter_maf_vcf_2( self ):
        """
        Test vcfs_to_snp_calls_tab.py with filtering. Inputs are maf and vcf 2 files.
        """
        # Create test environment
        str_snp_calls_script = os.path.join( self.str_script_dir, "vcfs_to_snp_calls_tab.py" )
        str_snp_calls_input_file_1 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_filter.maf" )
        str_maf_tumor_key = "test"
        str_snp_calls_input_file_2 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_2_filter.vcf" )
        str_snp_calls_input_depth_1 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_1_filter.depth" )
        str_snp_calls_input_depth_2 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_2_filter.depth" )
        str_snp_calls_answer = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_filter_maf_vcf_2_ANSWER_sorted.tab" )
        str_snp_calls_result = os.path.join( self.str_test_data_dir_working, "vcfs_to_snp_calls_tab_filter_maf_vcf_2_RESULT.tab" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call Example script
        str_command = " ".join( [ str_snp_calls_script, "--maf_reference", str_snp_calls_input_file_1, "--tumor", str_maf_tumor_key,
                                  "--vcf", str_snp_calls_input_file_2, "--count_reference", str_snp_calls_input_depth_1,
                                  "--count", str_snp_calls_input_depth_2, str_snp_calls_result ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        lstr_answer_lines = None
        with open( str_snp_calls_answer, "r" ) as hndl_answer:
            lstr_answer_lines = [ str_line for str_line in hndl_answer.read().split("\n") if str_line ]
        lstr_answer_lines.sort()
        lstr_result_lines = None
        with open( str_snp_calls_result, "r" ) as hndl_result:
            lstr_result_lines = [ str_line for str_line in hndl_result.read().split("\n") if str_line ]
        lstr_result_lines.sort()
        # Destroy environment
        self.func_remove_files( [ str_snp_calls_result ] )
        # Evaluate
        self.func_test_equals( "\n".join( lstr_answer_lines), "\n".join( lstr_result_lines ) )

    def test_vcfs_to_snp_calls_tab_filter_vcf_1_2( self ):
        """
        Test vcfs_to_snp_calls_tab.py with filtering. Inputs are and vcf 1 and 2 files.
        """
        # Create test environment
        str_snp_calls_script = os.path.join( self.str_script_dir, "vcfs_to_snp_calls_tab.py" )
        str_snp_calls_input_file_1 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_1_filter.vcf" )
        str_snp_calls_input_file_2 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_2_filter.vcf" )
        str_snp_calls_input_depth_1 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_1_filter.depth" )
        str_snp_calls_input_depth_2 = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_2_filter.depth" )
        str_snp_calls_answer = os.path.join( self.str_test_data_dir, "vcfs_to_snp_calls_tab_filter_vcf_1_2_ANSWER_sorted.tab" )
        str_snp_calls_result = os.path.join( self.str_test_data_dir_working, "vcfs_to_snp_calls_tab_filter_vcf_1_2_RESULT.tab" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call Example script
        str_command = " ".join( [ str_snp_calls_script, "--vcf_reference", str_snp_calls_input_file_1,
                                  "--vcf", str_snp_calls_input_file_2, "--count_reference", str_snp_calls_input_depth_1,
                                  "--count", str_snp_calls_input_depth_2, str_snp_calls_result ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        lstr_answer_lines = None
        with open( str_snp_calls_answer, "r" ) as hndl_answer:
            lstr_answer_lines = [ str_line for str_line in hndl_answer.read().split("\n") if str_line ]
        lstr_answer_lines.sort()
        lstr_result_lines = None
        with open( str_snp_calls_result, "r" ) as hndl_result:
            lstr_result_lines = [ str_line for str_line in hndl_result.read().split("\n") if str_line ]
        lstr_result_lines.sort()
        # Destroy environment
        self.func_remove_files( [ str_snp_calls_result ] )
        # Evaluate
        self.func_test_equals( "\n".join( lstr_answer_lines), "\n".join( lstr_result_lines ) )

# vcfs_to_genotype_matrix.py
# This is for validation only, not for the pipeline runs.
# Under development.
    def not_test_vcfs_to_genotype_matrix_1_file( self ):
        """
        Test vcfs_to_genotype_matrix.py with one input file.
        """
        # Create test environment
        str_genotype_script = os.path.join( self.str_script_dir, "vcfs_to_genotype_matrix.py" )
        str_vcf_directory = os.path.join( self.str_test_data_dir, "test_vcf_genotype_matrix_dir_1" )
        str_genotype_answer = os.path.join( self.str_test_data_dir, "vcfs_to_genotype_matrix_1_ANSWER.txt" )
        str_genotype_result = os.path.join( self.str_test_data_dir_working, "vcfs_to_genotype_matrix_1_RESULT.txt" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call Example script
        str_command = " ".join( [ str_genotype_script, "--matrix", str_genotype_result, str_vcf_directory ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_genotype_answer, str_genotype_result )
        # Destroy environment
        self.func_remove_files( [ str_genotype_result ] )
        # Evaluate
        self.func_test_true( f_success )

    def not_test_vcfs_to_genotype_matrix_3_file( self ):
        """
        Test vcfs_to_genotype_matrix.py with one input file in one directory and 2 in another.
        """
        # Create test environment
        str_genotype_script = os.path.join( self.str_script_dir, "vcfs_to_genotype_matrix.py" )
        str_vcf_directory_1 = os.path.join( self.str_test_data_dir, "test_vcf_genotype_matrix_dir_1" )
        str_vcf_directory_2 = os.path.join( self.str_test_data_dir, "test_vcf_genotype_matrix_dir_2" )
        str_genotype_answer = os.path.join( self.str_test_data_dir, "vcfs_to_genotype_matrix_3_ANSWER.txt" )
        str_genotype_result = os.path.join( self.str_test_data_dir_working, "vcfs_to_genotype_matrix_3_RESULT.txt" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call Example script
        str_command = " ".join( [ str_genotype_script, "--matrix", str_genotype_result, str_vcf_directory_1, str_vcf_directory_2 ] )
        Commandline.Commandline().func_CMD( str_command )
        # Check test environment for results
        f_success = self.func_are_files_equivalent( str_genotype_answer, str_genotype_result )
        # Destroy environment
        self.func_remove_files( [ str_genotype_result ] )
        # Evaluate
        self.func_test_true( f_success )

# visualize_mutation_depth_tab_files.R
# This is for validation only, not for the pipeline runs.
# Under development.
    def not_test_visualize_mutation_depth_tab_files_for_error_counts_opt( self ):
        """
        Tests to make sure the TP, FP, FN, senstivity, and specificity measurements are correct from a test data set.
        This is testing output that has a changing feature space (optimization figure) and not the "ROC" plot.
        """
        # Create environment
        str_vis_script = os.path.join( self.str_script_dir, "visualize_mutation_depth_tab_files.R" )
        str_test_input_file = os.path.join( self.str_test_data_dir, "test_visualize_tab.tab" )
        str_answer_file = os.path.join( self.str_test_data_dir, "test_visualize_mutation_depth_tab_files_for_error_counts_opt_ANSWER.txt" )
        str_result_file = os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_data.txt" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call example script
        str_command = " ".join( [ str_vis_script, "-o", self.str_test_data_dir_working, "-k RNA_DNA", str_test_input_file ])
        Commandline.Commandline().func_CMD( str_command )
        # Check for sucess
        f_success = self.func_are_files_equivalent( str_answer_file, str_result_file )
        # Destroy environment
        self.func_remove_files( [ str_result_file, os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_depth_distributions.pdf" ),
                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_fdr_min_read_coverage_norm.pdf" ),
                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_data_truth_held.txt" ),
                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_optimize_detail_validation.pdf" ),
                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_raw_class_distributions_detail_validation.pdf" ),
                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_roc_detail_validation.pdf" ),
                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab.tab_sensitivity_min_read_coverage_norm.pdf" ) ] )
        self.func_remove_dirs( [ self.str_test_data_dir_working ] )
        # Evaluate
        self.func_test_true( f_success )

    def not_test_visualize_mutation_depth_tab_files_for_error_counts_roc( self ):
        """
        Tests to make sure the TP, FP, FN, senstivity, and specificity measurements are correct from a test data set.
        This is testing output that has a set feature space (the "ROC" plot) and not the optimization plot.
        """
        # Create environment
        str_vis_script = os.path.join( self.str_script_dir, "visualize_mutation_depth_tab_files.R" )
        str_test_input_file = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab" )
        str_answer_file_1 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_1_answer.txt" )
        str_answer_file_2 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_2_answer.txt" )
        str_answer_file_3 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_3_answer.txt" )
        str_answer_file_4 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_4_answer.txt" )
        str_result_file_1 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_1.txt" )
        str_result_file_2 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_2.txt" )
        str_result_file_3 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_3.txt" )
        str_result_file_4 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_4.txt" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call example script
        str_command = " ".join( [ str_vis_script, "-o", self.str_test_data_dir_working, "-k RNA_DNA", str_test_input_file ])
        Commandline.Commandline().func_CMD( str_command )
        # Check for sucess
        f_success_1 = self.func_are_files_equivalent( str_answer_file_1, str_result_file_1 )
        f_success_2 = self.func_are_files_equivalent( str_answer_file_2, str_result_file_2 )
        f_success_3 = self.func_are_files_equivalent( str_answer_file_3, str_result_file_3 )
        f_success_4 = self.func_are_files_equivalent( str_answer_file_4, str_result_file_4 )
        # Destroy environment
#        self.func_remove_files( [ os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_depth_distributions.pdf" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_fdr_min_read_coverage_norm.pdf" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data.txt" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_raw_class_distributions_detail_validation.pdf" ),
 #                                 os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_roc.pdf" ),
 #                                 os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_optimize_detail_validation.pdf" ),
 #                                 os.path.join( str_result_file_1 ),
 #                                 os.path.join( str_result_file_2 ),
 #                                 os.path.join( str_result_file_3 ),
 #                                 os.path.join( str_result_file_4 ),
 #                                 os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_sensitivity_min_read_coverage_norm.pdf" ) ] )
 #       self.func_remove_dirs( [ self.str_test_data_dir_working ] )
        # Evaluate
        self.func_test_true( f_success_1 and f_success_2 and f_success_3 and f_success_4 )

    def not_test_visualize_mutation_depth_tab_files_for_roc_like_rnaseq( self ):
        """
        Tests to make sure the TP, FP, FN, senstivity, and specificity measurements are correct from a test data set.
        This is testing output that has a set feature space (the "ROC" plot) and not the optimization plot.

        The other test uses a simple input data set similar to traditional ROC data, this one have varying RNA seq depth and
        such that allows a more authentic test.
        """
        # Create environment
        str_vis_script = os.path.join( self.str_script_dir, "visualize_mutation_depth_tab_files.R" )
        str_test_input_file = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc_like_rnaseq.tab" )
        str_answer_file_1 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_like_rnaseq_1_answer.txt" )
        str_answer_file_2 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_like_ranseq_2_answer.txt" )
        str_answer_file_3 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_like_rnaseq_3_answer.txt" )
        str_answer_file_4 = os.path.join( self.str_test_data_dir, "test_visualize_tab_roc.tab_data_roc_like_rnaseq_4_answer.txt" )
        str_result_file_1 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_like_rnaseq_1.txt" )
        str_result_file_2 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_like_rnaseq_2.txt" )
        str_result_file_3 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_like_rnaseq_3.txt" )
        str_result_file_4 = os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc.tab_data_roc_like_rnaseq_4.txt" )
        self.func_make_dummy_dir( self.str_test_data_dir_working )
        # Call example script
        str_command = " ".join( [ str_vis_script, "-o", self.str_test_data_dir_working, "-k RNA_DNA", str_test_input_file ])
        Commandline.Commandline().func_CMD( str_command )
        # Check for sucess
        f_success_1 = self.func_are_files_equivalent( str_answer_file_1, str_result_file_1 )
        f_success_2 = self.func_are_files_equivalent( str_answer_file_2, str_result_file_2 )
        f_success_3 = self.func_are_files_equivalent( str_answer_file_3, str_result_file_3 )
        f_success_4 = self.func_are_files_equivalent( str_answer_file_4, str_result_file_4 )
        # Destroy environment
#        self.func_remove_files( [ os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc_like_rnaseq.tab_depth_distributions.pdf" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc_like_rnaseq.tab_fdr_min_read_coverage_norm.pdf" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc_like_rnaseq.tab_data.txt" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc_like_rnaseq.tab_raw_class_distributions_detail_validation.pdf" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc_like_rnaseq.tab_roc.pdf" ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc_like_rnaseq.tab_optimize_detail_validation.pdf" ),
#                                  os.path.join( str_result_file_1 ),
#                                  os.path.join( str_result_file_2 ),
#                                  os.path.join( str_result_file_3 ),
#                                  os.path.join( str_result_file_4 ),
#                                  os.path.join( self.str_test_data_dir_working, "test_visualize_tab_roc_like_rnaseq.tab_sensitivity_min_read_coverage_norm.pdf" ) ] )
 #       self.func_remove_dirs( [ self.str_test_data_dir_working ] )
        # Evaluate
        self.func_test_true( f_success_1 and f_success_2 and f_success_3 and f_success_4 )

# Creates a suite of tests
def suite():
    return unittest.TestLoader().loadTestsFromTestCase( ScriptTester )


from flask import Flask
from config import config_options
from flask_bootstrap import Bootstrap
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager
from flask_uploads import UploadSet,configure_uploads,IMAGES
from flask_mail import Mail

bootstrap = Bootstrap()

db = SQLAlchemy()

login_manager = LoginManager()
login_manager.session_protection = 'strong'
login_manager.login_view = 'auth.login'

photos = UploadSet('photos',IMAGES)

mail = Mail()

def create_app(config_name):

  app = Flask(__name__)

 # Creating the app configurations
  app.config.from_object(config_options[config_name])

  # Initializing flask extensions
  bootstrap.init_app(app)
  db.init_app(app)
  login_manager.init_app(app)
  mail.init_app(app)

  # configure UploadSet
  configure_uploads(app,photos)

  # Registering the blueprints
  from .rental_hub import rental_hub as rental_hub_blueprint
  app.register_blueprint(rental_hub_blueprint)

  from .auth import auth as auth_blueprint
  app.register_blueprint(auth_blueprint,url_prefix = '/authenticate')

  return app


"""Test of Ray-tune without RLLib"""
from ray import tune

def objective(step, alpha, beta):
    return (0.1 + alpha * step / 100)**(-1) + beta * 0.1

def train(config):
    alpha, beta = config["alpha"], config["beta"]
    for step in range(10):
        score = objective(step, alpha, beta)
        tune.report(mean_loss=score)

analysis = tune.run(
    train,
    config={
        "alpha": tune.grid_search([0.001, 0.01]),
        "beta": tune.choice([1, 2])
    })

print("Best config: ", analysis.get_best_config(metric="mean_loss", mode="min"))


from concurrent import futures
from bs4 import BeautifulSoup
import pprint
import concurrent
import requests
import re
import time
import sys

MILE_TO_KM = 1.60934
CINEMA_MATCH_REGEX = r"(?P<cinema>[^,]*)"
NUM_OF_CINEMAS = 25

# TODO: PLEASE DON'T USE THESE METHODS TOO MUCH. THEY ACTUALLY QUERY THE API.
# IF YOU'RE GOING TO USE A LOT, SAVE THE DATA YOURSELF FOR TESTING.
CINEMAS = []
CINEMA_CID = {}
CINEMA_DIST = {}
F_TO_CINEMAS = {}
FCID_TO_TIMES = {}
F_CINEMA_TO_TIMES = {}

class DataParser:
    def get_cinemas_latlong(self, latitude, longitude):
        """
        Give this function a longitude and latitude and CINEMAS, CINEMA_IDS and
        DISTANCES lists are populated with (up to) 5 results.
        """
        print latitude, longitude
        sys.stdout.flush()
        global CINEMAS, CINEMA_CID, CINEMA_DIST
        film_names = requests.get(
            "https://api.cinelist.co.uk/search/cinemas/coordinates/{}/{}".
            format(latitude, longitude))
        cinemas = film_names.json()["cinemas"][:NUM_OF_CINEMAS]
        for i in cinemas:
            # Runs regex over cinemas to remove the location
            cinema_name = re.match(CINEMA_MATCH_REGEX, i['name']).group("cinema")
            # Dict storing {cinema name: cinema ID}
            CINEMA_CID[cinema_name] = i['id']
            # Converts distance from mile to km and rounds to 3dp.
            # Dict storing {cinema name: distance}
            CINEMA_DIST[cinema_name] = round(i['distance'] * MILE_TO_KM, 3)

    def get_latlong(self, postcode):
        """
        Give this function a postcode and get the corresponding latitude and
        longitude.
        """
        location_data = requests.get(
            "http://api.postcodes.io/postcodes/{}".format(postcode))
        location_data = location_data.json()
        latitude = location_data['result']['latitude']
        longitude = location_data['result']['longitude']
        return round(latitude, 6), round(longitude, 6)

    def get_cinema_url(self, cinema):
        search_url = 'https://www.google.co.uk/search?q=' + cinema
        res = requests.get(search_url)
        soup = BeautifulSoup(res.text, "lxml")

        # Parsing the html page to get the first url link in the google search
        # results, which will be the wikipedia page link
        g_search_res = soup.select('.r a')

        if not g_search_res:
            print 'VERY BAD: No google search results could be obtained'
            sys.stdout.flush()
            return ''

        fst_ref_url = g_search_res[0].get('href')

        if not fst_ref_url:
            print 'RED ALERT: First google search result has no href tag'
            sys.stdout.flush()
            return ''

        cinema_url = fst_ref_url.split('=')[1].split('&')[0]
        print(cinema_url)
        sys.stdout.flush()
        return cinema_url

    def fast_get_film_info(self, film_name):
        """
        Given FILM_NAME, this will find the corresponding movie poster and
        return the image url for the movie poster.
        """
        error_url = 'https://literalminded.files.wordpress.com' \
                    '/2010/11/image-unavailable1.png'
        error_overview = ''
        if '&' in film_name:
            film_name = 'and'.join(film_name.split('&'))

        film_name = film_name.encode('utf-8')
        api_url = 'https://api.themoviedb.org/3/search/movie?api_key=' \
                  'ab499564677631cc1c25f6749d42a16e' \
                  '&language=en-US&query={}'.format(film_name)
        res = requests.get(api_url).json()

        if res['total_results'] == 0:
            return error_url, error_overview

        first_result = res['results'][0]

        poster_path = first_result['poster_path']
        if poster_path is None:
            img_url = error_url
        else:
            img_url = 'http://image.tmdb.org/t/p/w154' + poster_path

        overview = first_result['overview']
        if overview is None:
            overview = error_overview
        else:
            groups = overview.split('.')
            overview = '.'.join(groups[:2])
            if overview[-1] != '.':
                overview += '.'

        return img_url, overview

    def get_films_for_cinemas(self, date):
        """
        Give this function a cinema ID and day and we can populate FILMS with
        all film showings and times.
        :param date: Date to get cinema films for.
        :return:
        """
        global CINEMA_CID, CINEMA_DIST
        local_data = {}

        def get_films_for_cinema(cinema):
            # Get the cinema ID for a given cinema,
            # E.g. Cineworld London - Enfield: 10477
            cinema_id = CINEMA_CID[cinema]
            # Get list of films showing at this cinema
            url = "https://mysterious-eyrie-40497.herokuapp.com/cinemas/{}/" \
                  "showings/{}".format(cinema_id, date)
            #url = "http://moviesapi.herokuapp.com/cinemas/{}/" \
            #      "showings/{}".format(cinema_id, date)
            films = requests.get(url)
            # Create a JSON object storing film name, cinema, showtimes and
            # distance to the cinema.
            films_json = films.json()

            for i in films_json:
                filmname = i["title"]
                times = i['time']
                if filmname in local_data:
                    local_data[filmname]["cinema"].append(
                        {cinema: [{"showtimes": times}]})
                else:
                    local_data[filmname] = {}
                    poster, overview = self.fast_get_film_info(filmname)
                    local_data[filmname]["image"] = poster
                    local_data[filmname]["overview"] = overview
                    local_data[filmname]["cinema"] = \
                        [{cinema: [{"showtimes": times}]}]

        executor = concurrent.futures.ThreadPoolExecutor(NUM_OF_CINEMAS)
        futures = [executor.submit(get_films_for_cinema, cinema_name)
                   for cinema_name in CINEMA_CID.keys()]
        concurrent.futures.wait(futures)

        return local_data

    def parse_date(self, day, month, year):
        """Convert date into a suitable format for use by the external API."""
        day = str(day)
        month = str(month)
        year = str(year)

        if len(day) == 1:
            day = "0" + day
        if len(month) == 1:
            month = "0" + month
        return year + "-" + month + "-" + day

    def get_local_data(self, day, month, year, latitude, longitude):
        """Get all film data for a given location."""
        self.get_cinemas_latlong(latitude, longitude)
        formatted_date = self.parse_date(day, month, year)
        return self.get_films_for_cinemas(formatted_date)

if __name__ == '__main__':
    dParser = DataParser()
    # print dParser.get_latlong("en12lz")
    start_time = time.time()
    pprint.PrettyPrinter(indent=4).pprint(
        dParser.get_local_data(25, 6, 2017, 51.636743, -0.069069))
    print dParser.get_cinema_url('Cineworld fullham road')
    print time.time() - start_time