dashboard/constants.py
WEBLATE_SLUGS = ('WLTEPUB', 'WLTEFED')
dashboard/constants.py
PIPELINE_CONFIG_EVENTS = ('Push Translations', 'Pull Translations', 'Update Source')
dashboard/constants.py
TRANSIFEX_SLUGS = ('TNFXPUB', )
dashboard/models.py
class Job(ModelMixin, models.Model):
    """
    Jobs Model
    """
    job_id = models.AutoField(primary_key=True)
    job_uuid = models.UUIDField(default=uuid4, editable=False)
    job_type = models.CharField(max_length=200)
    job_start_time = models.DateTimeField()
    job_end_time = models.DateTimeField(null=True)
    job_yml_text = models.CharField(max_length=2000, null=True, blank=True)
    job_log_json_str = models.TextField(null=True, blank=True)
    job_result = models.NullBooleanField()
    job_remarks = models.CharField(max_length=200, null=True)
    job_template = models.ForeignKey(JobTemplate, on_delete=models.PROTECT,
                                     verbose_name="Job Template", null=True)
    ci_pipeline = models.ForeignKey(CIPipeline, on_delete=models.PROTECT,
                                    verbose_name="CI Pipeline", null=True)
    job_params_json_str = models.TextField(null=True, blank=True)
    job_output_json_str = models.TextField(null=True, blank=True)
    triggered_by = models.EmailField(null=True)
    job_visible_on_url = models.BooleanField(default=False)

    @property
    def job_log_json(self):
        return self.str2json(self.job_log_json_str)

    @property
    def job_params_json(self):
        return self.str2json(self.job_params_json_str)

    @property
    def job_output_json(self):
        return self.str2json(self.job_output_json_str)

    @property
    def duration(self):
        time_diff = self.job_end_time - self.job_start_time
        return time_diff.total_seconds()

    class Meta:
        db_table = TABLE_PREFIX + 'jobs'
        verbose_name = "Job"
dashboard/models.py
class SyncStats(ModelMixin, models.Model):
    """
    Sync Stats Model
    """
    sync_id = models.AutoField(primary_key=True)
    package_name = models.ForeignKey(
        Package, on_delete=models.PROTECT,
        to_field='package_name', verbose_name="Package"
    )
    job_uuid = models.UUIDField()
    project_version = models.CharField(max_length=500, null=True)
    source = models.CharField(max_length=500, null=True)
    stats_raw_json_str = models.TextField(null=True, blank=True)
    stats_processed_json_str = models.TextField(null=True, blank=True)
    sync_iter_count = models.IntegerField()
    sync_visibility = models.BooleanField()

    @property
    def stats_raw_json(self):
        return self.str2json(self.stats_raw_json_str)

    @property
    def stats_processed_json(self):
        return self.str2json(self.stats_processed_json_str)

    class Meta:
        db_table = TABLE_PREFIX + 'syncstats'
dashboard/models.py
class CIPipeline(ModelMixin, models.Model):
    """
    Continuous Integration Pipeline Model
    """
    ci_pipeline_id = models.AutoField(primary_key=True)
    ci_pipeline_uuid = models.UUIDField(default=uuid4, unique=True, editable=False)
    ci_package = models.ForeignKey(
        Package, on_delete=models.PROTECT, verbose_name="Package"
    )
    ci_platform = models.ForeignKey(
        Platform, on_delete=models.PROTECT, verbose_name="Platform"
    )
    ci_release = models.ForeignKey(
        Release, on_delete=models.PROTECT, verbose_name="Release", null=True
    )
    ci_push_job_template = models.ForeignKey(
        JobTemplate, on_delete=models.PROTECT, verbose_name="Push Job Template",
        related_name="push_template", null=True
    )
    ci_pull_job_template = models.ForeignKey(
        JobTemplate, on_delete=models.PROTECT, verbose_name="Pull Job Template",
        related_name="pull_template", null=True
    )
    ci_project_web_url = models.URLField(max_length=500, null=True, blank=True,
                                         verbose_name="Platform Project URL")
    ci_project_details_json_str = models.TextField(null=True, blank=True)
    ci_platform_jobs_json_str = models.TextField(null=True, blank=True)
    ci_project_analyses_json_str = models.TextField(null=True, blank=True)
    ci_project_import_settings_json_str = models.TextField(null=True, blank=True)
    ci_project_assign_templates_json_str = models.TextField(null=True, blank=True)
    ci_project_workflow_steps_json_str = models.TextField(null=True, blank=True)
    ci_project_providers_json_str = models.TextField(null=True, blank=True)
    ci_project_term_bases_json_str = models.TextField(null=True, blank=True)
    ci_project_qa_checks_json_str = models.TextField(null=True, blank=True)
    ci_project_trans_memory_json_str = models.TextField(null=True, blank=True)
    ci_pipeline_last_updated = models.DateTimeField(null=True, blank=True)
    ci_pipeline_visibility = models.BooleanField(
        default=True, verbose_name='CI Pipeline Visibility'
    )

    @property
    def ci_project_details_json(self):
        return self.str2json(self.ci_project_details_json_str)

    @property
    def ci_platform_jobs_json(self):
        return self.str2json(self.ci_platform_jobs_json_str)

    @property
    def ci_project_analyses_json(self):
        return self.str2json(self.ci_project_analyses_json_str)

    @property
    def ci_project_import_settings_json(self):
        return self.str2json(self.ci_project_import_settings_json_str)

    @property
    def ci_project_assign_templates_json(self):
        return self.str2json(self.ci_project_assign_templates_json_str)

    @property
    def ci_project_workflow_steps_json(self):
        return self.str2json(self.ci_project_workflow_steps_json_str)

    @property
    def ci_project_providers_json(self):
        return self.str2json(self.ci_project_providers_json_str)

    @property
    def ci_project_term_bases_json(self):
        return self.str2json(self.ci_project_term_bases_json_str)

    @property
    def ci_project_qa_checks_json(self):
        return self.str2json(self.ci_project_qa_checks_json_str)

    @property
    def ci_project_trans_memory_json(self):
        return self.str2json(self.ci_project_trans_memory_json_str)

    def __str__(self):
        return "{} | {}".format(
            str(self.ci_pipeline_uuid)[:8],
            self.ci_package.package_name
        )

    class Meta:
        db_table = TABLE_PREFIX + 'cipipeline'
        verbose_name = "CI Pipeline"
dashboard/models.py
class LanguageSet(models.Model):
    """
    Language Set Model
    """
    lang_set_id = models.AutoField(primary_key=True)
    lang_set_name = models.CharField(
        max_length=1000, verbose_name="Language Set Name"
    )
    lang_set_slug = models.CharField(
        max_length=400, unique=True, verbose_name="Language Set SLUG"
    )
    lang_set_color = models.CharField(
        max_length=100, unique=True, verbose_name="Tag Colour"
    )
    locale_ids = ArrayField(
        models.CharField(max_length=50, blank=True),
        default=list, null=True, verbose_name="Locale IDs"
    )

    def __str__(self):
        return self.lang_set_name

    class Meta:
        db_table = TABLE_PREFIX + 'langset'
        verbose_name = "Language Set"
dashboard/models.py
class GraphRule(models.Model):
    """
    Graph Rules Model
    """
    graph_rule_id = models.AutoField(primary_key=True)
    rule_name = models.CharField(max_length=1000, unique=True,
                                 verbose_name="Rule Name")
    rule_packages = ArrayField(
        models.CharField(max_length=1000, blank=True), default=list,
        verbose_name="Rule Packages"
    )
    rule_languages = ArrayField(
        models.CharField(max_length=400, blank=True), default=list,
        verbose_name="Rule Languages"
    )
    rule_release_slug = models.ForeignKey(
        Release, on_delete=models.PROTECT,
        to_field='release_slug', verbose_name="Release"
    )
    rule_build_tags = ArrayField(
        models.CharField(max_length=200, blank=True),
        default=list, null=True, verbose_name="Rule Build Tags"
    )
    created_on = models.DateTimeField()
    rule_status = models.BooleanField(default=True)
    rule_visibility_public = models.BooleanField(default=False)
    created_by = models.EmailField(null=True)

    def __str__(self):
        return self.rule_name

    class Meta:
        db_table = TABLE_PREFIX + 'graphrules'
        verbose_name = "Graph Rule"
dashboard/models.py
class Platform(ModelMixin, models.Model):
    """
    Translation Platforms Model
    """
    platform_id = models.AutoField(primary_key=True)
    engine_name = models.CharField(
        max_length=200, verbose_name="Platform Engine"
    )
    subject = models.CharField(
        max_length=200, null=True, verbose_name="Platform Subject"
    )
    api_url = models.URLField(max_length=500, unique=True, verbose_name="Server URL")
    platform_slug = models.CharField(
        max_length=400, unique=True, verbose_name="Platform SLUG"
    )
    server_status = models.BooleanField(verbose_name="Enable/Disable")
    ci_status = models.BooleanField(verbose_name="CI Enable/Disable", default=False)
    projects_json_str = models.TextField(null=True, blank=True)
    projects_last_updated = models.DateTimeField(null=True)
    auth_login_id = models.CharField(
        max_length=200, null=True, blank=True, verbose_name="Auth User"
    )
    auth_token_key = models.CharField(
        max_length=200, null=True, blank=True, verbose_name="Auth Password/Token"
    )
    token_api_json_str = models.TextField(null=True, blank=True, verbose_name="Auth Token JSON")
    token_expiry = models.DateTimeField(null=True, blank=True, verbose_name="Auth Token Expiry")

    @property
    def projects_json(self):
        return self.str2json(self.projects_json_str)

    @property
    def token_api_json(self):
        return self.str2json(self.token_api_json_str)

    @property
    def token_status(self):
        if not self.token_expiry:
            return
        return self.token_expiry > timezone.now()

    def __str__(self):
        return "{0} {1}".format(self.engine_name, self.subject)

    class Meta:
        db_table = TABLE_PREFIX + 'platforms'
        verbose_name = "Translation Platform"
dashboard/models.py
class Visitor(models.Model):
    """
    Visitors Model
    """
    visitor_id = models.AutoField(primary_key=True)
    visitor_ip = models.GenericIPAddressField()
    visitor_user_agent = models.CharField(max_length=500)
    visitor_accept = models.CharField(max_length=500, null=True, blank=True)
    visitor_encoding = models.CharField(max_length=500, null=True, blank=True)
    visitor_language = models.CharField(max_length=500, null=True, blank=True)
    visitor_host = models.CharField(max_length=500, null=True, blank=True)
    first_visit_time = models.DateTimeField()
    last_visit_time = models.DateTimeField()

    def save(self, *args, **kwargs):
        if not self.visitor_id:
            self.first_visit_time = timezone.now()
        self.last_visit_time = timezone.now()
        return super(Visitor, self).save(*args, **kwargs)

    def __str__(self):
        return "%s: %s" % (str(self.visitor_ip), self.visitor_user_agent)

    class Meta:
        db_table = TABLE_PREFIX + 'visitors'
        verbose_name = "Visitor"
dashboard/models.py
class CacheBuildDetails(ModelMixin, models.Model):
    """
    Cache Build Details Model
    """
    cache_build_details_id = models.AutoField(primary_key=True)
    package_name = models.ForeignKey(
        Package, on_delete=models.PROTECT,
        to_field='package_name', verbose_name="Package"
    )
    build_system = models.CharField(
        max_length=200, verbose_name="Build System"
    )
    build_tag = models.CharField(
        max_length=200, verbose_name="Build Tag"
    )
    build_details_json_str = models.TextField(null=True, blank=True)
    job_log_json_str = models.TextField(null=True, blank=True)

    @property
    def build_details_json(self):
        return self.str2json(self.build_details_json_str)

    @property
    def job_log_json(self):
        return self.str2json(self.job_log_json_str)

    class Meta:
        db_table = TABLE_PREFIX + 'cachebuilddetails'
dashboard/models.py
class Package(ModelMixin, models.Model):
    """
    Packages Model
    """
    package_id = models.AutoField(primary_key=True)
    package_name = models.CharField(max_length=1000, unique=True, verbose_name="Package Name")
    upstream_name = models.CharField(max_length=1000, null=True, blank=True,
                                     verbose_name="Upstream Name")
    downstream_name = models.CharField(max_length=1000, null=True, blank=True,
                                       verbose_name="Downstream Name")
    component = models.CharField(max_length=200, null=True, blank=True, verbose_name="Component")
    upstream_url = models.URLField(max_length=2000, unique=True, verbose_name="Upstream URL")
    upstream_l10n_url = models.URLField(max_length=2000, null=True, blank=True,
                                        verbose_name="Upstream Localization URL")
    platform_slug = models.ForeignKey(
        Platform, on_delete=models.PROTECT,
        to_field='platform_slug', verbose_name="Translation Platform"
    )
    platform_name = models.CharField(max_length=1000, null=True, blank=True,
                                     verbose_name="Package Name at Translation Platform")
    # translation platform project http url
    platform_url = models.URLField(max_length=500, null=True, blank=True,
                                   verbose_name="Translation Platform Project URL")
    products = ArrayField(
        models.CharField(max_length=400, blank=True),
        default=list, null=True, verbose_name="Release Streams"
    )
    package_details_json_str = models.TextField(null=True, blank=True)
    details_json_last_updated = models.DateTimeField(null=True)
    package_name_mapping_json_str = models.TextField(null=True, blank=True)
    name_map_last_updated = models.DateTimeField(null=True, blank=True)
    release_branch_mapping = models.TextField(null=True, blank=True)
    release_branch_map_last_updated = models.DateTimeField(null=True, blank=True)
    package_latest_builds = models.TextField(null=True, blank=True)
    package_latest_builds_last_updated = models.DateTimeField(null=True, blank=True)
    stats_diff = models.TextField(null=True, blank=True)
    stats_diff_last_updated = models.DateTimeField(null=True, blank=True)
    platform_last_updated = models.DateTimeField(null=True, blank=True)
    upstream_last_updated = models.DateTimeField(null=True, blank=True)
    downstream_last_updated = models.DateTimeField(null=True, blank=True)
    translation_file_ext = models.CharField(
        max_length=10, null=True, blank=True, default='po',
        verbose_name="Translation Format (po)"
    )
    created_by = models.EmailField(null=True)
    maintainers = models.TextField(null=True, blank=True)

    @property
    def package_details_json(self):
        return self.str2json(self.package_details_json_str)

    @property
    def package_name_mapping_json(self):
        return self.str2json(self.package_name_mapping_json_str)

    @property
    def release_branch_mapping_json(self):
        return self.str2json(self.release_branch_mapping)

    @property
    def package_latest_builds_json(self):
        return self.str2json(self.package_latest_builds)

    @property
    def release_branch_mapping_health(self):
        release_branch_mapping_dict = \
            self.str2json(self.release_branch_mapping)
        if not release_branch_mapping_dict:
            return False
        for release, mapping in release_branch_mapping_dict.items():
            if isinstance(mapping, dict):
                for k, v in mapping.items():
                    if not mapping.get(k):
                        return False
        return True

    @property
    def stats_diff_json(self):
        return self.str2json(self.stats_diff)

    @property
    def stats_diff_health(self):
        stats_diff_dict = self.str2json(self.stats_diff)
        if not stats_diff_dict:
            return True
        for release, diff in stats_diff_dict.items():
            if stats_diff_dict.get(release):
                return False
        return True

    @property
    def maintainers_json(self):
        return self.str2json(self.maintainers)

    def __str__(self):
        return self.package_name

    class Meta:
        db_table = TABLE_PREFIX + 'packages'
        verbose_name = "Package"
dashboard/constants.py
TS_JOB_TYPES = ('synctransplatform', 'syncrelschedule', 'syncupstream', 'syncdownstream',
                'syncbuildtags', 'stringchange', 'verifytrans', 'pushtrans', 'pulltrans',
                'dpushtrans')
dashboard/models.py
class PipelineConfig(ModelMixin, models.Model):
    """
    Pipeline Configurations Model
    """
    pipeline_config_id = models.AutoField(primary_key=True)
    ci_pipeline = models.ForeignKey(CIPipeline, on_delete=models.PROTECT,
                                    verbose_name="CI Pipeline", null=True)
    pipeline_config_event = models.CharField(max_length=1000)
    pipeline_config_active = models.BooleanField(default=False)
    pipeline_config_json_str = models.TextField(unique=True)
    pipeline_config_repo_branches = ArrayField(
        models.CharField(max_length=1000, blank=True), default=list,
        verbose_name="Repo Branches"
    )
    pipeline_config_created_on = models.DateTimeField(null=True)
    pipeline_config_updated_on = models.DateTimeField(null=True)
    pipeline_config_last_accessed = models.DateTimeField(null=True)
    pipeline_config_created_by = models.EmailField(null=True)

    @property
    def pipeline_config_json(self):
        return self.str2json(self.pipeline_config_json_str)

    @property
    def pipeline_config_yaml(self):
        return yaml.dump(
            self.str2json(self.pipeline_config_json_str),
            default_flow_style=False
        ).replace("\'", "")

    class Meta:
        db_table = TABLE_PREFIX + 'cipipelineconfig'
        verbose_name = "Pipeline Config"
dashboard/models.py
class Language(models.Model):
    """
    Language Model
    """
    locale_id = models.CharField(
        max_length=50, primary_key=True, verbose_name="Locale ID"
    )
    lang_name = models.CharField(
        max_length=400, unique=True, verbose_name="Language Name"
    )
    locale_alias = models.CharField(
        max_length=50, unique=True, null=True, blank=True, verbose_name="Locale Alias"
    )
    locale_script = models.CharField(
        max_length=100, null=True, blank=True, verbose_name="Locale Script"
    )
    lang_status = models.BooleanField(verbose_name="Enable/Disable")

    def __str__(self):
        return self.lang_name

    class Meta:
        db_table = TABLE_PREFIX + 'languages'
        verbose_name = "Language"
dashboard/constants.py
RELSTREAM_SLUGS = ('RHEL', 'fedora', 'RHV', 'satellite')
dashboard/managers/pipelines.py
class PipelineConfigManager(CIPipelineManager):
    """
    Pipeline Configurations Manager
    """

    def get_pipeline_configs(self, fields=None, ci_pipelines=None, pipeline_config_ids=None):
        """
        fetch ci pipeline(s) from db
        :return: queryset
        """
        pipeline_configs = None
        required_params = fields if fields and isinstance(fields, (list, tuple)) \
            else ('pipeline_config_id', 'ci_pipeline', 'pipeline_config_event', 'pipeline_config_repo_branches',
                  'pipeline_config_active', 'pipeline_config_created_on', 'pipeline_config_updated_on',
                  'pipeline_config_last_accessed', 'pipeline_config_created_by')
        kwargs = {}
        if ci_pipelines:
            kwargs.update(dict(ci_pipeline__in=ci_pipelines))
        if pipeline_config_ids:
            kwargs.update(dict(pipeline_config_id__in=pipeline_config_ids))
        try:
            pipeline_configs = PipelineConfig.objects.only(*required_params).filter(**kwargs).all()
        except Exception as e:
            self.app_logger(
                'ERROR', "Pipeline Configs could not be fetched, details: " + str(e)
            )
        return pipeline_configs

    def save_pipeline_config(self, config_params):
        """
        Save Pipeline Config in db
        :param config_params: dict
        :return: boolean
        """
        if not config_params:
            return
        try:
            pipeline_config = PipelineConfig(**config_params)
            pipeline_config.save()
        except Exception as e:
            self.app_logger(
                'ERROR', "Pipeline Config could not be saved, details: " + str(e)
            )
        else:
            return True
        return False

    def _pipeline_config_html_form(self, pipeline, action, tenant):
        """
        Pipeline Configuration HTML form fields
        :param pipeline: Pipeline Object
        :param action: str
        :param tenant: str or None
        :return: dict
        """

        repo_branches = self.package_manager.git_branches(
            package_name=pipeline.ci_package.package_name,
            repo_type=pipeline.ci_package.platform_slug.engine_name
        )

        if tenant == RELSTREAM_SLUGS[3]:
            repo_branches.insert(0, "%RESPECTIVE%")

        def _format_val(value_str):
            return "<strong>{}</strong>".format(str(value_str))

        def _format_choices(field_id, values):
            html_select = ""
            html_select += "<select name='{}' id='{}'>".format(field_id, field_id)
            for value in values:
                html_select += "<option value='{}'>{}</option>".format(value, value)
            html_select += "</select>"
            return html_select

        def _format_checkboxes(field_id, values):
            html_select = ""
            counter = 0
            for value in values:
                counter += 1
                html_select += "<label class='checkbox-inline'>" \
                               "<input id='{}' class='checkbox-inline' type='checkbox' value='{}' {}>" \
                               "{}</label>".format(field_id + str(counter), value, "checked", value)
            return html_select

        prepend_branch_field = "<input id='prependBranch' name='prependBranch' type='checkbox'>"
        if tenant == RELSTREAM_SLUGS[3]:
            prepend_branch_field = "<input id='prependBranch' name='prependBranch' type='checkbox' checked>"

        upload_update_field = "<input id='uploadUpdate' name='uploadUpdate' type='checkbox'>"
        if action == PIPELINE_CONFIG_EVENTS[2]:
            upload_update_field = "<input id='uploadUpdate' name='uploadUpdate' type='checkbox' checked>"

        key_val_map = {
            "ci_pipeline": _format_val(pipeline.ci_pipeline_uuid),
            "package": _format_val(pipeline.ci_package.package_name),
            "clone.type": _format_val(pipeline.ci_package.platform_slug.engine_name),
            "clone.branch": _format_choices("repoCloneBranch", repo_branches),
            "clone.recursive": "<input type='checkbox' id='cloneRecursive' name='cloneRecursive'>",
            "filter.domain": "<input id='filterDomain' type='text' value='{}'>".format(
                pipeline.ci_package.package_name),
            "filter.dir": "<input id='filterDir' type='text' value=''>",
            "download.target_langs": _format_checkboxes(
                'downloadTargetLangs', pipeline.ci_project_details_json.get("targetLangs", [])),
            "download.type": _format_val(pipeline.ci_package.platform_slug.engine_name),
            "download.branch": _format_choices("downloadRepoBranch", repo_branches),
            "download.workflow_step": _format_choices(
                "workflowStep", self.get_ci_platform_workflow_steps(pipeline.ci_pipeline_uuid)
            ),
            "download.prepend_branch": prepend_branch_field,
            "upload.type": _format_val(pipeline.ci_package.platform_slug.engine_name),
            "upload.branch": _format_choices("uploadRepoBranch", repo_branches),
            "upload.target_langs": _format_checkboxes(
                'uploadTargetLangs', pipeline.ci_project_details_json.get("targetLangs", [])),
            "upload.import_settings": "<input id='importSettings' type='text' value='project'>",
            "upload.update": upload_update_field,
            "upload.prepend_branch": prepend_branch_field,
        }
        return key_val_map

    @staticmethod
    def __true_false_type(str_val):
        type_conv_map = {
            "true": True,
            "True": True,
            "false": False,
            "False": False
        }
        return type_conv_map.get(str_val)

    def _pipeline_config_values(self, config_values):
        """
        Pipeline Configuration YAML values
        :param config_values: dict
        :return: dict
        """

        key_val_map = {
            "ci_pipeline": config_values.get('ciPipeline', ''),
            "package": config_values.get('package', ''),
            "clone.type": config_values.get('cloneType', ''),
            "clone.branch": config_values.get('cloneBranch', ''),
            "clone.recursive": self.__true_false_type(config_values.get('cloneRecursive', '')),
            "filter.domain": config_values.get('filterDomain', ''),
            "filter.dir": config_values.get('filterDir', ''),
            "download.target_langs": config_values.get('downloadTargetLangs', '').split(','),
            "download.type": config_values.get('downloadType', ''),
            "download.branch": config_values.get('downloadBranch', ''),
            "download.workflow_step": config_values.get('downloadWorkflowStep', ''),
            "download.prepend_branch": self.__true_false_type(config_values.get('downloadPrependBranch', '')),
            "upload.type": config_values.get('uploadType', ''),
            "upload.branch": config_values.get('uploadBranch', ''),
            "upload.target_langs": config_values.get('uploadTargetLangs', '').split(','),
            "upload.import_settings": config_values.get('uploadImportSettings', ''),
            "upload.update": self.__true_false_type(config_values.get('uploadUpdate', '')),
            "upload.prepend_branch": self.__true_false_type(config_values.get('uploadPrependBranch', '')),
        }
        return key_val_map

    @staticmethod
    def get_job_action_template(pipeline, action):
        job_action_map = {
            PIPELINE_CONFIG_EVENTS[0]: pipeline.ci_push_job_template,
            PIPELINE_CONFIG_EVENTS[1]: pipeline.ci_pull_job_template,
            PIPELINE_CONFIG_EVENTS[2]: pipeline.ci_push_job_template
        }
        return job_action_map.get(action)

    def format_pipeline_config(self, pipeline, action, output_format=None, tenant=None, **kwargs):
        """
        Formats Job Template for the Pipeline Configurations
        :param pipeline: Pipeline Object
        :param action: str
        :param output_format: str
        :param tenant: str or None
        :return: dict
        """
        key_val_map = dict()
        if not pipeline and not action:
            return key_val_map

        respective_job_template_json = self.get_job_action_template(pipeline, action).job_template_json or dict()
        pipeline_config = respective_job_template_json.copy()

        if output_format and output_format == 'html_form':
            key_val_map = self._pipeline_config_html_form(
                pipeline=pipeline, action=action, tenant=tenant
            )
        if output_format and output_format == 'values':
            key_val_map = self._pipeline_config_values(config_values=kwargs)

        def _traverse_steps(task_steps):
            counter = 0
            for t_step in task_steps:
                for step_cmd, step_params in t_step.items():
                    inner_counter = 0
                    for step_param in step_params:
                        for param, value in step_param.items():
                            niddle = "{}.{}".format(step_cmd, param)
                            if niddle in key_val_map:
                                task_steps[counter][step_cmd][inner_counter][param] = \
                                    key_val_map[niddle]
                        inner_counter = inner_counter + 1
                counter = counter + 1
            return task_steps

        # lets parse job_template and fill values
        for k, v in pipeline_config.items():
            if isinstance(v, dict):
                for p, q in v.items():
                    if p in key_val_map:
                        pipeline_config[k][p] = key_val_map[p]
                    if p == "tasks":
                        pipeline_config[k][p] = _traverse_steps(q)
        return pipeline_config

    def process_save_pipeline_config(self, *args, **kwargs):
        """
        Process data and save pipeline configuration
        :param args: list
        :param args: dict
        :return: boolean
        """
        repo_type, repo_branch, pipeline_branches, target_langs, u_email = args

        copy_config = kwargs.get('chkCopyConfig')
        pipeline_uuid = kwargs.get('ciPipeline', '')
        pipeline_action = kwargs.get('pipelineAction', '')
        ci_pipeline = self.get_ci_pipelines(uuids=[pipeline_uuid]).get()

        def _save_pipeline_config(*params):
            pc_kwargs = dict()
            pc_kwargs.update(dict(ci_pipeline=params[0]))
            pc_kwargs.update(dict(pipeline_config_event=params[1]))
            pc_kwargs.update(dict(pipeline_config_active=True))
            pc_kwargs.update(dict(pipeline_config_json_str=params[2]))
            pc_kwargs.update(dict(pipeline_config_repo_branches=params[3]))
            pc_kwargs.update(dict(pipeline_config_created_on=timezone.now()))
            pc_kwargs.update(dict(pipeline_config_created_by=params[4]))
            return self.save_pipeline_config(pc_kwargs)

        if not self.__true_false_type(copy_config):
            pipeline_config = self.format_pipeline_config(pipeline=ci_pipeline, action=pipeline_action,
                                                          output_format='values', **kwargs)
            return _save_pipeline_config(
                ci_pipeline, pipeline_action, json.dumps(pipeline_config), pipeline_branches, u_email
            )
        else:
            save_results = []
            for action in PIPELINE_CONFIG_EVENTS:
                kwargs['downloadType'] = repo_type
                kwargs['downloadBranch'] = repo_branch
                kwargs['downloadTargetLangs'] = target_langs
                kwargs['uploadType'] = repo_type
                kwargs['uploadBranch'] = repo_branch
                kwargs['uploadTargetLangs'] = target_langs

                if action == PIPELINE_CONFIG_EVENTS[1]:
                    workflow_steps = self.get_ci_platform_workflow_steps(
                        pipeline_uuid=ci_pipeline.ci_pipeline_uuid
                    )
                    kwargs['downloadWorkflowStep'] = workflow_steps[0] \
                        if isinstance(workflow_steps, list) and len(workflow_steps) >= 1 else 'Translation'
                if action == PIPELINE_CONFIG_EVENTS[2]:
                    kwargs['uploadUpdate'] = 'true'

                pipeline_config = self.format_pipeline_config(pipeline=ci_pipeline, action=action,
                                                              output_format='values', **kwargs)
                save_results.append(_save_pipeline_config(
                    ci_pipeline, action, json.dumps(pipeline_config), pipeline_branches, u_email
                ))
            return functools.reduce(lambda a, b: a and b, save_results) if save_results else False

    def toggle_pipeline_configuration(self, pipeline_config_id):
        """
        Deactivate or Activate Pipeline Configuration
        :param pipeline_config_id: number
        :return: boolean
        """
        if not pipeline_config_id:
            return False
        filter_kwargs = {}
        filter_kwargs.update(dict(pipeline_config_id=pipeline_config_id))
        try:
            PipelineConfig.objects.filter(**filter_kwargs).update(
                pipeline_config_active=Case(
                    When(pipeline_config_active=True, then=Value(False)),
                    When(pipeline_config_active=False, then=Value(True)),
                    default=Value(False)
                )
            )
        except Exception as e:
            self.app_logger(
                'ERROR', "Pipeline Configuration could not be toggled, details: " + str(e)
            )
        else:
            return True
        return False

    def delete_pipeline_configuration(self, pipeline_config_id):
        """
        Delete Pipeline Configuration
        :param pipeline_config_id: number
        :return: boolean
        """
        if not pipeline_config_id:
            return False
        filter_kwargs = {}
        filter_kwargs.update(dict(pipeline_config_id=pipeline_config_id))
        try:
            PipelineConfig.objects.filter(**filter_kwargs).delete()
        except Exception as e:
            self.app_logger(
                'ERROR', "Pipeline Configuration could not be deleted, details: " + str(e)
            )
        else:
            return True
        return False
dashboard/constants.py
TRANSPLATFORM_ENGINES = ('damnedlies', 'transifex', 'zanata', 'weblate', 'memsource')
dashboard/constants.py
TS_CI_JOBS = (TS_JOB_TYPES[7], TS_JOB_TYPES[8], TS_JOB_TYPES[9])
dashboard/managers/pipelines.py
class CIPipelineManager(BaseManager):
    """
    Continuous Integration Pipeline Manager
    """

    package_manager = PackagesManager()

    def get_ci_pipelines(self, fields=None, packages=None, platforms=None,
                         releases=None, uuids=None, pipeline_ids=None):
        """
        fetch ci pipeline(s) from db
        :return: queryset
        """
        ci_pipelines = None
        required_params = fields if fields and isinstance(fields, (list, tuple)) \
            else ('ci_pipeline_uuid', 'ci_package', 'ci_platform', 'ci_release',
                  'ci_push_job_template', 'ci_pull_job_template', 'ci_project_web_url')
        kwargs = {}
        kwargs.update(dict(ci_pipeline_visibility=True))
        if packages:
            kwargs.update(dict(ci_package__in=packages))
        if platforms:
            kwargs.update(dict(ci_platform__in=platforms))
        if releases:
            kwargs.update(dict(ci_release__in=releases))
        if uuids:
            kwargs.update(dict(ci_pipeline_uuid__in=uuids))
        if pipeline_ids:
            kwargs.update(dict(ci_pipeline_id__in=pipeline_ids))

        try:
            ci_pipelines = CIPipeline.objects.only(*required_params).filter(**kwargs).all()
        except Exception as e:
            self.app_logger(
                'ERROR', "CI Pipelines could not be fetched, details: " + str(e)
            )
        return ci_pipelines

    def ci_platform_project_details(self, ci_platform, project_url):
        """
        Return CI Platform Project Details
            Assuming either project slug/uuid forms the last part of project_url
            Or, project_url is the project slug/uuid
        :param ci_platform: object
        :param project_url: str
        :return: json
        """
        if not ci_platform and not project_url:
            return '', {}
        project_uuid = project_url
        if ci_platform.engine_name == TRANSPLATFORM_ENGINES[4] and "/" in project_url:
            project_uuid = project_url.split("/")[-1:][0]
        response_dict = dict()
        kwargs = dict()
        kwargs.update(dict(no_cache_api=True))
        response_dict = self.api_resources.fetch_project_details(
            ci_platform.engine_name, ci_platform.api_url, project_uuid, **kwargs
        ) or response_dict
        return project_uuid, response_dict

    def refresh_ci_pipeline(self, pipeline_id, toggle_visibility=True):
        """
        Refresh a CI Pipeline
        :param pipeline_id: int
        :param toggle_visibility: boolean
        :return: boolean
        """
        ci_pipeline = self.get_ci_pipelines(pipeline_ids=[pipeline_id])
        if ci_pipeline:
            ci_pipeline = ci_pipeline.get()
            _, resp_dict = self.ci_platform_project_details(
                ci_pipeline.ci_platform, ci_pipeline.ci_project_web_url
            )
            if resp_dict:
                if 'errorCode' in resp_dict and 'errorDescription' in resp_dict:
                    project_uid = ci_pipeline.ci_project_web_url.split("/")[-1:][0]
                    if project_uid in resp_dict.get('errorDescription') and \
                            'not found' in resp_dict.get('errorDescription'):
                        if toggle_visibility and self.toggle_visibility(pipeline_id):
                            return True
                else:
                    pipeline_params = dict()
                    pipeline_params['ci_project_web_url'] = ci_pipeline.ci_project_web_url
                    pipeline_params['ci_project_details_json_str'] = json.dumps(resp_dict)
                    pipeline_params['ci_platform_jobs_json_str'] = json.dumps(
                        resp_dict.get('project_jobs', {})
                    )
                    pipeline_params['ci_pipeline_last_updated'] = timezone.now()
                    if self.save_ci_pipeline(pipeline_params):
                        return True
        return False

    def refresh_pkg_pipelines(self, package_name):
        """
        Refresh CI Pipelines that belong to a package
        :param package_name: str
        """
        pipelines = self.get_ci_pipelines(
            packages=self.package_manager.get_packages(pkgs=[package_name])
        )
        for pipeline in pipelines or []:
            self.refresh_ci_pipeline(pipeline_id=pipeline.ci_pipeline_id,
                                     toggle_visibility=False)

    def save_ci_pipeline(self, ci_pipeline):
        """
        Save CI Pipeline in db
        :param ci_pipeline: dict
        :return: boolean
        """
        if not ci_pipeline:
            return

        match_params = {}
        match_params.update(dict(
            ci_project_web_url=ci_pipeline.get('ci_project_web_url'))
        )

        try:
            ci_pipeline['ci_pipeline_visibility'] = True
            CIPipeline.objects.update_or_create(
                **match_params, defaults=ci_pipeline
            )
        except Exception as e:
            self.app_logger(
                'ERROR', "CI Pipeline could not be saved, details: " + str(e)
            )
        else:
            return True
        return False

    def toggle_visibility(self, pipeline_id):
        """
        Toggle CI Pipeline Visibility
        :param pipeline_id: int
        :return: boolean
        """
        filter_kwargs = {}
        filter_kwargs.update(dict(ci_pipeline_id=pipeline_id))
        try:
            CIPipeline.objects.filter(**filter_kwargs).update(
                ci_pipeline_visibility=Case(
                    When(ci_pipeline_visibility=True, then=Value(False)),
                    When(ci_pipeline_visibility=False, then=Value(True)),
                    default=Value(True)
                )
            )
        except Exception as e:
            self.app_logger(
                'ERROR', "CI Pipeline visibility could not be toggled, details: " + str(e)
            )
        else:
            return True
        return False

    def get_ci_platform_jobs(self, ci_pipelines=None):
        """
        fetch ci pipeline(s) from db
        :return: queryset
        """
        ci_platform_jobs = None
        kwargs = {}
        kwargs.update(dict(ci_platform_job_visibility=True))
        if ci_pipelines:
            kwargs.update(dict(ci_pipeline__in=ci_pipelines))

        try:
            ci_platform_jobs = CIPlatformJob.objects.filter(**kwargs).all()
        except Exception as e:
            self.app_logger(
                'ERROR', "CI Pipelines could not be fetched, details: " + str(e)
            )
        return ci_platform_jobs

    def save_ci_platform_job(self, platform_job):
        """
        Save CI Platform Job in db
        :param platform_job: dict
        :return: boolean
        """
        if not platform_job:
            return
        try:
            ci_platform_job = CIPlatformJob(**platform_job)
            ci_platform_job.save()
        except Exception as e:
            self.app_logger(
                'ERROR', "CI Platform Job could not be saved, details: " + str(e)
            )
        else:
            return True
        return False

    def ci_lang_job_map(self, pipelines, workflow_step='default'):
        """
        Get CI Platform Target Language: Job UID map
        :param pipelines: Query Objects
        :param workflow_step: Str
        :return: dict
        """
        ci_lang_job_map = dict()
        if not pipelines:
            return ci_lang_job_map
        for pipeline in pipelines:
            if pipeline.ci_platform_jobs_json:
                ci_lang_job_map[pipeline.ci_pipeline_uuid] = dict()
                for job_detail in pipeline.ci_platform_jobs_json:
                    if job_detail.get('targetLang') and job_detail.get('uid') and job_detail.get('filename'):

                        if not job_detail.get('workflowStep'):
                            ci_lang_job_map[pipeline.ci_pipeline_uuid].update(
                                {job_detail['uid']: (job_detail['targetLang'], job_detail['filename'])}
                            )
                        elif workflow_step == 'default' and job_detail.get('workflowLevel') == 1:
                            ci_lang_job_map[pipeline.ci_pipeline_uuid].update(
                                {job_detail['uid']: (job_detail['targetLang'], job_detail['filename'])}
                            )
                        elif job_detail.get('workflowStep') and \
                                job_detail.get('workflowStep').get('name') == workflow_step:
                            ci_lang_job_map[pipeline.ci_pipeline_uuid].update(
                                {job_detail['uid']: (job_detail['targetLang'], job_detail['filename'])}
                            )

        return ci_lang_job_map

    def get_ci_platform_workflow_steps(self, pipeline_uuid):
        """
        Get CI Platform Workflow Step
        :param pipeline_uuid: str
        :return: list
        """

        def unique(sequence):
            seen = set()
            return [x for x in sequence if not (x in seen or seen.add(x))]

        workflow_steps = []
        if not pipeline_uuid:
            return workflow_steps
        ci_pipeline = self.get_ci_pipelines(uuids=[pipeline_uuid])
        if not ci_pipeline:
            return workflow_steps
        ci_pipeline = ci_pipeline.get()
        return unique([p_job['workflowStep'].get('name', 'default') if p_job.get('workflowStep')
                       else 'default' for p_job in ci_pipeline.ci_platform_jobs_json])


Question: import os
import ast
import csv
import json
import shutil
import threading
from datetime import datetime
from pathlib import Path
from django.contrib import messages
from django.contrib.messages.views import SuccessMessageMixin
from django.core.exceptions import PermissionDenied
from django.forms.utils import ErrorList
from django.http import (
    HttpResponse, HttpResponseRedirect, JsonResponse, Http404
)
from django.shortcuts import render
from django.template import Context, Template
from django.views.generic import (
    TemplateView, ListView, FormView, DetailView
)
from django.views.generic.edit import (CreateView, UpdateView, DeleteView)
from django.urls import reverse, reverse_lazy
from dashboard.constants import (
    TS_JOB_TYPES, TRANSPLATFORM_ENGINES, RELSTREAM_SLUGS,
    WEBLATE_SLUGS, TRANSIFEX_SLUGS, TS_CI_JOBS, PIPELINE_CONFIG_EVENTS
)
from dashboard.forms import (
    NewPackageForm, UpdatePackageForm, NewReleaseBranchForm, NewGraphRuleForm,
    NewLanguageForm, UpdateLanguageForm, LanguageSetForm, NewCIPipelineForm,
    NewTransPlatformForm, UpdateTransPlatformForm, UpdateGraphRuleForm,
    CreateCIPipelineForm
)
from dashboard.managers.inventory import (
    InventoryManager, ReleaseBranchManager, SyncStatsManager
)
from dashboard.managers.packages import PackagesManager
from dashboard.managers.pipelines import (
    CIPipelineManager, PipelineConfigManager
)
from dashboard.managers.jobs import (
    YMLBasedJobManager, JobsLogManager, TransplatformSyncManager,
    ReleaseScheduleSyncManager, BuildTagsSyncManager, JobTemplateManager
)
from dashboard.managers.graphs import (
    GraphManager, ReportsManager, GeoLocationManager
)
from dashboard.models import (
    Job, Language, LanguageSet, Platform, Visitor, Package,
    GraphRule, SyncStats, CacheBuildDetails, CIPipeline, PipelineConfig
)
# Copyright 2016 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.


# django

# dashboard



class ManagersMixin(object):
    """
    Managers Mixin
    """
    inventory_manager = InventoryManager()
    packages_manager = PackagesManager()
    jobs_log_manager = JobsLogManager()
    jobs_template_manager = JobTemplateManager()
    release_branch_manager = ReleaseBranchManager()
    graph_manager = GraphManager()
    reports_manager = ReportsManager()
    geo_location_manager = GeoLocationManager()

Answer:     ci_pipeline_manager = CIPipelineManager()