#!/usr/bin/python
# copyright (c) 2014 wladimir j. van der laan
# distributed under the mit software license, see the accompanying
# file copying or http://www.opensource.org/licenses/mit-license.php.
'''
run this script from the root of the repository to update all translations from
transifex.
it will do the following automatically:

- fetch all translations using the tx tool
- post-process them into valid and committable format
  - remove invalid control characters
  - remove location tags (makes diffs less noisy)

todo:
- auto-add new translations to the build system according to the translation process
'''
from __future__ import division, print_function
import subprocess
import re
import sys
import os
import io
import xml.etree.elementtree as et

# name of transifex tool
tx = 'tx'
# name of source language file
source_lang = 'moorecoin_en.ts'
# directory with locale files
locale_dir = 'src/qt/locale'

def check_at_repository_root():
    if not os.path.exists('.git'):
        print('no .git directory found')
        print('execute this script at the root of the repository', file=sys.stderr)
        exit(1)

def fetch_all_translations():
    if subprocess.call([tx, 'pull', '-f']):
        print('error while fetching translations', file=sys.stderr)
        exit(1)

def find_format_specifiers(s):
    '''find all format specifiers in a string.'''
    pos = 0
    specifiers = []
    while true:
        percent = s.find('%', pos)
        if percent < 0:
            break
        specifiers.append(s[percent+1])
        pos = percent+2
    return specifiers

def split_format_specifiers(specifiers):
    '''split format specifiers between numeric (qt) and others (strprintf)'''
    numeric = []
    other = []
    for s in specifiers:
        if s in {'1','2','3','4','5','6','7','8','9'}:
            numeric.append(s)
        else:
            other.append(s)

    # numeric (qt) can be present in any order, others (strprintf) must be in specified order
    return set(numeric),other

def sanitize_string(s):
    '''sanitize string for printing'''
    return s.replace('\n',' ')

def check_format_specifiers(source, translation, errors):
    source_f = split_format_specifiers(find_format_specifiers(source))
    # assert that no source messages contain both qt and strprintf format specifiers
    # if this fails, go change the source as this is hacky and confusing!
    assert(not(source_f[0] and source_f[1]))
    try:
        translation_f = split_format_specifiers(find_format_specifiers(translation))
    except indexerror:
        errors.append("parse error in translation '%s'" % sanitize_string(translation))
        return false
    else:
        if source_f != translation_f:
            errors.append("mismatch between '%s' and '%s'" % (sanitize_string(source), sanitize_string(translation)))
            return false
    return true

def all_ts_files(suffix=''):
    for filename in os.listdir(locale_dir):
        # process only language files, and do not process source language
        if not filename.endswith('.ts'+suffix) or filename == source_lang+suffix:
            continue
        if suffix: # remove provided suffix
            filename = filename[0:-len(suffix)]
        filepath = os.path.join(locale_dir, filename)
        yield(filename, filepath)

fix_re = re.compile(b'[\x00-\x09\x0b\x0c\x0e-\x1f]')
def remove_invalid_characters(s):
    '''remove invalid characters from translation string'''
    return fix_re.sub(b'', s)

# override cdata escape function to make our output match qt's (optional, just for cleaner diffs for
# comparison, disable by default)
_orig_escape_cdata = none
def escape_cdata(text):
    text = _orig_escape_cdata(text)
    text = text.replace("'", '&apos;')
    text = text.replace('"', '&quot;')
    return text

def postprocess_translations(reduce_diff_hacks=false):
    print('checking and postprocessing...')

    if reduce_diff_hacks:
        global _orig_escape_cdata
        _orig_escape_cdata = et._escape_cdata
        et._escape_cdata = escape_cdata

    for (filename,filepath) in all_ts_files():
        os.rename(filepath, filepath+'.orig')

    have_errors = false
    for (filename,filepath) in all_ts_files('.orig'):
        # pre-fixups to cope with transifex output
        parser = et.xmlparser(encoding='utf-8') # need to override encoding because 'utf8' is not understood only 'utf-8'
        with open(filepath + '.orig', 'rb') as f:
            data = f.read()
        # remove control characters; this must be done over the entire file otherwise the xml parser will fail
        data = remove_invalid_characters(data)
        tree = et.parse(io.bytesio(data), parser=parser)

        # iterate over all messages in file
        root = tree.getroot()
        for context in root.findall('context'):
            for message in context.findall('message'):
                numerus = message.get('numerus') == 'yes'
                source = message.find('source').text
                translation_node = message.find('translation')
                # pick all numerusforms
                if numerus:
                    translations = [i.text for i in translation_node.findall('numerusform')]
                else:
                    translations = [translation_node.text]

                for translation in translations:
                    if translation is none:
                        continue
                    errors = []
                    valid = check_format_specifiers(source, translation, errors)

                    for error in errors:
                        print('%s: %s' % (filename, error))

                    if not valid: # set type to unfinished and clear string if invalid
                        translation_node.clear()
                        translation_node.set('type', 'unfinished')
                        have_errors = true

                # remove location tags
                for location in message.findall('location'):
                    message.remove(location)

                # remove entire message if it is an unfinished translation
                if translation_node.get('type') == 'unfinished':
                    context.remove(message)

        # write fixed-up tree
        # if diff reduction requested, replace some xml to 'sanitize' to qt formatting
        if reduce_diff_hacks:
            out = io.bytesio()
            tree.write(out, encoding='utf-8')
            out = out.getvalue()
            out = out.replace(b' />', b'/>')
            with open(filepath, 'wb') as f:
                f.write(out)
        else:
            tree.write(filepath, encoding='utf-8')
    return have_errors

if __name__ == '__main__':
    check_at_repository_root()
    fetch_all_translations()
    postprocess_translations()

