# Buggy code
def fix_all_git_symlinked(topdir):
    """inplace conversion of git symlinks to real content

    Most (all?) of git implementations in windows store a symlink pointing
    into the repo as a text file, the text being the relative path to the
    file with the real content.

    So, in a clone of nikola in windows the symlinked files will have the
    wrong content; a .zip download from Github has the same problem.

    This function will rewrite each symlinked file with the correct contents, but
    keep in mind that the working copy will be seen as dirty by git after operation.

    Expects to find a list of symlinked files at nikola/data/symlinked.txt

    The list can be generated by scripts/generate_symlinked_list.sh , which is
    basically a redirect of
         cd nikola_checkout
         git ls-files -s | awk '/120000/{print $4}'

    Weakness: if interrupted of fail amidst a directory copy, next run will not
    see the missing files.
    """
    with open(topdir + r"\nikola\data\symlinked.txt", "rb") as f:
        all_bytes = f.read()
        text = all_bytes.decode("utf8")
    # expect each line a relpath from git or zip root,
    # smoke test relpaths are relative to git root
    if text.startswith("."):
        raise Exception(r"Bad data in \nikola\data\symlinked.txt")
    relnames = text.split("\n")
    relnames = [name.strip().replace("/", "\\") for name in relnames]
    relnames = [name for name in relnames if name]

    failures = 0
    for name in relnames:
        # build dst path and do some basic validation
        dst = os.path.join(topdir, name)
        # don't access files outside topdir
        if not is_file_into_dir(dst, topdir):
            continue
        if os.path.isdir(dst):
            # assume the file was de-symlinked
            continue

        # build src path and do some basic validation
        with open(os.path.join(topdir, dst), "r") as f:
            text = f.read()
        dst_dir = os.path.dirname(dst)
        try:
            src = os.path.normpath(os.path.join(dst_dir, text))
            if not os.path.exists(src):
                # assume the file was de-symlinked before
                continue
            # don't access files outside topdir
            if not is_file_into_dir(src, topdir):
                continue
        except Exception:
            # assume the file was de-symlinked before
            continue

        # copy src to dst
        try:
            if os.path.isdir(src):
                os.unlink(dst)
                shutil.copytree(src, dst)
            else:
                shutil.copy2(src, dst)
        except Exception:
            failures += 1
            print("*** copy failed for")
            print("\t src:", src)
            print("\t dst:", dst)

    return failures


# Traceback (UnicodeDecodeError)
[nikola] C:\Users\leo\Documents\dev>pip install nikola
Downloading/unpacking nikola
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\nik
ola\setup.py) egg_info for package nikola

Downloading/unpacking doit>=0.23.0 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\doi
t\setup.py) egg_info for package doit

Requirement already satisfied (use --upgrade to upgrade): Pygments>=1.6 in c:\us
ers\leo\anaconda3\lib\site-packages (from nikola)
Downloading/unpacking Pillow>=2.4.0 (from nikola)
Downloading/unpacking python-dateutil>=2.2 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\pyt
hon-dateutil\setup.py) egg_info for package python-dateutil

Downloading/unpacking docutils>=0.12 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\doc
utils\setup.py) egg_info for package docutils

warning: no files found matching 'MANIFEST'
warning: no files found matching '*' under directory 'extras'
warning: no previously-included files matching '.cvsignore' found under dire
ctory '*'
warning: no previously-included files matching '*.pyc' found under directory
'*'
warning: no previously-included files matching '*~' found under directory '*
'
warning: no previously-included files matching '.DS_Store' found under direc
tory '*'
Downloading/unpacking mako>=1.0.0 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\mak
o\setup.py) egg_info for package mako

warning: no files found matching '*.xml' under directory 'examples'
warning: no files found matching '*.mako' under directory 'examples'
warning: no files found matching 'ez_setup.py'
no previously-included directories found matching 'doc\build\output'
Downloading/unpacking unidecode>=0.04.16 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\uni
decode\setup.py) egg_info for package unidecode

Requirement already satisfied (use --upgrade to upgrade): lxml>=3.3.5 in c:\user
s\leo\anaconda3\lib\site-packages (from nikola)
Downloading/unpacking yapsy>=1.10.423 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\yap
sy\setup.py) egg_info for package yapsy

no previously-included directories found matching 'src2\package\doc\_build'
no previously-included directories found matching 'src3\package\doc\_build'
Downloading/unpacking PyRSS2Gen>=1.1 (from nikola)
Downloading PyRSS2Gen-1.1.tar.gz
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\PyR
SS2Gen\setup.py) egg_info for package PyRSS2Gen

Downloading/unpacking logbook>=0.7.0 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\log
book\setup.py) egg_info for package logbook

warning: no files found matching 'logbook\_speedups.c'
Downloading/unpacking blinker>=1.3 (from nikola)
Running setup.py (path:C:\Users\leo\AppData\Local\Temp\pip_build_leo\bli
nker\setup.py) egg_info for package blinker

Requirement already satisfied (use --upgrade to upgrade): setuptools>=5.4.1 in c
:\users\leo\anaconda3\lib\site-packages\setuptools-5.7-py3.4.egg (from nikola
)
Downloading/unpacking natsort>=3.3.0 (from nikola)
Downloading natsort-3.5.0-py2.py3-none-any.whl
Requirement already satisfied (use --upgrade to upgrade): six in c:\users\leo
\anaconda3\lib\site-packages (from doit>=0.23.0->nikola)
Requirement already satisfied (use --upgrade to upgrade): MarkupSafe>=0.9.2 in c
:\users\leo\anaconda3\lib\site-packages (from mako>=1.0.0->nikola)
Installing collected packages: nikola, doit, Pillow, python-dateutil, docutils,
mako, unidecode, yapsy, PyRSS2Gen, logbook, blinker, natsort
Running setup.py install for nikola
Traceback (most recent call last):
File "<string>", line 1, in <module>
File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\setup.py"
, line 199, in <module>
'nikola = nikola.__main__:main'
File "C:\Users\leo\anaconda3\lib\distutils\core.py", line 148, in setup

dist.run_commands()
File "C:\Users\leo\anaconda3\lib\distutils\dist.py", line 955, in run_c
ommands
self.run_command(cmd)
File "C:\Users\leo\anaconda3\lib\distutils\dist.py", line 974, in run_c
ommand
cmd_obj.run()
File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\setup.py"
, line 141, in run
expands_symlinks_for_windows()
File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\setup.py"
, line 95, in expands_symlinks_for_windows
failures = winutils.fix_all_git_symlinked(localdir)
File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\nikola\wi
nutils.py", line 90, in fix_all_git_symlinked
text = f.read()
File "C:\Users\leo\anaconda3\lib\encodings\cp1252.py", line 23, in deco
de
return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 26224
: character maps to <undefined>
Complete output from command C:\Users\leo\anaconda3\python.exe -c "import
setuptools, tokenize;__file__='C:\\Users\\leo\\AppData\\Local\\Temp\\pip_bui
ld_leo\\nikola\\setup.py';exec(compile(getattr(tokenize, 'open', open)(__file
__).read().replace('\r\n', '\n'), __file__, 'exec'))" install --record C:\Users\
leo\AppData\Local\Temp\pip-5jphpkn9-record\install-record.txt --single-versio
n-externally-managed --compile:
running install

Traceback (most recent call last):

File "<string>", line 1, in <module>

File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\setup.py", li
ne 199, in <module>

'nikola = nikola.__main__:main'

File "C:\Users\leo\anaconda3\lib\distutils\core.py", line 148, in setup

dist.run_commands()

File "C:\Users\leo\anaconda3\lib\distutils\dist.py", line 955, in run_comma
nds

self.run_command(cmd)

File "C:\Users\leo\anaconda3\lib\distutils\dist.py", line 974, in run_comma
nd

cmd_obj.run()

File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\setup.py", li
ne 141, in run

expands_symlinks_for_windows()

File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\setup.py", li
ne 95, in expands_symlinks_for_windows

failures = winutils.fix_all_git_symlinked(localdir)

File "C:\Users\leo\AppData\Local\Temp\pip_build_leo\nikola\nikola\winuti
ls.py", line 90, in fix_all_git_symlinked

text = f.read()

File "C:\Users\leo\anaconda3\lib\encodings\cp1252.py", line 23, in decode

return codecs.charmap_decode(input,self.errors,decoding_table)[0]

UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 26224: ch
aracter maps to <undefined>

----------------------------------------
Cleaning up...
Command C:\Users\leo\anaconda3\python.exe -c "import setuptools, tokenize;__f
ile__='C:\\Users\\leo\\AppData\\Local\\Temp\\pip_build_leo\\nikola\\setup.
py';exec(compile(getattr(tokenize, 'open', open)(__file__).read().replace('\r\n'
, '\n'), __file__, 'exec'))" install --record C:\Users\leo\AppData\Local\Temp
\pip-5jphpkn9-record\install-record.txt --single-version-externally-managed --co
mpile failed with error code 1 in C:\Users\leo\AppData\Local\Temp\pip_build_s
tefan\nikola
Storing debug log for failure in C:\Users\leo\pip\pip.log

[nikola] C:\Users\leo\Documents\dev>

# Fixed code
def fix_all_git_symlinked(topdir):
    """inplace conversion of git symlinks to real content

    Most (all?) of git implementations in windows store a symlink pointing
    into the repo as a text file, the text being the relative path to the
    file with the real content.

    So, in a clone of nikola in windows the symlinked files will have the
    wrong content; a .zip download from Github has the same problem.

    This function will rewrite each symlinked file with the correct contents, but
    keep in mind that the working copy will be seen as dirty by git after operation.

    Expects to find a list of symlinked files at nikola/data/symlinked.txt

    The list can be generated by scripts/generate_symlinked_list.sh , which is
    basically a redirect of
         cd nikola_checkout
         git ls-files -s | awk '/120000/{print $4}'

    Weakness: if interrupted of fail amidst a directory copy, next run will not
    see the missing files.
    """
    # Determine whether or not symlinks need fixing (they don’t if installing
    # from a .tar.gz file)
    with io.open(
        topdir + r"\nikola\data\symlink-test-link.txt", "r", encoding="utf-8"
    ) as f:
        text = f.read()
        if text.startswith("NIKOLA_SYMLINKS=OK"):
            return -1
    with io.open(topdir + r"\nikola\data\symlinked.txt", "r", encoding="utf-8") as f:
        text = f.read()
    # expect each line a relpath from git or zip root,
    # smoke test relpaths are relative to git root
    if text.startswith("."):
        raise Exception(r"Bad data in \nikola\data\symlinked.txt")
    relnames = text.split("\n")
    relnames = [name.strip().replace("/", "\\") for name in relnames]
    relnames = [name for name in relnames if name]

    failures = 0
    for name in relnames:
        # build dst path and do some basic validation
        dst = os.path.join(topdir, name)
        # don't access files outside topdir
        if not is_file_into_dir(dst, topdir):
            continue
        if os.path.isdir(dst):
            # assume the file was de-symlinked
            continue

        # build src path and do some basic validation
        with io.open(os.path.join(topdir, dst), "r", encoding="utf-8") as f:
            text = f.read()
        dst_dir = os.path.dirname(dst)
        try:
            src = os.path.normpath(os.path.join(dst_dir, text))
            if not os.path.exists(src):
                # assume the file was de-symlinked before
                continue
            # don't access files outside topdir
            if not is_file_into_dir(src, topdir):
                continue
        except Exception:
            # assume the file was de-symlinked before
            continue

        # copy src to dst
        try:
            if os.path.isdir(src):
                os.unlink(dst)
                shutil.copytree(src, dst)
            else:
                shutil.copy2(src, dst)
        except Exception:
            failures += 1
            print("*** copy failed for")
            print("\t src:", src)
            print("\t dst:", dst)

    return failures