Repository: jazzband/pip-tools

Issue:
Add parameter `--no-trusted-host`: Don't add trusted host to generated file
Describe the issue briefly here.
##### Steps to replicate
1. Prepare `~/.pip/pip.conf`

``` ini
[global]
index-url = https://pypi.tuna.tsinghua.edu.cn/simple
extra-index-url = http://devpi.arpanet/private/dev/+simple
trusted-host = devpi.arpanet

[search]
index = http://devpi.arpanet/private/dev/
```

Then
2. `pip-compile --no-index --upgrade --output-file requirements.txt requirements.in`
##### Expected result

`--trusted-host devpi.arpanet` should not existed in `requirements.txt`
##### Actual result

I wish there's a option in `pip-sync` like `--no-index` which won't write `--trusted-host devpi.arpanet` into the generated `requirements.txt`. 



Discussion:
+1 this is kind of awkward. My global .pypirc or pip config or whatever it's reading from has a trusted host that I use for development on a specific project, but the host is being included in every project for which I pip-compile now. There's probably other solutions to this problem, but this seems obvious.


Resolving patch:
diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py
--- a/piptools/scripts/compile.py
+++ b/piptools/scripts/compile.py
@@ -6,7 +6,6 @@
 import os
 import sys
 import tempfile
-from collections import OrderedDict
 
 import pip
 from pip.req import InstallRequirement, parse_requirements
@@ -17,7 +16,7 @@
 from ..repositories import LocalRequirementsRepository, PyPIRepository
 from ..resolver import Resolver
 from ..utils import (assert_compatible_pip_version, is_pinned_requirement,
-                     key_from_req)
+                     key_from_req, dedup)
 from ..writer import OutputWriter
 
 # Make sure we're using a compatible version of pip
@@ -47,6 +46,8 @@ class PipCommand(pip.basecommand.Command):
               help="Add header to generated file")
 @click.option('--index/--no-index', is_flag=True, default=True,
               help="Add index URL to generated file")
+@click.option('--emit-trusted-host/--no-emit-trusted-host', is_flag=True,
+              default=True, help="Add trusted host option to generated file")
 @click.option('--annotate/--no-annotate', is_flag=True, default=True,
               help="Annotate results, indicating where dependencies come from")
 @click.option('-U', '--upgrade', is_flag=True, default=False,
@@ -64,8 +65,9 @@ class PipCommand(pip.basecommand.Command):
               help="Maximum number of rounds before resolving the requirements aborts.")
 @click.argument('src_files', nargs=-1, type=click.Path(exists=True, allow_dash=True))
 def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url,
-        client_cert, trusted_host, header, index, annotate, upgrade, upgrade_packages,
-        output_file, allow_unsafe, generate_hashes, src_files, max_rounds):
+        client_cert, trusted_host, header, index, emit_trusted_host, annotate,
+        upgrade, upgrade_packages, output_file, allow_unsafe, generate_hashes,
+        src_files, max_rounds):
     """Compiles requirements.txt from requirements.in specs."""
     log.verbose = verbose
 
@@ -138,7 +140,7 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url,
 
     log.debug('Using indexes:')
     # remove duplicate index urls before processing
-    repository.finder.index_urls = list(OrderedDict.fromkeys(repository.finder.index_urls))
+    repository.finder.index_urls = list(dedup(repository.finder.index_urls))
     for index_url in repository.finder.index_urls:
         log.debug('  {}'.format(index_url))
 
@@ -219,6 +221,7 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url,
 
     writer = OutputWriter(src_files, dst_file, dry_run=dry_run,
                           emit_header=header, emit_index=index,
+                          emit_trusted_host=emit_trusted_host,
                           annotate=annotate,
                           generate_hashes=generate_hashes,
                           default_index_url=repository.DEFAULT_INDEX_URL,
diff --git a/piptools/utils.py b/piptools/utils.py
--- a/piptools/utils.py
+++ b/piptools/utils.py
@@ -4,6 +4,7 @@
 
 import sys
 from itertools import chain, groupby
+from collections import OrderedDict
 
 import pip
 from pip.req import InstallRequirement
@@ -200,3 +201,10 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False):
         else:
             s.add(v)
     return dict(lut)
+
+
+def dedup(iterable):
+    """Deduplicate an iterable object like iter(set(iterable)) but
+    order-reserved.
+    """
+    return iter(OrderedDict.fromkeys(iterable))
diff --git a/piptools/writer.py b/piptools/writer.py
--- a/piptools/writer.py
+++ b/piptools/writer.py
@@ -5,18 +5,19 @@
 from .click import unstyle
 from .io import AtomicSaver
 from .logging import log
-from .utils import comment, format_requirement, UNSAFE_PACKAGES
+from .utils import comment, format_requirement, dedup, UNSAFE_PACKAGES
 
 
 class OutputWriter(object):
     def __init__(self, src_files, dst_file, dry_run, emit_header, emit_index,
-                 annotate, generate_hashes, default_index_url, index_urls,
-                 trusted_hosts, format_control):
+                 emit_trusted_host, annotate, generate_hashes,
+                 default_index_url, index_urls, trusted_hosts, format_control):
         self.src_files = src_files
         self.dst_file = dst_file
         self.dry_run = dry_run
         self.emit_header = emit_header
         self.emit_index = emit_index
+        self.emit_trusted_host = emit_trusted_host
         self.annotate = annotate
         self.generate_hashes = generate_hashes
         self.default_index_url = default_index_url
@@ -36,6 +37,8 @@ def write_header(self):
             params = []
             if not self.emit_index:
                 params += ['--no-index']
+            if not self.emit_trusted_host:
+                params += ['--no-emit-trusted-host']
             if not self.annotate:
                 params += ['--no-annotate']
             if self.generate_hashes:
@@ -47,20 +50,21 @@ def write_header(self):
 
     def write_index_options(self):
         if self.emit_index:
-            for index, index_url in enumerate(self.index_urls):
+            for index, index_url in enumerate(dedup(self.index_urls)):
                 if index_url.rstrip('/') == self.default_index_url:
                     continue
                 flag = '--index-url' if index == 0 else '--extra-index-url'
                 yield '{} {}'.format(flag, index_url)
 
     def write_trusted_hosts(self):
-        for trusted_host in self.trusted_hosts:
-            yield '--trusted-host {}'.format(trusted_host)
+        if self.emit_trusted_host:
+            for trusted_host in dedup(self.trusted_hosts):
+                yield '--trusted-host {}'.format(trusted_host)
 
     def write_format_controls(self):
-        for nb in self.format_control.no_binary:
+        for nb in dedup(self.format_control.no_binary):
             yield '--no-binary {}'.format(nb)
-        for ob in self.format_control.only_binary:
+        for ob in dedup(self.format_control.only_binary):
             yield '--only-binary {}'.format(ob)
 
     def write_flags(self):


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -107,22 +107,39 @@ def test_extra_index_option(pip_conf):
                 '  http://example.com\n'
                 '  http://extraindex1.com\n'
                 '  http://extraindex2.com' in out.output)
+        assert ('--index-url http://example.com\n'
+                '--extra-index-url http://extraindex1.com\n'
+                '--extra-index-url http://extraindex2.com' in out.output)
 
 
 def test_trusted_host(pip_conf):
-
     assert os.path.exists(pip_conf)
 
     runner = CliRunner()
     with runner.isolated_filesystem():
         open('requirements.in', 'w').close()
         out = runner.invoke(cli, ['-v',
+                                  '--trusted-host', 'example.com',
                                   '--trusted-host', 'example2.com'])
         print(out.output)
         assert ('--trusted-host example.com\n'
                 '--trusted-host example2.com\n' in out.output)
 
 
+def test_trusted_host_no_emit(pip_conf):
+    assert os.path.exists(pip_conf)
+
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        open('requirements.in', 'w').close()
+        out = runner.invoke(cli, ['-v',
+                                  '--trusted-host', 'example.com',
+                                  '--no-emit-trusted-host'])
+        print(out.output)
+        assert '--trusted-host example.com' not in out.output
+        assert '--no-emit-trusted-host' in out.output
+
+
 def test_realistic_complex_sub_dependencies(tmpdir):
 
     # make a temporary wheel of a fake package
diff --git a/tests/test_utils.py b/tests/test_utils.py
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,6 +1,7 @@
 from pytest import raises
 
-from piptools.utils import as_tuple, format_requirement, format_specifier, flat_map
+from piptools.utils import (
+    as_tuple, format_requirement, format_specifier, flat_map, dedup)
 
 
 def test_format_requirement(from_line):
@@ -53,3 +54,7 @@ def test_as_tuple(from_line):
 
 def test_flat_map():
     assert [1, 2, 4, 1, 3, 9] == list(flat_map(lambda x: [1, x, x * x], [2, 3]))
+
+
+def test_dedup():
+    assert list(dedup([3, 1, 2, 4, 3, 5])) == [3, 1, 2, 4, 5]
diff --git a/tests/test_writer.py b/tests/test_writer.py
--- a/tests/test_writer.py
+++ b/tests/test_writer.py
@@ -7,8 +7,10 @@
 
 @fixture
 def writer():
-    return OutputWriter(src_files=["src_file", "src_file2"], dst_file="dst_file", dry_run=True,
-                        emit_header=True, emit_index=True, annotate=True,
+    return OutputWriter(src_files=["src_file", "src_file2"], dst_file="dst_file",
+                        dry_run=True,
+                        emit_header=True, emit_index=True, emit_trusted_host=True,
+                        annotate=True,
                         generate_hashes=False,
                         default_index_url=None, index_urls=[],
                         trusted_hosts=[],