Repository: ansible/ansible

Issue:
Malformed --diff for long files when using the copy module
**Ansible version**: 2.0.0.2
**Summary**:

I'm trying to generate `/etc/collectd/collectd.conf` using the ~~`template`~~ `copy` module.  When I preview my changes, I get diffs like this:

``` diff
--- before: /etc/collectd/collectd.conf
+++ after: /home/mg/src/deployments/provisioning/roles/common/files/collectd.conf.precise
@@ -417,397 +417,11 @@
 #      User "db_user"
 #      Password "secret"
 #      Database "db_name"
 #      MasterStats true
 #  </Database>
 #
 #  <Database db_name2>
 #      Host "localhost"
 #      Socket "/var/run/mysql/mysqld.sock"
 #      SlaveStats true
-#      SlaveNotifications true
-#  </Database>
-#</Plugin>
-
-#<Plugin netlink>
-#  Interface "All"
-#  VerboseInterface "All"
-#  QDisc "eth0" "pfifo_fast-1:0"
-#  Class "ppp0" "htb-1:10"
-#  Filter "ppp0" "u32-1:0"
-#  IgnoreSelected false
-#</Plugin>
-
-#<Plugin network>
-#  # client setup:
-#  Server "ff18::efc0:4a42" "25826"
-#  <Server "239.192.74.66" "25826">
-#      SecurityLevel Encrypt
-#      Username "user"
-#      Password "secret"
-#      Interface "eth0"
-#  </Server>
-#  TimeToLive "128"
-#
-#  # server setup:
-#  Listen "ff18::efc0:4a42" "25826"
-#  <Listen "239.192.74.66" "25826">
-#      SecurityLevel Sign
-#      AuthFile "/etc/collectd/passwd"
-#      Interface "eth0"
-#  </Listen>
-#  MaxPacketSize 1024
-#
-#  # proxy setup (client and server as above):
-#  Forward true
-#
-#  # statistics about the network plugin itself
-#  ReportStats false
-#
-#  # "garbage collection"
-#  CacheFlush 1800
-#</Plugin>
-
-#<Plugin nginx>
-#  URL "http://localhost/status?auto"
-#  User "www-user"
-#  Password "secret"
-#  VerifyPeer false
-#  VerifyHost false
-#  CACert "/etc/ssl/ca.crt"
-#</Plugin>
-
-#<Plugin notify_desktop>
-#  OkayTimeout 1000
-#  WarningTimeout 5000
-#  FailureTimeout 0
-#</Plugin>
-
-#<Plugin notify_email>
-#  SMTPServer "localhost"
-#  SMTPPort 25
-#  SMTPUser "my-username"
-#  SMTPPassword "my-password"
-#  From "collectd@main0server.com"
-#  # <WARNING/FAILURE/OK> on <hostname>.
-#  # Beware! Do not use not more than two placeholders (%)!
-#  Subject "[collectd] %s on %s!"
-#  Recipient "email1@domain1.net"
-#  Recipient "email2@domain2.com"
-#</Plugin>
-
-#<Plugin ntpd>
-#  Host "localhost"
-#  Port 123
-#  ReverseLookups false
-#</Plugin>
-
-#<Plugin nut>
-#  UPS "upsname@hostname:port"
-#</Plugin>
-
-#<Plugin olsrd>
-#  Host "127.0.0.1"
-#  Port "2006"
-#  CollectLinks "Summary"
-#  CollectRoutes "Summary"
-#  CollectTopology "Summary"
-#</Plugin>
-
-#<Plugin openvpn>
-#  StatusFile "/etc/openvpn/openvpn-status.log"
-#  ImprovedNamingSchema false
-#  CollectCompression true
-#  CollectIndividualUsers true
-#  CollectUserCount false
-#</Plugin>
-
-#<Plugin perl>
-#  IncludeDir "/my/include/path"
-#  BaseName "Collectd::Plugins"
-#  EnableDebugger ""
-#  LoadPlugin Monitorus
-#  LoadPlugin OpenVZ
-#
-#  <Plugin foo>
-#      Foo "Bar"
-#      Qux "Baz"
-#  </Plugin>
-#</Plugin>
-
-#<Plugin pinba>
-#  Address "::0"
-#  Port "30002"
-#  <View "name">
-#      Host "host name"
-#      Server "server name"
-#      Script "script name"
-#  <View>
-#</Plugin>
-
-#<Plugin ping>
-#  Host "host.foo.bar"
-#  Host "host.baz.qux"
-#  Interval 1.0
-#  Timeout 0.9
-#  TTL 255
-#  SourceAddress "1.2.3.4"
-#  Device "eth0"
-#  MaxMissed -1
-#</Plugin>
-
-#<Plugin postgresql>
-#  <Query magic>
-#      Statement "SELECT magic FROM wizard WHERE host = $1;"
-#      Param hostname
-#
-#      <Result>
-#          Type gauge
-#          InstancePrefix "magic"
-#          ValuesFrom "magic"
-#      </Result>
-#  </Query>
-#
-#  <Query rt36_tickets>
-#      Statement "SELECT COUNT(type) AS count, type \
-#                        FROM (SELECT CASE \
-#                                     WHEN resolved = 'epoch' THEN 'open' \
-#                                     ELSE 'resolved' END AS type \
-#                                     FROM tickets) type \
-#                        GROUP BY type;"
-#
-#      <Result>
-#          Type counter
-#          InstancePrefix "rt36_tickets"
-#          InstancesFrom "type"
-#          ValuesFrom "count"
-#      </Result>
-#  </Query>
-#
-#  <Database foo>
-#      Host "hostname"
-#      Port 5432
-#      User "username"
-#      Password "secret"
-#
-#      SSLMode "prefer"
-#      KRBSrvName "kerberos_service_name"
-#
-#      Query magic
-#  </Database>
-#
-#  <Database bar>
-#      Interval 60
-#      Service "service_name"
-#
-#      Query backend # predefined
-#      Query rt36_tickets
-#  </Database>
-#</Plugin>
-
-#<Plugin powerdns>
-#  <Server "server_name">
-#      Collect "latency"
-#      Collect "udp-answers" "udp-queries"
-#      Socket "/var/run/pdns.controlsocket"
-#  </Server>
-#  <Recursor "recursor_name">
-#      Collect "questions"
-#      Collect "cache-hits" "cache-misses"
-#      Socket "/var/run/pdns_recursor.controlsocket"
-#  </Recursor>
-#  LocalSocket "/opt/collectd/var/run/collectd-powerdns"
-#</Plugin>
-
-#<Plugin processes>
-#  Process "name"
-#  ProcessMatch "foobar" "/usr/bin/perl foobar\\.pl.*"
-#</Plugin>
-
-#<Plugin protocols>
-#  Value "/^Tcp:/"
-#  IgnoreSelected false
-#</Plugin>
-
-#<Plugin python>
-#  ModulePath "/path/to/your/python/modules"
-#  LogTraces true
-#  Interactive true
-#  Import "spam"
-#
-#  <Module spam>
-#      spam "wonderful" "lovely"
-#  </Module>
-#</Plugin>
-
-#<Plugin rrdcached>
-#  DaemonAddress "unix:/var/run/rrdcached.sock"
-#  DataDir "/var/lib/rrdcached/db/collectd"
-#  CreateFiles true
-#  CollectStatistics true
-#</Plugin>
-
-<Plugin rrdtool>
-   DataDir "/var/lib/collectd/rrd"
-#  CacheTimeout 120
-#  CacheFlush 900
-#  WritesPerSecond 30
-#  RandomTimeout 0
-#
-# The following settings are rather advanced
-# and should usually not be touched:
-#  StepSize 10
-#  HeartBeat 20
-#  RRARows 1200
-#  RRATimespan 158112000
-#  XFF 0.1
-</Plugin>
-
-#<Plugin sensors>
-#  Sensor "it8712-isa-0290/temperature-temp1"
-#  Sensor "it8712-isa-0290/fanspeed-fan3"
-#  Sensor "it8712-isa-0290/voltage-in8"
-#  IgnoreSelected false
-#</Plugin>
-
-# See /usr/share/doc/collectd/examples/snmp-data.conf.gz for a
-# comprehensive sample configuration.
-#<Plugin snmp>
-#  <Data "powerplus_voltge_input">
-#      Type "voltage"
-#      Table false
-#      Instance "input_line1"
-#      Scale 0.1
-#      Values "SNMPv2-SMI::enterprises.6050.5.4.1.1.2.1"
-#  </Data>
-#  <Data "hr_users">
-#      Type "users"
-#      Table false
-#      Instance ""
-#      Shift -1
-#      Values "HOST-RESOURCES-MIB::hrSystemNumUsers.0"
-#  </Data>
-#  <Data "std_traffic">
-#      Type "if_octets"
-#      Table true
-#      InstancePrefix "traffic"
-#      Instance "IF-MIB::ifDescr"
-#      Values "IF-MIB::ifInOctets" "IF-MIB::ifOutOctets"
-#  </Data>
-#
-#  <Host "some.switch.mydomain.org">
-#      Address "192.168.0.2"
-#      Version 1
-#      Community "community_string"
-#      Collect "std_traffic"
-#      Inverval 120
-#  </Host>
-#  <Host "some.server.mydomain.org">
-#      Address "192.168.0.42"
-#      Version 2
-#      Community "another_string"
-#      Collect "std_traffic" "hr_users"
-#  </Host>
-#  <Host "some.ups.mydomain.org">
-#      Address "192.168.0.3"
-#      Version 1
-#      Community "more_communities"
-#      Collect "powerplus_voltge_input"
-#      Interval 300
-#  </Host>
-#</Plugin>
-
-#<Plugin table>
-#  <Table "/proc/slabinfo">
-#      Instance "slabinfo"
-#      Separator " "
-#      <Result>
-#          Type gauge
-#          InstancePrefix "active_objs"
-#          InstancesFrom 0
-#          ValuesFrom 1
-#      </Result>
-#      <Result>
-#          Type gauge
-#          InstancePrefix "objperslab"
-#          InstancesFrom 0
-#          ValuesFrom 4
-#      </Result>
-#  </Table>
-#</Plugin>
-
-#<Plugin "tail">
-#  <File "/var/log/exim4/mainlog">
-#      Instance "exim"
-#      <Match>
-#          Regex "S=([1-9][0-9]*)"
-#          DSType "CounterAdd"
-#          Type "ipt_bytes"
-#          Instance "total"
-#      </Match>
-#      <Match>
-#          Regex "\\<R=local_user\\>"
-#          ExcludeRegex "\\<R=local_user\\>.*mail_spool defer"
-#          DSType "CounterInc"
-#          Type "counter"
-#          Instance "local_user"
-#      </Match>
-#  </File>
-#</Plugin>
-
-#<Plugin tcpconns>
-#  ListeningPorts false
-#  LocalPort "25"
-#  RemotePort "25"
-#</Plugin>
-
-#<Plugin teamspeak2>
-#  Host "127.0.0.1"
-#  Port "51234"
-#  Server "8767"
-#</Plugin>
-
-#<Plugin ted>
-#  Device "/dev/ttyUSB0"
-#  Retries 0
-#</Plugin>
-
-#<Plugin thermal>
-#  ForceUseProcfs false
-#  Device "THRM"
-#  IgnoreSelected false
-#</Plugin>
-
-#<Plugin tokyotyrant>
-#  Host "localhost"
-#  Port "1978"
-#</Plugin>
-
-#<Plugin unixsock>
-#  SocketFile "/var/run/collectd-unixsock"
-#  SocketGroup "collectd"
-#  SocketPerms "0660"
-#</Plugin>
-
-#<Plugin uuid>
-#  UUIDFile "/etc/uuid"
-#</Plugin>
-
-#<Plugin vmem>
-#  Verbose false
-#</Plugin>
-
-#<Plugin write_http>
-#  <URL "http://example.com/collectd-post">
-#      User "collectd"
-#      Password "secret"
-#      VerifyPeer true
-#      VerifyHost true
-#      CACert "/etc/ssl/ca.crt"
-#      Format "Command"
-#      StoreRates false
-#  </URL>
-#</Plugin>
-
-Include "/etc/collectd/filters.conf"
-Include "/etc/collectd/thresholds.conf"
-
+#      Sl
```

Note how the generated version of the template appears to be truncated.

Actually applying the change works fine; the apparent truncation happens only during diff.



Discussion:
I can reproduce with

```
ansible 2.0.1.0 (stable-2.0 18ffa8a4da) last updated 2016/02/10 12:19:09 (GMT +300)
  lib/ansible/modules/core: (detached HEAD 0508a4a4b8) last updated 2016/02/10 12:19:13 (GMT +300)
  lib/ansible/modules/extras: (detached HEAD f40fe33648) last updated 2016/02/10 12:19:13 (GMT +300)
  config file = 
  configured module search path = Default w/o overrides
```

I can reproduce with

```
ansible-playbook 2.1.0 (devel 5a88478ccc) last updated 2016/02/10 12:21:18 (GMT +300)
  lib/ansible/modules/core: (detached HEAD 9fc4ed0458) last updated 2016/02/10 12:21:20 (GMT +300)
  lib/ansible/modules/extras: (detached HEAD 6aeb2ab6cf) last updated 2016/02/10 12:21:20 (GMT +300)
  config file = /home/mg/src/deployments/provisioning/ansible.cfg
  configured module search path = Default w/o overrides
```

and I get a bonus

```
ERROR! Unexpected Exception: 'NoneType' object has no attribute 'terminate'
to see the full traceback, use -vvv
```

at the end (which disappears when I re-run with `-vvv`).

This would be because of the hardcoded 8KB limit in [lib/ansible/plugins/action/__init__.py](https://github.com/ansible/ansible/blob/5a88478ccca44c1b0d531a35e91e88f834db9210/lib/ansible/plugins/action/__init__.py#L602-L620):

``` python
            if source_file:
                display.debug("Reading local copy of the file %s" % source)
                try:
                    src = open(source)
                    src_contents = src.read(8192)
                    st = os.stat(source)
                except Exception as e:
                    raise AnsibleError("Unexpected error while reading source (%s) for diff: %s " % (source, str(e)))
                if "\x00" in src_contents:
                    diff['src_binary'] = 1
                elif st[stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF:
                    diff['src_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
                else:
                    diff['after_header'] = source
                    diff['after'] = src_contents
```

Already have an open PR for this @ #14051

Oh hi!  :-)

I like my fix better a little -- no seeking and re-reading already read bytes -- but it's essentially the same.


Resolving patch:
diff --git a/lib/ansible/plugins/action/__init__.py b/lib/ansible/plugins/action/__init__.py
--- a/lib/ansible/plugins/action/__init__.py
+++ b/lib/ansible/plugins/action/__init__.py
@@ -600,20 +600,21 @@ def _get_diff_data(self, destination, source, task_vars, source_file=True):
                     diff['before'] = dest_contents
 
             if source_file:
-                display.debug("Reading local copy of the file %s" % source)
-                try:
-                    src = open(source)
-                    src_contents = src.read(8192)
-                    st = os.stat(source)
-                except Exception as e:
-                    raise AnsibleError("Unexpected error while reading source (%s) for diff: %s " % (source, str(e)))
-                if "\x00" in src_contents:
-                    diff['src_binary'] = 1
-                elif st[stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF:
+                st = os.stat(source)
+                if st[stat.ST_SIZE] > C.MAX_FILE_SIZE_FOR_DIFF:
                     diff['src_larger'] = C.MAX_FILE_SIZE_FOR_DIFF
                 else:
-                    diff['after_header'] = source
-                    diff['after'] = src_contents
+                    display.debug("Reading local copy of the file %s" % source)
+                    try:
+                        src = open(source)
+                        src_contents = src.read()
+                    except Exception as e:
+                        raise AnsibleError("Unexpected error while reading source (%s) for diff: %s " % (source, str(e)))
+                    if "\x00" in src_contents:
+                        diff['src_binary'] = 1
+                    else:
+                        diff['after_header'] = source
+                        diff['after'] = src_contents
             else:
                 display.debug("source of file passed in")
                 diff['after_header'] = 'dynamically generated'


Test patch: