Repository: obspy/obspy

Issue:
Possible memory leak in SeedLinkConnection.collect()
I'm seeing memory usage growth when using SeedLinkConnection.collect() in a loop.

Linux or Windows
obspy 0.9.2
numpy 1.8.2
python 2.7.6

Can anyone confirm?



Discussion:
Didn't you have a code snippet for reproduction?

Can you try this?

``` python
import os
import gc
import psutil
from obspy import UTCDateTime
from obspy.seedlink.slpacket import SLPacket
from obspy.seedlink.client.seedlinkconnection import SeedLinkConnection

def getMemoryUsage():
    """ return memory rss in byte """
    process = psutil.Process(os.getpid())
    mem = process.memory_info()[0]
    return mem

# create a seedlink connection 
selectors = None
begin = UTCDateTime().now() - 12*60*60
connection = SeedLinkConnection()
connection.setSLAddress("geofon.gfz-potsdam.de:18000")
connection.setBeginTime(begin.formatSeedLink())
connection.parseStreamlist("GE_STU:HH?", selectors)

# enable gc debugging
gc.set_debug(gc.DEBUG_LEAK)

# main loop
count = 0
while True:

    # force gc
    unreach = gc.collect()
    if unreach <> 0:
        print "Unreachable objects: %d" % (unreach)

    # get next packet
    packet = connection.collect()

    if packet is None or packet == SLPacket.SLTERMINATE:
       break

    count = count + 1
    if count == 1:
        mem1 = getMemoryUsage()

    # print some stats
    if count > 1 and count % 10 == 1:
        mem2 = getMemoryUsage()
        print "Additional memory usage after %d packets: %d bytes, %d bytes/packet" \
            % (count-1, mem2-mem1, (mem2-mem1)/(count-1)) 
```

There was a [recent mailing list post](http://lists.swapbytes.de/archives/obspy-users/2015-January/001619.html) that claims this leak is caused by `range`. That being said, the development version will use the generator version of `range`, and this will likely fix the issue (assuming that is the cause).

@mbankdmt could you comment on the status of this ticket?

Based on the above comment, the memory could be due to range (why ever???). However, this is replaced in current master --- therefore, does this problem still persist in master?

I just cloned the latest master and did a run of the above code snippet against it. On my system the memory leak is still showing up.

Just tried your script, and I think I can reproduce the issue. I got the following output:

```
...
gc: collectable <list 0x7f1ce7ebb9e0>
gc: collectable <list 0x7f1ce7ebb878>
gc: collectable <list 0x7f1ce7eb7b00>
Unreachable objects: 25
Additional memory usage after 10 packets: 0 bytes, 0 bytes/packet
Additional memory usage after 20 packets: 0 bytes, 0 bytes/packet
...
Additional memory usage after 150 packets: 4096 bytes, 27 bytes/packet
Additional memory usage after 160 packets: 24576 bytes, 153 bytes/packet
Additional memory usage after 170 packets: 49152 bytes, 289 bytes/packet
Additional memory usage after 180 packets: 65536 bytes, 364 bytes/packet
Additional memory usage after 190 packets: 90112 bytes, 474 bytes/packet
...
```


Resolving patch:
diff --git a/obspy/mseed/util.py b/obspy/mseed/util.py
--- a/obspy/mseed/util.py
+++ b/obspy/mseed/util.py
@@ -479,8 +479,6 @@ def _ctypesArray2NumpyArray(buffer_, buffer_elements, sampletype):
     Takes a Ctypes array and its length and type and returns it as a
     NumPy array.
 
-    This works by reference and no data is copied.
-
     :param buffer_: Ctypes c_void_p pointer to buffer.
     :param buffer_elements: length of the whole buffer
     :param sampletype: type of sample, on of "a", "i", "f", "d"
diff --git a/obspy/seedlink/client/seedlinkconnection.py b/obspy/seedlink/client/seedlinkconnection.py
--- a/obspy/seedlink/client/seedlinkconnection.py
+++ b/obspy/seedlink/client/seedlinkconnection.py
@@ -1232,8 +1232,8 @@ def sayHello(self):
         """
         sendStr = b"HELLO"
         logger.debug("sending: %s" % (sendStr.decode()))
-        bytes = sendStr + b"\r"
-        bytesread = self.sendData(bytes, self.sladdr,
+        bytes_ = sendStr + b"\r"
+        bytesread = self.sendData(bytes_, self.sladdr,
                                   SeedLinkConnection.DFT_READBUF_SIZE)
 
         # Parse the server ID and version from the returned string
@@ -1291,13 +1291,13 @@ def sendInfoRequest(self, infoLevel, verb_level):
         :raise IOException: if an I/O error occurs.
         """
         if self.checkVersion(2.92) >= 0:
-            bytes = b"INFO " + infoLevel.encode('ascii', 'strict') + b"\r"
+            bytes_ = b"INFO " + infoLevel.encode('ascii', 'strict') + b"\r"
             msg = "sending: requesting INFO level %s" % (infoLevel)
             if verb_level == 1:
                 logger.info(msg)
             else:
                 logger.debug(msg)
-            self.sendData(bytes, self.sladdr, 0)
+            self.sendData(bytes_, self.sladdr, 0)
         else:
             msg = "detected SeedLink version %s does not support INFO requests"
             raise SeedLinkException(msg % (self.server_version))
@@ -1367,9 +1367,9 @@ def negotiateStation(self, curstream):
                 # Build SELECT command, send it and receive response
                 sendStr = b"SELECT " + selector
                 logger.debug("sending: %s" % (sendStr))
-                bytes = sendStr + b"\r"
+                bytes_ = sendStr + b"\r"
                 bytesread = None
-                bytesread = self.sendData(bytes, self.sladdr,
+                bytesread = self.sendData(bytes_, self.sladdr,
                                           SeedLinkConnection.DFT_READBUF_SIZE)
                 readStr = bytesread.decode()
 
@@ -1443,9 +1443,9 @@ def negotiateStation(self, curstream):
 
         # Send action command and receive response
         logger.debug("sending: %s" % (sendStr))
-        bytes = sendStr + b"\r"
+        bytes_ = sendStr + b"\r"
         bytesread = None
-        bytesread = self.sendData(bytes, self.sladdr,
+        bytesread = self.sendData(bytes_, self.sladdr,
                                   SeedLinkConnection.DFT_READBUF_SIZE)
 
         # Check response to DATA/FETCH/TIME
@@ -1517,9 +1517,9 @@ def negotiateMultiStation(self):
             sendStr = ("STATION  " + curstream.station + " " +
                        curstream.net).encode('ascii', 'strict')
             logger.debug("sending: %s" % sendStr.decode())
-            bytes = sendStr + b"\r"
+            bytes_ = sendStr + b"\r"
             bytesread = None
-            bytesread = self.sendData(bytes, self.sladdr,
+            bytesread = self.sendData(bytes_, self.sladdr,
                                       SeedLinkConnection.DFT_READBUF_SIZE)
             readStr = bytesread
 
@@ -1553,8 +1553,8 @@ def negotiateMultiStation(self):
         # Issue END action command
         sendStr = b"END"
         logger.debug("sending: %s" % (sendStr.decode()))
-        bytes = sendStr + b"\r"
-        self.sendData(bytes, self.sladdr, 0)
+        bytes_ = sendStr + b"\r"
+        self.sendData(bytes_, self.sladdr, 0)
 
     def updateStream(self, slpacket):
         """
diff --git a/obspy/seedlink/slpacket.py b/obspy/seedlink/slpacket.py
--- a/obspy/seedlink/slpacket.py
+++ b/obspy/seedlink/slpacket.py
@@ -17,7 +17,8 @@
 
 from obspy.core.compatibility import frombuffer
 from obspy.core.trace import Trace
-from obspy.mseed.headers import clibmseed, HPTMODULUS, MSRecord
+from obspy.core.util.decorator import deprecated_keywords
+from obspy.mseed.headers import clibmseed, HPTMODULUS
 from obspy.mseed.util import _convertMSRToDict, _ctypesArray2NumpyArray
 from obspy.seedlink.seedlinkexception import SeedLinkException
 import ctypes as C
@@ -68,15 +69,16 @@ class SLPacket(object):
     ERRORSIGNATURE = b"ERROR\r\n"
     ENDSIGNATURE = b"END"
 
-    def __init__(self, bytes=None, offset=None):
-        if bytes is None or offset is None:
+    @deprecated_keywords({'bytes': 'data'})
+    def __init__(self, data=None, offset=None):
+        if data is None or offset is None:
             return
-        if len(bytes) - offset < self.SLHEADSIZE + self.SLRECSIZE:
+        if len(data) - offset < self.SLHEADSIZE + self.SLRECSIZE:
             msg = "not enough bytes in sub array to construct a new SLPacket"
             raise SeedLinkException(msg)
-        self.slhead = bytes[offset: offset + self.SLHEADSIZE]
-        self.msrecord = bytes[offset + self.SLHEADSIZE:
-                              offset + self.SLHEADSIZE + self.SLRECSIZE]
+        self.slhead = data[offset: offset + self.SLHEADSIZE]
+        self.msrecord = data[offset + self.SLHEADSIZE:
+                             offset + self.SLHEADSIZE + self.SLRECSIZE]
         self.trace = None
 
     def getSequenceNumber(self):
@@ -108,7 +110,7 @@ def getSequenceNumber(self):
 
     def getMSRecord(self):
         # following from  obspy.mseed.tests.test_libmseed.py -> test_msrParse
-        msr = clibmseed.msr_init(C.POINTER(MSRecord)())
+        msr = clibmseed.msr_init(None)
         pyobj = frombuffer(self.msrecord, dtype=np.uint8)
         errcode = \
             clibmseed.msr_parse(pyobj.ctypes.data_as(C.POINTER(C.c_char)),
@@ -119,16 +121,32 @@ def getMSRecord(self):
         # print "DEBUG: msr:", msr
         msrecord_py = msr.contents
         # print "DEBUG: msrecord_py:", msrecord_py
-        return msrecord_py
+        return msr, msrecord_py
+
+    def freeMSRecord(self, msr, msrecord_py):
+        clibmseed.msr_free(msr)
 
     def getTrace(self):
 
         if self.trace is not None:
             return self.trace
 
-        msrecord_py = self.getMSRecord()
-        # print "DEBUG: msrecord_py:", msrecord_py
-        header = _convertMSRToDict(msrecord_py)
+        msr, msrecord_py = self.getMSRecord()
+        try:
+            header = _convertMSRToDict(msrecord_py)
+
+            # XXX Workaround: in Python 3 msrecord_py.sampletype is a byte
+            # (e.g. b'i'), while keys of mseed.headers.SAMPLESIZES are
+            # unicode ('i') (see above)
+            sampletype = msrecord_py.sampletype
+            if not isinstance(sampletype, str):
+                sampletype = sampletype.decode()
+
+            data = _ctypesArray2NumpyArray(msrecord_py.datasamples,
+                                           msrecord_py.numsamples,
+                                           sampletype)
+        finally:
+            self.freeMSRecord(msr, msrecord_py)
 
         # XXX Workaround: the fields in the returned struct of type
         # obspy.mseed.header.MSRecord_s have byte values in Python 3, while
@@ -148,16 +166,6 @@ def getTrace(self):
             del header['samprate']
         # Access data directly as NumPy array.
 
-        # XXX Workaround: in Python 3 msrecord_py.sampletype is a byte
-        # (e.g. b'i'), while keys of mseed.headers.SAMPLESIZES are
-        # unicode ('i') (see above)
-        sampletype = msrecord_py.sampletype
-        if not isinstance(sampletype, str):
-            sampletype = sampletype.decode()
-
-        data = _ctypesArray2NumpyArray(msrecord_py.datasamples,
-                                       msrecord_py.numsamples,
-                                       sampletype)
         self.trace = Trace(data, header)
         return self.trace
 
@@ -165,11 +173,15 @@ def getStringPayload(self):
         """
         Get the MiniSEED payload, parsed as string.
         """
-        msrecord_py = self.getMSRecord()
+        msr, msrecord_py = self.getMSRecord()
 
-        # This is the same data buffer that is accessed by
-        # _ctypesArray2NumpyArray in getTrace above.
-        payload = C.string_at(msrecord_py.datasamples, msrecord_py.samplecnt)
+        try:
+            # This is the same data buffer that is accessed by
+            # _ctypesArray2NumpyArray in getTrace above.
+            payload = C.string_at(msrecord_py.datasamples,
+                                  msrecord_py.samplecnt)
+        finally:
+            self.freeMSRecord(msr, msrecord_py)
 
         return payload
 
@@ -181,27 +193,9 @@ def getType(self):
                 return self.TYPE_SLINFT
             else:
                 return self.TYPE_SLINF
-        msrecord_py = self.getMSRecord()
-        # print "DEBUG: msrecord_py:", msrecord_py
-        # print "DEBUG: msrecord_py.reclen:", msrecord_py.reclen
-        # print "DEBUG: msrecord_py.sequence_number:",
-        # print msrecord_py.sequence_number
-        # print "DEBUG: msrecord_py.samplecnt:", msrecord_py.samplecnt
-        # print "DEBUG: msrecord_py.encoding:", msrecord_py.encoding
-        # print "DEBUG: msrecord_py.byteorder:", msrecord_py.byteorder
-        # print "DEBUG: msrecord_py.numsamples:", msrecord_py.numsamples
-        # print "DEBUG: msrecord_py.sampletype:", msrecord_py.sampletype
-        # print "DEBUG: msrecord_py.blkts:", msrecord_py.blkts
-        blockette = msrecord_py.blkts.contents
-        while blockette:
-            # print "DEBUG: ===================="
-            # print "DEBUG: blkt_type:", blockette.blkt_type
-            # print "DEBUG: next_blkt:", blockette.next_blkt
-            # print "DEBUG: blktdata:", blockette.blktdata
-            # print "DEBUG: blktdatalen:", blockette.blktdatalen
-            # print "DEBUG: next:", blockette.next
-            try:
-                blockette = blockette.next.contents
-            except:
-                blockette = None
-        return msrecord_py.blkts.contents.blkt_type
+        msr, msrecord_py = self.getMSRecord()
+        try:
+            ret = msrecord_py.blkts.contents.blkt_type
+        finally:
+            self.freeMSRecord(msr, msrecord_py)
+        return ret


Test patch: