tricycle/lesswrong — pull request #47 by johnsoft (2012-09-16 01:21:41+00:00)
Title: Upgrade memcached client to v1.48, preserving modifications

For [Issue 337](http://code.google.com/p/lesswrong/issues/detail?id=337) -

Upgrading the memcached client is probably a good place to start. The latest version is a bit more deliberate about handling socket timeouts (translation: it doesn't ignore the issue entirely). I ran the new version's test code, and then just did a quick sanity check on the site.

For ease of review, here's a diff of our fork against the otficial version. which should be a lot cleaner to read through than the actual diff.

--- memcache.py	2011-11-28 02:17:32.000000000 +0100
+++ /home/lesswrong/lesswrong/r2/r2/lib/contrib/memcache.py	2012-09-16 02:52:14.660964073 +0200
@@ -48,6 +48,7 @@
import time
import os
import re
+from hashlib import md5
try:
import cPickle as pickle
except ImportError:
@@ -109,7 +110,7 @@
pass

-_DEAD_RETRY = 30 # number of seconds before retrying a dead server.
+_DEAD_RETRY = 1 # number of seconds before retrying a dead server.
_SOCKET_TIMEOUT = 3 # number of seconds before sockets timeout.

@@ -332,13 +333,28 @@
else:
serverhash = serverHashFunction(key)

- for i in range(Client._SERVER_RETRIES):
- server = self.buckets[serverhash % len(self.buckets)]
+ # the original version suffered from a failure rate of
+ # 1 in n^_SERVER_RETRIES, where n = number of bukets
+ # if one server is down. This is particularly bad for
+ # n = 2, and the hashing is poor enough to guarantee
+ # that making _SERVER_RETRIES larger doesn't help.
+
+ #make a copy
+ good_servers = list(self.buckets)
+ while good_servers:
+ server = good_servers.pop(serverhash % len(good_servers))
if server.connect():
- #print "(using server %s)" % server,
return server, key
- serverhash = serverHashFunction(str(serverhash) + str(i))
- return None, None
+
+ # removed by chris
+ #for i in range(Client._SERVER_RETRIES):
+ # server = self.buckets[serverhash % len(self.buckets)]
+ # if server.connect():
+ # #print "(using server %s)" % server,
+ # return server, key
+ # serverhash = serverHashFunction(str(serverhash) + str(i))
+
+ return None, key

def disconnect_all(self):
for s in self.servers:
@@ -418,8 +434,8 @@
should fail. Defaults to None for no delay.
@rtype: int
'''
- self.check_key(key)
server, key = self._get_server(key)
+ key = self.check_key(key)
if not server:
return 0
self._statlog('delete')
@@ -478,8 +494,8 @@
return self._incrdecr("decr", key, delta)

def _incrdecr(self, cmd, key, delta):
- self.check_key(key)
server, key = self._get_server(key)
+ key = self.check_key(key)
if not server:
return 0
self._statlog(cmd)
@@ -603,8 +619,9 @@
"""
# Check it just once ...
key_extra_len=len(key_prefix)
- if key_prefix:
- self.check_key(key_prefix)
+ #changed by steve
+ #if key_prefix:
+ #self.check_key(key_prefix)

# server (_Host) -> list of unprefixed server keys in mapping
server_keys = {}
@@ -622,7 +639,9 @@
server, key = self._get_server(key_prefix + str_orig_key)

# Now check to make sure key length is proper ...
- self.check_key(str_orig_key, key_extra_len=key_extra_len)
+ #changed by steve
+ #self.check_key(str_orig_key, key_extra_len=key_extra_len)
+ key = self.check_key(key_prefix + str_orig_key)

if not server:
continue
@@ -768,9 +787,9 @@

return (flags, len(val), val)

- def _set(self, cmd, key, val, time, min_compress_len = 0):
- self.check_key(key)
+ def _set(self, cmd, key, val, time, min_compress_len=0):
server, key = self._get_server(key)
+ key = self.check_key(key)
if not server:
return 0

@@ -810,8 +829,8 @@
return 0

def _get(self, cmd, key):
- self.check_key(key)
server, key = self._get_server(key)
+ key = self.check_key(key)
if not server:
return None

@@ -1016,15 +1035,17 @@
if not isinstance(key, str):
raise Client.MemcachedKeyTypeError("Key must be str()'s")

- if isinstance(key, basestring):
- if self.server_max_key_length != 0 and \
- len(key) + key_extra_len > self.server_max_key_length:
- raise Client.MemcachedKeyLengthError("Key length is > %s"
- % self.server_max_key_length)
- for char in key:
- if ord(char) < 33 or ord(char) == 127:
- raise Client.MemcachedKeyCharacterError(
- "Control characters not allowed")
+ #if isinstance(key, basestring):
+ # if self.server_max_key_length != 0 and \
+ # len(key) + key_extra_len > self.server_max_key_length:
+ # raise Client.MemcachedKeyLengthError("Key length is > %s"
+ # % self.server_max_key_length)
+ # for char in key:
+ # if ord(char) < 33 or ord(char) == 127:
+ # raise Client.MemcachedKeyCharacterError(
+ # "Control characters not allowed")
+
+ return md5(key).hexdigest()

class _Host(object):