GitHub PullRequestReviewCommentEvent at 2014-03-18 19:48:40
Repository: soledad
Actor: chiiph
Payload: {"comment":{"url":"https://api.github.com/repos/leapcode/soledad/pulls/comments/10722127","id":10722127,"diff_hunk":"@@ -173,41 +241,511 @@ def doc_passphrase(self, doc_id):\n             raise NoSymmetricSecret()\n         return hmac.new(\n             self.secret[\n-                self.MAC_KEY_LENGTH:\n+                MAC_KEY_LENGTH:\n                 self._soledad.REMOTE_STORAGE_SECRET_LENGTH],\n             doc_id,\n             hashlib.sha256).digest()\n \n-    def doc_mac_key(self, doc_id):\n+    #\n+    # secret setters/getters\n+    #\n+\n+    def _get_secret(self):\n+        return self._soledad.storage_secret\n+\n+    secret = property(\n+        _get_secret, doc='The secret used for symmetric encryption')\n+\n+#\n+# Crypto utilities for a SoledadDocument.\n+#\n+\n+\n+def mac_doc(doc_id, doc_rev, ciphertext, mac_method, secret):\n+    \"\"\"\n+    Calculate a MAC for C{doc} using C{ciphertext}.\n+\n+    Current MAC method used is HMAC, with the following parameters:\n+\n+        * key: sha256(storage_secret, doc_id)\n+        * msg: doc_id + doc_rev + ciphertext\n+        * digestmod: sha256\n+\n+    :param doc_id: The id of the document.\n+    :type doc_id: str\n+    :param doc_rev: The revision of the document.\n+    :type doc_rev: str\n+    :param ciphertext: The content of the document.\n+    :type ciphertext: str\n+    :param mac_method: The MAC method to use.\n+    :type mac_method: str\n+    :param secret: soledad secret\n+    :type secret: Soledad.secret_storage\n+\n+    :return: The calculated MAC.\n+    :rtype: str\n+    \"\"\"\n+    if mac_method == MacMethods.HMAC:\n+        return hmac.new(\n+            doc_mac_key(doc_id, secret),\n+            str(doc_id) + str(doc_rev) + ciphertext,\n+            hashlib.sha256).digest()\n+    # raise if we do not know how to handle this MAC method\n+    raise UnknownMacMethod('Unknown MAC method: %s.' % mac_method)\n+\n+\n+def encrypt_docstr(docstr, doc_id, doc_rev, key, secret):\n+    \"\"\"\n+    Encrypt C{doc}'s content.\n+\n+    Encrypt doc's contents using AES-256 CTR mode and return a valid JSON\n+    string representing the following:\n+\n+        {\n+            ENC_JSON_KEY: '<encrypted doc JSON string>',\n+            ENC_SCHEME_KEY: 'symkey',\n+            ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,\n+            ENC_IV_KEY: '<the initial value used to encrypt>',\n+            MAC_KEY: '<mac>'\n+            MAC_METHOD_KEY: 'hmac'\n+        }\n+\n+    :param docstr: A representation of the document to be encrypted.\n+    :type docstr: str or unicode.\n+\n+    :param doc_id: The document id.\n+    :type doc_id: str\n+\n+    :param doc_rev: The document revision.\n+    :type doc_rev: str\n+\n+    :param key: The key used to encrypt ``data`` (must be 256 bits long).\n+    :type key: str\n+\n+    :param secret:\n+    :type secret:\n+\n+    :return: The JSON serialization of the dict representing the encrypted\n+        content.\n+    :rtype: str\n+    \"\"\"\n+    # encrypt content using AES-256 CTR mode\n+    iv, ciphertext = encrypt_sym(\n+        str(docstr),  # encryption/decryption routines expect str\n+        key, method=EncryptionMethods.AES_256_CTR)\n+    # Return a representation for the encrypted content. In the following, we\n+    # convert binary data to hexadecimal representation so the JSON\n+    # serialization does not complain about what it tries to serialize.\n+    hex_ciphertext = binascii.b2a_hex(ciphertext)\n+    return json.dumps({\n+        ENC_JSON_KEY: hex_ciphertext,\n+        ENC_SCHEME_KEY: EncryptionSchemes.SYMKEY,\n+        ENC_METHOD_KEY: EncryptionMethods.AES_256_CTR,\n+        ENC_IV_KEY: iv,\n+        MAC_KEY: binascii.b2a_hex(mac_doc(  # store the mac as hex.\n+            doc_id, doc_rev, ciphertext,\n+            MacMethods.HMAC, secret)),\n+        MAC_METHOD_KEY: MacMethods.HMAC,\n+    })\n+\n+\n+def decrypt_doc_dict(doc_dict, doc_id, doc_rev, key, secret):\n+    \"\"\"\n+    Decrypt C{doc}'s content.\n+\n+    Return the JSON string representation of the document's decrypted content.\n+\n+    The passed doc_dict argument should have the following structure:\n+\n+        {\n+            ENC_JSON_KEY: '<enc_blob>',\n+            ENC_SCHEME_KEY: '<enc_scheme>',\n+            ENC_METHOD_KEY: '<enc_method>',\n+            ENC_IV_KEY: '<initial value used to encrypt>',  # (optional)\n+            MAC_KEY: '<mac>'\n+            MAC_METHOD_KEY: 'hmac'\n+        }\n+\n+    C{enc_blob} is the encryption of the JSON serialization of the document's\n+    content. For now Soledad just deals with documents whose C{enc_scheme} is\n+    EncryptionSchemes.SYMKEY and C{enc_method} is\n+    EncryptionMethods.AES_256_CTR.\n+\n+    :param doc_dict: The content of the document to be decrypted.\n+    :type doc_dict: dict\n+\n+    :param doc_id: The document id.\n+    :type doc_id: str\n+\n+    :param doc_rev: The document revision.\n+    :type doc_rev: str\n+\n+    :param key: The key used to encrypt ``data`` (must be 256 bits long).\n+    :type key: str\n+\n+    :param secret:\n+    :type secret:\n+\n+    :return: The JSON serialization of the decrypted content.\n+    :rtype: str\n+    \"\"\"\n+    # TODO where should we move these assertions, now that we're passed the\n+    # string?\n+    #soledad_assert(doc.is_tombstone() is False)\n+\n+    soledad_assert(ENC_JSON_KEY in doc_dict)\n+    soledad_assert(ENC_SCHEME_KEY in doc_dict)\n+    soledad_assert(ENC_METHOD_KEY in doc_dict)\n+    soledad_assert(MAC_KEY in doc_dict)\n+    soledad_assert(MAC_METHOD_KEY in doc_dict)\n+\n+    # verify MAC\n+    ciphertext = binascii.a2b_hex(  # content is stored as hex.\n+        doc_dict[ENC_JSON_KEY])\n+    mac = mac_doc(\n+        doc_id, doc_rev,\n+        ciphertext,\n+        doc_dict[MAC_METHOD_KEY], secret)\n+    # we compare mac's hashes to avoid possible timing attacks that might\n+    # exploit python's builtin comparison operator behaviour, which fails\n+    # immediatelly when non-matching bytes are found.\n+    doc_mac_hash = hashlib.sha256(\n+        binascii.a2b_hex(  # the mac is stored as hex\n+            doc_dict[MAC_KEY])).digest()\n+    calculated_mac_hash = hashlib.sha256(mac).digest()\n+    if doc_mac_hash != calculated_mac_hash:\n+        raise WrongMac('Could not authenticate document\\'s contents.')\n+    # decrypt doc's content\n+    enc_scheme = doc_dict[ENC_SCHEME_KEY]\n+    plainjson = None\n+    if enc_scheme == EncryptionSchemes.SYMKEY:\n+        enc_method = doc_dict[ENC_METHOD_KEY]\n+        if enc_method == EncryptionMethods.AES_256_CTR:\n+            soledad_assert(ENC_IV_KEY in doc_dict)\n+            plainjson = decrypt_sym(\n+                ciphertext, key,\n+                method=enc_method,\n+                iv=doc_dict[ENC_IV_KEY])\n+        else:\n+            raise UnknownEncryptionMethod(enc_method)\n+    else:\n+        raise UnknownEncryptionScheme(enc_scheme)\n+\n+    return plainjson\n+\n+\n+def is_symmetrically_encrypted(doc):\n+    \"\"\"\n+    Return True if the document was symmetrically encrypted.\n+\n+    :param doc: The document to check.\n+    :type doc: SoledadDocument\n+\n+    :rtype: bool\n+    \"\"\"\n+    if doc.content and ENC_SCHEME_KEY in doc.content:\n+        if doc.content[ENC_SCHEME_KEY] == EncryptionSchemes.SYMKEY:\n+            return True\n+    return False\n+\n+\n+#\n+# Encrypt/decrypt pools of workers\n+#\n+\n+class SyncEncryptDecryptPool(object):\n+    \"\"\"\n+    Base class for encrypter/decrypter pools.\n+    \"\"\"\n+    WORKERS = 5\n+\n+    def __init__(self, crypto, sync_db):\n+        \"\"\"\n+        Initialize the pool of encryption-workers.\n+\n+        :param crypto: A SoledadCryto instance to perform the encryption.\n+        :type crypto: leap.soledad.crypto.SoledadCrypto\n+\n+        :param sync_db: a database connection handle\n+        :type sync_db: handle\n+\n+        :param insert_doc_cb: Optional callback for inserting doc.\n+        :type insert_doc_cb: callable\n+        \"\"\"\n+        self._pool = multiprocessing.Pool(self.WORKERS)\n+        self._crypto = crypto\n+        self._sync_db = sync_db\n+\n+\n+def encrypt_doc_task(doc_id, doc_rev, content, key, secret):\n+    encrypted_content = encrypt_docstr(\n+        content, doc_id, doc_rev, key, secret)\n+    return doc_id, doc_rev, encrypted_content\n+\n+\n+class SyncEncrypterPool(SyncEncryptDecryptPool):\n+    \"\"\"\n+    Pool of workers that spawn subprocesses to execute the symmetric encryption\n+    of documents to be synced.\n+    \"\"\"\n+    # TODO implement throttling to reduce cpu usage??\n+    WORKERS = 5\n+    TABLE_NAME = \"docs_tosync\"\n+    FIELD_NAMES = \"doc_id, rev, content\"\n+\n+    def encrypt_doc(self, doc):\n+        \"\"\"\n+        Symmetrically encrypt a document.\n+\n+        :param doc: The document with contents to be encrypted.\n+        :type doc: SoledadDocument\n+        \"\"\"\n+        docstr = doc.get_json()\n+        key = self._crypto.doc_passphrase(doc.doc_id)\n+        secret = self._crypto.secret\n+        args = doc.doc_id, doc.rev, docstr, key, secret\n+\n+        try:\n+            self._pool.apply_async(encrypt_doc_task, args,\n+                                   callback=self.encrypt_doc_cb)\n+        except Exception as exc:\n+            logger.exception(exc)\n+\n+    def encrypt_doc_cb(self, result):\n+        doc_id, doc_rev, content = result\n+        self.insert_encrypted_local_doc(doc_id, doc_rev, content)\n+\n+    def insert_encrypted_local_doc(self, doc_id, doc_rev, content):\n+        \"\"\"\n+        Insert the contents of the encrypted doc into the local sync\n+        database.\n+\n+        :param doc: The document with contents to be encrypted.\n+        :type doc: SoledadDocument\n+        :param content: The encrypted document.\n+        :type content: str\n+        \"\"\"\n+        c = self._sync_db.cursor()\n+        sql_del = \"DELETE FROM '%s' WHERE doc_id=?\" % (self.TABLE_NAME,)\n+        c.execute(sql_del, (doc_id, ))\n+        sql_ins = \"INSERT INTO '%s' VALUES (?, ?, ?)\" % (self.TABLE_NAME,)\n+        c.execute(sql_ins, (doc_id, doc_rev, content))\n+        self._sync_db.commit()\n+\n+\n+def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret):\n+    decrypted_content = decrypt_doc_dict(\n+        content, doc_id, doc_rev, key, secret)\n+    return doc_id, doc_rev, decrypted_content, gen, trans_id\n+\n+\n+def get_insertable_docs_by_gen(expected, got):\n+    \"\"\"\n+    Return a list of documents ready to be inserted. This list is computed\n+    by aligning the expected list with the already gotten docs, and returning\n+    the maximum number of docs that can be processed in the expected order\n+    before finding a gap.\n+\n+    :param expected: A list of generations to be inserted.\n+    :type expected: list\n+\n+    :param got: A dictionary whose values are the docs to be inserted.\n+    :type got: dict\n+    \"\"\"\n+    ordered = [got.get(i) for i in expected]\n+    if None in ordered:\n+        return ordered[:ordered.index(None)]\n+    else:\n+        return ordered\n+\n+\n+class SyncDecrypterPool(SyncEncryptDecryptPool):\n+    \"\"\"\n+    Pool of workers that spawn subprocesses to execute the symmetric decryption\n+    of documents that were received.\n+\n+    The decryption of the received documents is done in two steps:\n+\n+        1. All the encrypted docs are collected, together with their generation\n+           and transaction-id\n+        2. The docs are enqueued for decryption. When completed, they are\n+           inserted following the generation order.\n+    \"\"\"\n+    # TODO implement throttling to reduce cpu usage??\n+    TABLE_NAME = \"docs_received\"\n+    FIELD_NAMES = \"doc_id, rev, content, gen, trans_id\"\n+\n+    write_encrypted_lock = threading.Lock()\n+\n+    def __init__(self, *args, **kwargs):\n+        \"\"\"\n+        Initialize the decrypter pool, and setup a dict for putting the\n+        results of the decrypted docs until they are picked by the insert\n+        routine that gets them in order.\n         \"\"\"\n-        Generate a key for calculating a MAC for a document whose id is\n-        C{doc_id}.\n+        self._insert_doc_cb = kwargs.pop(\"insert_doc_cb\")\n+        SyncEncryptDecryptPool.__init__(self, *args)\n+        self.decrypted_docs = {}\n \n-        The key is derived using HMAC having sha256 as underlying hash\n-        function. The key used for HMAC is the first MAC_KEY_LENGTH characters\n-        of Soledad's storage secret. The HMAC message is C{doc_id}.\n+    def insert_encrypted_received_doc(self, doc_id, doc_rev, content,\n+                                      gen, trans_id):\n+        \"\"\"\n+        Insert a received message with encrypted content, to be decrypted later\n+        on.\n+        \"\"\"\n+        docstr = json.dumps(content)\n+        c = self._sync_db.cursor()\n+        sql_ins = \"INSERT INTO '%s' VALUES (?, ?, ?, ?, ?)\" % (\n+            self.TABLE_NAME,)\n+        c.execute(sql_ins, (doc_id, doc_rev, docstr, gen, trans_id))\n+        self._sync_db.commit()\n+\n+    def delete_encrypted_received_doc(self, doc_id, doc_rev):\n+        \"\"\"\n+        Delete a encrypted received doc after it was inserted into the local\n+        db.\n \n-        :param doc_id: The id of the document.\n+        :param doc_id: Document ID.\n         :type doc_id: str\n+        :param doc_rev: Document revision.\n+        :type doc_rev: str\n+        \"\"\"\n+        c = self._sync_db.cursor()\n+        sql_del = \"DELETE FROM '%s' WHERE doc_id=? AND rev=?\" % (\n+            self.TABLE_NAME,)\n+        c.execute(sql_del, (doc_id, doc_rev))\n+        self._sync_db.commit()\n \n-        :return: The key.\n-        :rtype: str\n+    def decrypt_doc(self, doc_id, rev, source_replica_uid):\n+        \"\"\"\n+        Symmetrically decrypt a document.\n+\n+        :param doc_id: The ID for the document with contents to be encrypted.\n+        :type doc: str\n+        :param rev: The revision of the document.\n+        :type rev: str\n+        :param source_replica_uid:\n+        :type source_replica_uid: str\n+        \"\"\"\n+        self.source_replica_uid = source_replica_uid\n+        if sameProxiedObjects(self._insert_doc_cb.get(source_replica_uid),\n+                              None):\n+            print self._insert_doc_cb\n+            logger.warning(\"No insert_doc callback, skipping decryption.\")\n+            return\n+\n+        # XXX move to get_doc function...\n+        c = self._sync_db.cursor()\n+        sql = \"SELECT * FROM '%s' WHERE doc_id=? AND rev=?\" % (\n+            self.TABLE_NAME,)\n+        c.execute(sql, (doc_id, rev))\n+        res = c.fetchone()\n+        if res is None:\n+            logger.debug(\"Doc %s:%s does not exist in sync db\" % (doc_id, rev))\n+            return\n+\n+        doc_id, rev, docstr, gen, trans_id = res\n+        content = json.loads(docstr)\n+        key = self._crypto.doc_passphrase(doc_id)\n+        secret = self._crypto.secret\n+\n+        args = doc_id, rev, content, gen, trans_id, key, secret\n+\n+        try:\n+            self._pool.apply_async(decrypt_doc_task, args,\n+                                   callback=self.decrypt_doc_cb)\n+        except Exception as exc:\n+            logger.exception(exc)\n+\n+    def decrypt_doc_cb(self, result):\n+        \"\"\"\n+        Temporarily store the decryption result in a dictionary where it will\n+        be picked by process_decrypted.\n \n-        :raise NoSymmetricSecret: if no symmetric secret was supplied.\n+        :param result: the result of the decryption routine.\n+        :type result: tuple\n         \"\"\"\n-        if self.secret is None:\n-            raise NoSymmetricSecret()\n-        return hmac.new(\n-            self.secret[:self.MAC_KEY_LENGTH],\n-            doc_id,\n-            hashlib.sha256).digest()\n+        doc_id, rev, content, gen, trans_id = result\n+        self.decrypted_docs[gen] = result\n \n-    #\n-    # secret setters/getters\n-    #\n+    def get_docs_by_generation(self):\n+        \"\"\"\n+        Get all documents in the received table from the sync db,\n+        ordered by generation.\n \n-    def _get_secret(self):\n-        return self._soledad.storage_secret\n+        :return: list of doc_id, rev, generation\n+        \"\"\"\n+        c = self._sync_db.cursor()\n+        sql = \"SELECT doc_id, rev, gen FROM %s ORDER BY gen\" % (\n+            self.TABLE_NAME,)\n+        c.execute(sql)\n+        return c.fetchall()\n \n-    secret = property(\n-        _get_secret, doc='The secret used for symmetric encryption')\n+    def count_received_encrypted_docs(self):\n+        \"\"\"\n+        Count how many documents we have in the table for received and\n+        encrypted docs.\n+\n+        :return: The count of documents.\n+        :rtype: int\n+        \"\"\"\n+        c = self._sync_db.cursor()\n+        sql = \"SELECT COUNT(*) FROM %s\" % (self.TABLE_NAME,)\n+        c.execute(sql)\n+        res = c.fetchone()\n+        if res is not None:\n+            return res[0]\n+        else:\n+            return 0","path":"client/src/leap/soledad/client/crypto.py","position":767,"original_position":767,"commit_id":"765283349b64d05878e05419d657365f476664f8","original_commit_id":"fdab831595cd6bdfaf11c637059dc831377028a9","user":{"login":"chiiph","id":167183,"avatar_url":"https://avatars.githubusercontent.com/u/167183?","gravatar_id":"633815b888d3e37c8d8403f0211e3fff","url":"https://api.github.com/users/chiiph","html_url":"https://github.com/chiiph","followers_url":"https://api.github.com/users/chiiph/followers","following_url":"https://api.github.com/users/chiiph/following{/other_user}","gists_url":"https://api.github.com/users/chiiph/gists{/gist_id}","starred_url":"https://api.github.com/users/chiiph/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chiiph/subscriptions","organizations_url":"https://api.github.com/users/chiiph/orgs","repos_url":"https://api.github.com/users/chiiph/repos","events_url":"https://api.github.com/users/chiiph/events{/privacy}","received_events_url":"https://api.github.com/users/chiiph/received_events","type":"User","site_admin":false},"body":"Ok, now I know you're an impostor for kali, he would've done `return res[0] if res is not None else 0` :)","created_at":"2014-03-18T19:48:40Z","updated_at":"2014-03-18T19:48:40Z","html_url":"https://github.com/leapcode/soledad/pull/140#discussion_r10722127","pull_request_url":"https://api.github.com/repos/leapcode/soledad/pulls/140","_links":{"self":{"href":"https://api.github.com/repos/leapcode/soledad/pulls/comments/10722127"},"html":{"href":"https://github.com/leapcode/soledad/pull/140#discussion_r10722127"},"pull_request":{"href":"https://api.github.com/repos/leapcode/soledad/pulls/140"}}}}