CVE-2012-1176: Improper Restriction of Operations within the Bounds of a Memory Buffer (CWE-119)
value: Buffer overflow in the fribidi_utf8_to_unicode function in PyFriBidi before 0.11.0 allows remote attackers to cause a denial of service (application crash) via a 4-byte utf-8 sequence.
CWE description: The product performs operations on a memory buffer, but it reads from or writes to a memory location outside the buffer's intended boundary. This may result in read or write operations on unexpected memory locations that could be linked to other variables, data structures, or internal program data.
Published: 2012-08-26T20:55Z
Repository: https://github.com/pediapress/pyfribidi
Language: Python
Files changed: MANIFEST.in, pyfribidi.c, pyfribidi.py, setup.py
Commit date: 2012-03-09 01:00:43 +0100
Commit message: refactor pyfribidi.c module

pyfribidi.c is now compiled as _pyfribidi. This module only handles
unicode internally and doesn't use the fribidi_utf8_to_unicode
function (which can't handle 4 byte utf-8 sequences). This fixes the
buffer overflow in issue #2.

The code is now also much simpler: pyfribidi.c is down from 280 to 130
lines of code.

We now ship a pure python pyfribidi that handles the case when
non-unicode strings are passed in.

We now also adapt the size of the output string if clean=True is
passed.

Diff: diff --git a/MANIFEST.in b/MANIFEST.in
index 1fdeb66..1eb57f0 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -158,11 +158,11 @@ include fribidi-src/test/test_UTF-8_persian.input
 include fribidi-src/test/test_UTF-8_persian.reference
 include fribidi-src/test/test_UTF-8_reordernsm.input
 include fribidi-src/test/test_UTF-8_reordernsm.reference
 include fribidi_systray.py
 include pyfribidi.c
-include pyfribidi.h
+include pyfribidi.py
 include pyfribidi2.py
 include setup.cfg
 include setup.py
 include test_pyfribidi.py
 include time_pyfribidi.py
diff --git a/pyfribidi.c b/pyfribidi.c
index ee17df0..82c4092 100644
--- a/pyfribidi.c
+++ b/pyfribidi.c
@@ -15,269 +15,118 @@
  *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  */
 
 /* Copyright (C) 2005,2006,2010 Yaacov Zamir, Nir Soffer */
 
-/* 	FriBidi python binding:
-    
-    Install:
-	python setup.py install
-    
- */
 
-#include <Python.h>		/* must be first */
+#include <Python.h>
 #include <fribidi.h>
-#undef _POSIX_C_SOURCE
-
-#include "pyfribidi.h"
 
-#define MAX_STR_LEN 65000
 
 static PyObject *
-_pyfribidi_log2vis (PyObject * self, PyObject * args, PyObject * kw)
+unicode_log2vis (PyUnicodeObject* string,
+                 FriBidiParType base_direction, int clean, int reordernsm)
 {
-	PyObject *logical = NULL;	/* input unicode or string object */
-	FriBidiParType base = FRIBIDI_TYPE_RTL;	/* optional direction */
-	const char *encoding = "utf-8";	/* optional input string encoding */
-	int clean = 0; /* optional flag to clean the string */
-	int reordernsm = 1; /* optional flag to allow reordering of non spacing marks*/
-
-	static char *kwargs[] =
-	        { "logical", "base_direction", "encoding", "clean", "reordernsm", NULL };
-
-        if (!PyArg_ParseTupleAndKeywords (args, kw, "O|isii", kwargs,
-					  &logical, &base, &encoding, &clean, &reordernsm))
-		return NULL;
-
-	/* Validate base */
-
-	if (!(base == FRIBIDI_TYPE_RTL ||
-	      base == FRIBIDI_TYPE_LTR || base == FRIBIDI_TYPE_ON))
-		return PyErr_Format (PyExc_ValueError,
-				     "invalid value %d: use either RTL, LTR or ON",
-				     base);
-
-	/* Check object type and delegate to one of the log2vis functions */
-
-	if (PyUnicode_Check (logical))
-	        return log2vis_unicode (logical, base, clean, reordernsm);
-	else if (PyString_Check (logical))
-	        return log2vis_encoded_string (logical, encoding, base, clean, reordernsm);
-	else
-		return PyErr_Format (PyExc_TypeError,
-				     "expected unicode or str, not %s",
-				     logical->ob_type->tp_name);
+    int i;
+    int length = string->length;
+    FriBidiChar *logical = NULL; /* input fribidi unicode buffer */
+    FriBidiChar *visual = NULL;      /* output fribidi unicode buffer */
+    FriBidiStrIndex new_len = 0; /* length of the UTF-8 buffer */
+    PyUnicodeObject *result = NULL;
+
+    /* Allocate fribidi unicode buffers
+       TODO - Don't copy strings if sizeof(FriBidiChar) == sizeof(Py_UNICODE)
+    */
+
+    logical = PyMem_New (FriBidiChar, length + 1);
+    if (logical == NULL) {
+        PyErr_NoMemory();
+        goto cleanup;
+    }
+
+    visual = PyMem_New (FriBidiChar, length + 1);
+    if (visual == NULL) {
+        PyErr_NoMemory();
+        goto cleanup;
+    }
+
+    for (i=0; i<length; ++i) {
+        logical[i] = string->str[i];
+    }
+
+    /* Convert to unicode and order visually */
+    fribidi_set_reorder_nsm(reordernsm);
+
+    if (!fribidi_log2vis (logical, length, &base_direction, visual,
+                          NULL, NULL, NULL)) {
+
+        PyErr_SetString (PyExc_RuntimeError,
+                         "fribidi failed to order string");
+        goto cleanup;
+    }
+
+    /* Cleanup the string if requested */
+    if (clean) {
+        length = fribidi_remove_bidi_marks (visual, length, NULL, NULL, NULL);
+    }
+
+    result = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, length);
+    if (result == NULL) {
+        goto cleanup;
+    }
+
+    for (i=0; i<length; ++i) {
+        result->str[i] = visual[i];
+    }
+
+  cleanup:
+    /* Delete unicode buffers */
+    PyMem_Del (logical);
+    PyMem_Del (visual);
+
+    return (PyObject *)result;
 }
 
-/*
-  log2vis_unicode - reorder unicode string visually
-
-  Return value: new reference
-
-  Return Python unicode object ordered visually or NULL if an exception
-  was raised.
-
-  Since Python and fribidi don't now know each other unicode format,
-  encode input string as utf-8 and invoke log2vis_utf8.
-  
-  Arguments:
-  
-  - unicode: Python unicode object
-  - base_direction: input string base direction, e.g right to left  
-*/
-
 static PyObject *
-log2vis_unicode (PyObject * unicode, FriBidiParType base_direction, int clean, int reordernsm)
+_pyfribidi_log2vis (PyObject * self, PyObject * args, PyObject * kw)
 {
-	PyObject *logical = NULL;	/* input string encoded in utf-8 */
-	PyObject *visual = NULL;	/* output string encoded in utf-8 */
-	PyObject *result = NULL;	/* unicode output string */
-
-	int length = PyUnicode_GET_SIZE (unicode);
-
-	logical = PyUnicode_AsUTF8String (unicode);
-	if (logical == NULL)
-		goto cleanup;
-
-	visual = log2vis_utf8 (logical, length, base_direction, clean, reordernsm);
-	if (visual == NULL)
-		goto cleanup;
-
-	result = PyUnicode_DecodeUTF8 (PyString_AS_STRING (visual),
-				       PyString_GET_SIZE (visual), "strict");
-
-      cleanup:
-	Py_XDECREF (logical);
-	Py_XDECREF (visual);
-
-	return result;
+    PyUnicodeObject *logical = NULL;	/* input unicode or string object */
+    FriBidiParType base = FRIBIDI_TYPE_RTL;	/* optional direction */
+    int clean = 0; /* optional flag to clean the string */
+    int reordernsm = 1; /* optional flag to allow reordering of non spacing marks*/
+
+    static char *kwargs[] =
+        { "logical", "base_direction", "clean", "reordernsm", NULL };
+
+    if (!PyArg_ParseTupleAndKeywords (args, kw, "U|iii", kwargs,
+                                      &logical, &base, &clean, &reordernsm)) {
+        return NULL;
+    }
+
+    /* Validate base */
+
+    if (!(base == FRIBIDI_TYPE_RTL
+          || base == FRIBIDI_TYPE_LTR
+          || base == FRIBIDI_TYPE_ON)) {
+        return PyErr_Format (PyExc_ValueError,
+                             "invalid value %d: use either RTL, LTR or ON",
+                             base);
+    }
+
+    return unicode_log2vis (logical, base, clean, reordernsm);
 }
 
-/*
-  log2vis_encoded_string - reorder encoded string visually
-
-  Return value: new reference
-
-  Return Python string object ordered visually or NULL if an exception
-  was raised. The returned string use the same encoding.
-
-  Invoke either log2vis_utf8 or log2vis_unicode.
-
-  - string: Python string object using encoding
-  - encoding: string encoding, any encoding name known to Python
-  - base_direction: input string base direction, e.g right to left
- */
-
-static PyObject *
-log2vis_encoded_string (PyObject * string, const char *encoding,
-			FriBidiParType base_direction, int clean, int reordernsm)
-{
-	PyObject *logical = NULL;	/* logical unicode object */
-	PyObject *result = NULL;	/* output string object */
-
-	/* Always needed for the string length */
-	logical = PyUnicode_Decode (PyString_AS_STRING (string),
-				    PyString_GET_SIZE (string),
-				    encoding, "strict");
-	if (logical == NULL)
-		return NULL;
-
-	if (strcmp (encoding, "utf-8") == 0)
-		/* Shortcut for utf8 strings (little faster) */
-		result = log2vis_utf8 (string,
-				       PyUnicode_GET_SIZE (logical),
-				       base_direction, clean, reordernsm);
-	else
-	{
-		/* Invoke log2vis_unicode and encode back to encoding */
-
-		PyObject *visual = log2vis_unicode (logical, base_direction, clean, reordernsm);
-
-		if (visual)
-		{
-			result = PyUnicode_Encode (PyUnicode_AS_UNICODE
-						   (visual),
-						   PyUnicode_GET_SIZE (visual),
-						   encoding, "strict");
-			Py_DECREF (visual);
-		}
-	}
-
-	Py_DECREF (logical);
-
-	return result;
-}
-
-/*
-  log2vis_utf8 - reorder string visually
-
-  Return value: new reference
-
-  Return Python string object ordered visually or NULL if an exception
-  was raised.
-  
-  Arguments:
-  
-  - string: Python string object using utf-8 encoding
-  - unicode_length: number of characters in string. This is not the
-    number of bytes in the string, which may be much bigger than the
-    number of characters, because utf-8 uses 1-4 bytes per character.
-  - base_direction: input string base direction, e.g right to left
-*/
-
-static PyObject *
-log2vis_utf8 (PyObject * string, int unicode_length,
-	      FriBidiParType base_direction, int clean, int reordernsm)
-{
-	FriBidiChar *logical = NULL; /* input fribidi unicode buffer */
-	FriBidiChar *visual = NULL;	 /* output fribidi unicode buffer */
-	char *visual_utf8 = NULL;    /* output fribidi UTF-8 buffer */
-	FriBidiStrIndex new_len = 0; /* length of the UTF-8 buffer */
-	PyObject *result = NULL;	 /* failure */
-
-	/* Allocate fribidi unicode buffers */
-
-	logical = PyMem_New (FriBidiChar, unicode_length + 1);
-	if (logical == NULL)
-	{
-		PyErr_SetString (PyExc_MemoryError,
-				 "failed to allocate unicode buffer");
-		goto cleanup;
-	}
-
-	visual = PyMem_New (FriBidiChar, unicode_length + 1);
-	if (visual == NULL)
-	{
-		PyErr_SetString (PyExc_MemoryError,
-				 "failed to allocate unicode buffer");
-		goto cleanup;
-	}
-
-	/* Convert to unicode and order visually */
-	fribidi_set_reorder_nsm(reordernsm);
-	fribidi_utf8_to_unicode (PyString_AS_STRING (string),
-				 PyString_GET_SIZE (string), logical);
-
-	if (!fribidi_log2vis (logical, unicode_length, &base_direction, visual,
-			      NULL, NULL, NULL))
-	{
-		PyErr_SetString (PyExc_RuntimeError,
-				 "fribidi failed to order string");
-		goto cleanup;
-	}
-
-	/* Cleanup the string if requested */
-	if (clean)
-		fribidi_remove_bidi_marks (visual, unicode_length, NULL, NULL, NULL);
-
-	/* Allocate fribidi UTF-8 buffer */
-
-	visual_utf8 = PyMem_New(char, (unicode_length * 4)+1);
-	if (visual_utf8 == NULL)
-	{
-		PyErr_SetString (PyExc_MemoryError,
-				"failed to allocate UTF-8 buffer");
-		goto cleanup;
-	}
-
-	/* Encode the reordered string  and create result string */
-
-	new_len = fribidi_unicode_to_utf8 (visual, unicode_length, visual_utf8);
-
-	result = PyString_FromStringAndSize (visual_utf8, new_len);
-	if (result == NULL)
-		/* XXX does it raise any error? */
-		goto cleanup;
-
-      cleanup:
-	/* Delete unicode buffers */
-	PyMem_Del (logical);
-	PyMem_Del (visual);
-	PyMem_Del (visual_utf8);
-
-	return result;
-}
 
 static PyMethodDef PyfribidiMethods[] = {
-	{"log2vis", (PyCFunction) _pyfribidi_log2vis,
-	 METH_VARARGS | METH_KEYWORDS,
-	 _pyfribidi_log2vis__doc__},
+        {"log2vis", (PyCFunction) _pyfribidi_log2vis, METH_VARARGS | METH_KEYWORDS, NULL},
 	{NULL, NULL, 0, NULL}
 };
 
+
 PyMODINIT_FUNC
-initpyfribidi (void)
+init_pyfribidi (void)
 {
-	PyObject *module;
-
-	/* XXX What should be done if we fail here? */
-
-	module = Py_InitModule3 ("pyfribidi", PyfribidiMethods,
-				 _pyfribidi__doc__);
-
+        PyObject *module = Py_InitModule ("_pyfribidi", PyfribidiMethods);
 	PyModule_AddIntConstant (module, "RTL", (long) FRIBIDI_TYPE_RTL);
 	PyModule_AddIntConstant (module, "LTR", (long) FRIBIDI_TYPE_LTR);
 	PyModule_AddIntConstant (module, "ON", (long) FRIBIDI_TYPE_ON);
-
-	PyModule_AddStringConstant (module, "__author__",
-				    "Yaacov Zamir and Nir Soffer");
 }
diff --git a/pyfribidi.h b/pyfribidi.h
deleted file mode 100644
index 3159b57..0000000
--- a/pyfribidi.h
+++ /dev/null
@@ -1,66 +0,0 @@
-
-/*
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; either version 2 of the License, or
- *  (at your option) any later version.
- *
- *  This program is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *  GNU General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License
- *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- */
-
-/* Copyright (C) 2005,2006,2010 Yaacov Zamir, Nir Soffer */
-
-/* 	FriBidi python binding:
-    
-    Install:
-	python setup.py install
-    
- */
-
-#ifndef __PYFRIBIDI_H__
-#define __PYFRIBIDI_H__
-
-static PyObject *_pyfribidi_log2vis (PyObject * self, PyObject * args,
-				     PyObject * kw);
-
-static PyObject *log2vis_unicode (PyObject * unicode,
-				  FriBidiParType base_direction, int clean, int reordernsm);
-
-static PyObject *log2vis_encoded_string (PyObject * string,
-					 const char *encoding,
-					 FriBidiParType base_direction, int clean, int reordernsm);
-
-static PyObject *log2vis_utf8 (PyObject * string, int unicode_length,
-			       FriBidiParType base_direction, int clean, int reordernsm);
-
-PyMODINIT_FUNC initpyfribidi (void);
-
-PyDoc_STRVAR (_pyfribidi__doc__,
-	      "simple Python binding for fribidi.\n\n"
-	      "pyfribidi uses libfribidi to order text visually using the unicode\n"
-	      "algorithm. pyfribidi can also convert text from visual order to\n"
-	      "logical order, but the conversion may be wrong in certain cases.\n");
-
-/* Based on fribidi-0.10.7 README */
-PyDoc_STRVAR (_pyfribidi_log2vis__doc__,
-	      "log2vis(logical, base_direction=RTL , encoding='utf-8') -> visual\n\n"
-	      "Return string reordered visually according to base direction.\n"
-	      "Return the same type of input string, either unicode or string using\n"
-	      "encoding.\n\n"
-	      "Note that this function does not handle line breaking. You should\n"
-	      "call log2vis with each line.\n\n"
-	      "Arguments:\n\n"
-	      "- logical: unicode or encoded string\n"
-	      "- base_direction: optional logical base direction. Accepts one of\n"
-	      "  the constants LTR, RTL or ON, defined in this module. ON calculate\n"
-	      "  the base direction according to the BiDi algorithm.\n"
-	      "- encoding: optional string encoding (ignored for unicode input)\n");
-
-#endif /* __PYFRIBIDI_H__ */
diff --git a/pyfribidi.py b/pyfribidi.py
new file mode 100644
index 0000000..99785cb
--- /dev/null
+++ b/pyfribidi.py
@@ -0,0 +1,34 @@
+"""simple Python binding for fribidi.
+
+pyfribidi uses libfribidi to order text visually using the unicode
+algorithm. pyfribidi can also convert text from visual order to
+logical order, but the conversion may be wrong in certain cases.
+"""
+
+from _pyfribidi import LTR, ON, RTL, log2vis as _log2vis
+
+def log2vis(logical, base_direction=RTL, encoding="utf-8", clean=False, reordernsm=True):
+    """
+    Return string reordered visually according to base direction.
+    Return the same type of input string, either unicode or string using
+    encoding.
+
+    Note that this function does not handle line breaking. You should
+    call log2vis with each line.
+
+    Arguments:
+    - logical: unicode or encoded string
+    - base_direction: optional logical base direction. Accepts one of
+      the constants LTR, RTL or ON, defined in this module. ON calculate
+      the base direction according to the BiDi algorithm.
+    - encoding: optional string encoding (ignored for unicode input)
+    """
+
+    if not isinstance(logical, unicode):
+        logical = unicode(logical, encoding)
+    else:
+        encoding = None
+    res = _log2vis(logical, base_direction=base_direction, clean=clean, reordernsm=reordernsm)
+    if encoding:
+        return res.encode(encoding)
+    return res
diff --git a/setup.py b/setup.py
index bc47a2f..3e0e24a 100755
--- a/setup.py
+++ b/setup.py
@@ -92,12 +92,12 @@ setup(name="pyfribidi",
       author_email="kzamir@walla.co.il",
       url="http://hspell-gui.sourceforge.net/pyfribidi.html",
       license="GPL",
       cmdclass={'build_ext': my_build_ext},
       long_description=read_long_description(),
-      py_modules=["pyfribidi2"],
+      py_modules=["pyfribidi", "pyfribidi2"],
       ext_modules=[Extension(
-            name='pyfribidi',
+            name='_pyfribidi',
             sources=['pyfribidi.c'] + lib_sources,
             define_macros=define_macros,
             libraries=libraries,
             include_dirs=include_dirs)])

Vulnerable code: include pyfribidi.h
/* 	FriBidi python binding:
    
    Install:
	python setup.py install
    
 */
#include <Python.h>		/* must be first */
#undef _POSIX_C_SOURCE

#include "pyfribidi.h"
#define MAX_STR_LEN 65000
_pyfribidi_log2vis (PyObject * self, PyObject * args, PyObject * kw)
	PyObject *logical = NULL;	/* input unicode or string object */
	FriBidiParType base = FRIBIDI_TYPE_RTL;	/* optional direction */
	const char *encoding = "utf-8";	/* optional input string encoding */
	int clean = 0; /* optional flag to clean the string */
	int reordernsm = 1; /* optional flag to allow reordering of non spacing marks*/

	static char *kwargs[] =
	        { "logical", "base_direction", "encoding", "clean", "reordernsm", NULL };

        if (!PyArg_ParseTupleAndKeywords (args, kw, "O|isii", kwargs,
					  &logical, &base, &encoding, &clean, &reordernsm))
		return NULL;

	/* Validate base */

	if (!(base == FRIBIDI_TYPE_RTL ||
	      base == FRIBIDI_TYPE_LTR || base == FRIBIDI_TYPE_ON))
		return PyErr_Format (PyExc_ValueError,
				     "invalid value %d: use either RTL, LTR or ON",
				     base);

	/* Check object type and delegate to one of the log2vis functions */

	if (PyUnicode_Check (logical))
	        return log2vis_unicode (logical, base, clean, reordernsm);
	else if (PyString_Check (logical))
	        return log2vis_encoded_string (logical, encoding, base, clean, reordernsm);
	else
		return PyErr_Format (PyExc_TypeError,
				     "expected unicode or str, not %s",
				     logical->ob_type->tp_name);
/*
  log2vis_unicode - reorder unicode string visually

  Return value: new reference

  Return Python unicode object ordered visually or NULL if an exception
  was raised.

  Since Python and fribidi don't now know each other unicode format,
  encode input string as utf-8 and invoke log2vis_utf8.
  
  Arguments:
  
  - unicode: Python unicode object
  - base_direction: input string base direction, e.g right to left  
*/

log2vis_unicode (PyObject * unicode, FriBidiParType base_direction, int clean, int reordernsm)
	PyObject *logical = NULL;	/* input string encoded in utf-8 */
	PyObject *visual = NULL;	/* output string encoded in utf-8 */
	PyObject *result = NULL;	/* unicode output string */

	int length = PyUnicode_GET_SIZE (unicode);

	logical = PyUnicode_AsUTF8String (unicode);
	if (logical == NULL)
		goto cleanup;

	visual = log2vis_utf8 (logical, length, base_direction, clean, reordernsm);
	if (visual == NULL)
		goto cleanup;

	result = PyUnicode_DecodeUTF8 (PyString_AS_STRING (visual),
				       PyString_GET_SIZE (visual), "strict");

      cleanup:
	Py_XDECREF (logical);
	Py_XDECREF (visual);

	return result;
/*
  log2vis_encoded_string - reorder encoded string visually

  Return value: new reference

  Return Python string object ordered visually or NULL if an exception
  was raised. The returned string use the same encoding.

  Invoke either log2vis_utf8 or log2vis_unicode.

  - string: Python string object using encoding
  - encoding: string encoding, any encoding name known to Python
  - base_direction: input string base direction, e.g right to left
 */

static PyObject *
log2vis_encoded_string (PyObject * string, const char *encoding,
			FriBidiParType base_direction, int clean, int reordernsm)
{
	PyObject *logical = NULL;	/* logical unicode object */
	PyObject *result = NULL;	/* output string object */

	/* Always needed for the string length */
	logical = PyUnicode_Decode (PyString_AS_STRING (string),
				    PyString_GET_SIZE (string),
				    encoding, "strict");
	if (logical == NULL)
		return NULL;

	if (strcmp (encoding, "utf-8") == 0)
		/* Shortcut for utf8 strings (little faster) */
		result = log2vis_utf8 (string,
				       PyUnicode_GET_SIZE (logical),
				       base_direction, clean, reordernsm);
	else
	{
		/* Invoke log2vis_unicode and encode back to encoding */

		PyObject *visual = log2vis_unicode (logical, base_direction, clean, reordernsm);

		if (visual)
		{
			result = PyUnicode_Encode (PyUnicode_AS_UNICODE
						   (visual),
						   PyUnicode_GET_SIZE (visual),
						   encoding, "strict");
			Py_DECREF (visual);
		}
	}

	Py_DECREF (logical);

	return result;
}

/*
  log2vis_utf8 - reorder string visually

  Return value: new reference

  Return Python string object ordered visually or NULL if an exception
  was raised.
  
  Arguments:
  
  - string: Python string object using utf-8 encoding
  - unicode_length: number of characters in string. This is not the
    number of bytes in the string, which may be much bigger than the
    number of characters, because utf-8 uses 1-4 bytes per character.
  - base_direction: input string base direction, e.g right to left
*/

static PyObject *
log2vis_utf8 (PyObject * string, int unicode_length,
	      FriBidiParType base_direction, int clean, int reordernsm)
{
	FriBidiChar *logical = NULL; /* input fribidi unicode buffer */
	FriBidiChar *visual = NULL;	 /* output fribidi unicode buffer */
	char *visual_utf8 = NULL;    /* output fribidi UTF-8 buffer */
	FriBidiStrIndex new_len = 0; /* length of the UTF-8 buffer */
	PyObject *result = NULL;	 /* failure */

	/* Allocate fribidi unicode buffers */

	logical = PyMem_New (FriBidiChar, unicode_length + 1);
	if (logical == NULL)
	{
		PyErr_SetString (PyExc_MemoryError,
				 "failed to allocate unicode buffer");
		goto cleanup;
	}

	visual = PyMem_New (FriBidiChar, unicode_length + 1);
	if (visual == NULL)
	{
		PyErr_SetString (PyExc_MemoryError,
				 "failed to allocate unicode buffer");
		goto cleanup;
	}

	/* Convert to unicode and order visually */
	fribidi_set_reorder_nsm(reordernsm);
	fribidi_utf8_to_unicode (PyString_AS_STRING (string),
				 PyString_GET_SIZE (string), logical);

	if (!fribidi_log2vis (logical, unicode_length, &base_direction, visual,
			      NULL, NULL, NULL))
	{
		PyErr_SetString (PyExc_RuntimeError,
				 "fribidi failed to order string");
		goto cleanup;
	}

	/* Cleanup the string if requested */
	if (clean)
		fribidi_remove_bidi_marks (visual, unicode_length, NULL, NULL, NULL);

	/* Allocate fribidi UTF-8 buffer */

	visual_utf8 = PyMem_New(char, (unicode_length * 4)+1);
	if (visual_utf8 == NULL)
	{
		PyErr_SetString (PyExc_MemoryError,
				"failed to allocate UTF-8 buffer");
		goto cleanup;
	}

	/* Encode the reordered string  and create result string */

	new_len = fribidi_unicode_to_utf8 (visual, unicode_length, visual_utf8);

	result = PyString_FromStringAndSize (visual_utf8, new_len);
	if (result == NULL)
		/* XXX does it raise any error? */
		goto cleanup;

      cleanup:
	/* Delete unicode buffers */
	PyMem_Del (logical);
	PyMem_Del (visual);
	PyMem_Del (visual_utf8);

	return result;
}
	{"log2vis", (PyCFunction) _pyfribidi_log2vis,
	 METH_VARARGS | METH_KEYWORDS,
	 _pyfribidi_log2vis__doc__},
initpyfribidi (void)
	PyObject *module;

	/* XXX What should be done if we fail here? */

	module = Py_InitModule3 ("pyfribidi", PyfribidiMethods,
				 _pyfribidi__doc__);


	PyModule_AddStringConstant (module, "__author__",
				    "Yaacov Zamir and Nir Soffer");

/*
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 */

/* Copyright (C) 2005,2006,2010 Yaacov Zamir, Nir Soffer */

/* 	FriBidi python binding:
    
    Install:
	python setup.py install
    
 */

#ifndef __PYFRIBIDI_H__
#define __PYFRIBIDI_H__

static PyObject *_pyfribidi_log2vis (PyObject * self, PyObject * args,
				     PyObject * kw);

static PyObject *log2vis_unicode (PyObject * unicode,
				  FriBidiParType base_direction, int clean, int reordernsm);

static PyObject *log2vis_encoded_string (PyObject * string,
					 const char *encoding,
					 FriBidiParType base_direction, int clean, int reordernsm);

static PyObject *log2vis_utf8 (PyObject * string, int unicode_length,
			       FriBidiParType base_direction, int clean, int reordernsm);

PyMODINIT_FUNC initpyfribidi (void);

PyDoc_STRVAR (_pyfribidi__doc__,
	      "simple Python binding for fribidi.\n\n"
	      "pyfribidi uses libfribidi to order text visually using the unicode\n"
	      "algorithm. pyfribidi can also convert text from visual order to\n"
	      "logical order, but the conversion may be wrong in certain cases.\n");

/* Based on fribidi-0.10.7 README */
PyDoc_STRVAR (_pyfribidi_log2vis__doc__,
	      "log2vis(logical, base_direction=RTL , encoding='utf-8') -> visual\n\n"
	      "Return string reordered visually according to base direction.\n"
	      "Return the same type of input string, either unicode or string using\n"
	      "encoding.\n\n"
	      "Note that this function does not handle line breaking. You should\n"
	      "call log2vis with each line.\n\n"
	      "Arguments:\n\n"
	      "- logical: unicode or encoded string\n"
	      "- base_direction: optional logical base direction. Accepts one of\n"
	      "  the constants LTR, RTL or ON, defined in this module. ON calculate\n"
	      "  the base direction according to the BiDi algorithm.\n"
	      "- encoding: optional string encoding (ignored for unicode input)\n");

#endif /* __PYFRIBIDI_H__ */
      py_modules=["pyfribidi2"],
            name='pyfribidi',
Fixed code: include pyfribidi.py
#include <Python.h>
unicode_log2vis (PyUnicodeObject* string,
                 FriBidiParType base_direction, int clean, int reordernsm)
    int i;
    int length = string->length;
    FriBidiChar *logical = NULL; /* input fribidi unicode buffer */
    FriBidiChar *visual = NULL;      /* output fribidi unicode buffer */
    FriBidiStrIndex new_len = 0; /* length of the UTF-8 buffer */
    PyUnicodeObject *result = NULL;

    /* Allocate fribidi unicode buffers
       TODO - Don't copy strings if sizeof(FriBidiChar) == sizeof(Py_UNICODE)
    */

    logical = PyMem_New (FriBidiChar, length + 1);
    if (logical == NULL) {
        PyErr_NoMemory();
        goto cleanup;
    }

    visual = PyMem_New (FriBidiChar, length + 1);
    if (visual == NULL) {
        PyErr_NoMemory();
        goto cleanup;
    }

    for (i=0; i<length; ++i) {
        logical[i] = string->str[i];
    }

    /* Convert to unicode and order visually */
    fribidi_set_reorder_nsm(reordernsm);

    if (!fribidi_log2vis (logical, length, &base_direction, visual,
                          NULL, NULL, NULL)) {

        PyErr_SetString (PyExc_RuntimeError,
                         "fribidi failed to order string");
        goto cleanup;
    }

    /* Cleanup the string if requested */
    if (clean) {
        length = fribidi_remove_bidi_marks (visual, length, NULL, NULL, NULL);
    }

    result = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, length);
    if (result == NULL) {
        goto cleanup;
    }

    for (i=0; i<length; ++i) {
        result->str[i] = visual[i];
    }

  cleanup:
    /* Delete unicode buffers */
    PyMem_Del (logical);
    PyMem_Del (visual);

    return (PyObject *)result;
_pyfribidi_log2vis (PyObject * self, PyObject * args, PyObject * kw)
    PyUnicodeObject *logical = NULL;	/* input unicode or string object */
    FriBidiParType base = FRIBIDI_TYPE_RTL;	/* optional direction */
    int clean = 0; /* optional flag to clean the string */
    int reordernsm = 1; /* optional flag to allow reordering of non spacing marks*/

    static char *kwargs[] =
        { "logical", "base_direction", "clean", "reordernsm", NULL };

    if (!PyArg_ParseTupleAndKeywords (args, kw, "U|iii", kwargs,
                                      &logical, &base, &clean, &reordernsm)) {
        return NULL;
    }

    /* Validate base */

    if (!(base == FRIBIDI_TYPE_RTL
          || base == FRIBIDI_TYPE_LTR
          || base == FRIBIDI_TYPE_ON)) {
        return PyErr_Format (PyExc_ValueError,
                             "invalid value %d: use either RTL, LTR or ON",
                             base);
    }

    return unicode_log2vis (logical, base, clean, reordernsm);
        {"log2vis", (PyCFunction) _pyfribidi_log2vis, METH_VARARGS | METH_KEYWORDS, NULL},

init_pyfribidi (void)
        PyObject *module = Py_InitModule ("_pyfribidi", PyfribidiMethods);
"""simple Python binding for fribidi.

pyfribidi uses libfribidi to order text visually using the unicode
algorithm. pyfribidi can also convert text from visual order to
logical order, but the conversion may be wrong in certain cases.
"""

from _pyfribidi import LTR, ON, RTL, log2vis as _log2vis

def log2vis(logical, base_direction=RTL, encoding="utf-8", clean=False, reordernsm=True):
    """
    Return string reordered visually according to base direction.
    Return the same type of input string, either unicode or string using
    encoding.

    Note that this function does not handle line breaking. You should
    call log2vis with each line.

    Arguments:
    - logical: unicode or encoded string
    - base_direction: optional logical base direction. Accepts one of
      the constants LTR, RTL or ON, defined in this module. ON calculate
      the base direction according to the BiDi algorithm.
    - encoding: optional string encoding (ignored for unicode input)
    """

    if not isinstance(logical, unicode):
        logical = unicode(logical, encoding)
    else:
        encoding = None
    res = _log2vis(logical, base_direction=base_direction, clean=clean, reordernsm=reordernsm)
    if encoding:
        return res.encode(encoding)
    return res
      py_modules=["pyfribidi", "pyfribidi2"],
            name='_pyfribidi',