Repository: flightaware/tohil (language: python)

Issue:
Running multiple lines of python via tohil::exec/eval is a bit clunky
```tcl
% package require tohil
4.1.0
% tohil::exec {
    def fn():
        pass
}
unexpected indent (<string>, line 2)
%
```
Python wants your globally scoped code to have no extraneous indentation.

This issue doesn't demonstrate terribly well in a shell where it's silly to indent your code, but consider the case where you're running some python within a tcl proc. Suddenly you'll need to remove all indentation from your python code for it to run correctly and you're left with this unfortunate-looking code
```tcl
proc dostuff {} {
	tohil::exec {
def fn():
	pass
}
}
```

tcllib's [undent command](https://core.tcl-lang.org/tcllib/doc/trunk/embedded/md/tcllib/files/modules/textutil/textutil.md#6) can handle removing the leading indentation properly, as can [python's dedent function](https://docs.python.org/3/library/textwrap.html#textwrap.dedent). Is there any way this whitespace can be stripped automatically in tohil's exec and eval commands? I'd accept an opt-in flag for the behavior or even a utility proc (I'm just not terribly interested in forcing a tcllib dependency for this).


Discussion / hints:
Hmm, yeah, pretty easy to solve in C to avoid a dependency.  I agree this seems like a good idea.  I'm wondering since it's an IndentationError to have any extraneous indent in there, if it even needs a switch or if it should just function that way all the time.

How about if we make the matching more stringent than undent, insofar as whatever whitespace we see on the first line is expected to be present on each additional line (except blank lines, which we'll ignore) and if it isn't, we'll error?
The more stringent approach fared well in private conversations with nasageek and another.  I foresee something in C that figures out the whitespace of the first line that isn't empty, copies it sans leading whitspace and all of any following lines stripping the same whitespace, making it an error if any of the following non-empty lines don't have the exact same whitespace, making a new string or Tcl object or dstring or whatever, containing the unindented code.

If it immediately detects that there is no leading whitespace on the first non-empty line, it might ought to give up right away and avoid copying the string if that's easy to implement.

I think this should only apply to tohil::exec as I don't see a legal way to call tohil::eval with a multiline sequence unless there's like a triple-quoted string argument or something, but even so, we about for sure don't want to do any unindenting on such a string.
What I've done in the past is something like:

```
proc dostuff {} {
	tohil::exec {
                |def fn():
                |	pass
        }
}
```

Where if the first nonblank character is | it deletes `$whitespace|` from the beginning of every line.
I'd rather have the implicit enforcement of consistent leading whitespace without the need for extra marker characters.
This was for generated HTML and I found the markers really helpful in keeping track of the indentation when working on the code.

Fix patch:
diff --git a/Doc/reference/tohil_tcl_functions.rst b/Doc/reference/tohil_tcl_functions.rst
index e26d2f3..100ba7a 100644
--- a/Doc/reference/tohil_tcl_functions.rst
+++ b/Doc/reference/tohil_tcl_functions.rst
@@ -52,6 +52,11 @@ from Tcl interpreter, the following commands are available:
    to a string, or whatever, in the normal Python manner.
    *tohil::run*, in fact, provides a way to do this.
 
+   To make it easier to comply with Python indentation rules, if the first
+   nonblank line starts with whitespace, exec will un-indent the code block
+   so the first line is not indented at all and following lines are undented
+   to match.
+
 .. function:: tohil::import module
 
    Import the specified module into the globals of the Python interpreter.
diff --git a/generic/tohil.c b/generic/tohil.c
index 1d0c39d..b4c0a81 100644
--- a/generic/tohil.c
+++ b/generic/tohil.c
@@ -85,7 +85,7 @@ static int
 tohil_TclObjIsNoneSentinel(Tcl_Obj *obj, const char *sentinel)
 {
     char *tclString = Tcl_GetString(obj);
-    if(sentinel == NULL)
+    if (sentinel == NULL)
         sentinel = TOHIL_NONE_SENTINEL;
     return (STREQU(tclString, sentinel));
 }
@@ -122,6 +122,82 @@ tohil_UTF8ToTclDString(Tcl_Interp *interp, char *utf8String, int utf8StringLen,
     return Tcl_ExternalToUtfDString(utf8encoding, utf8String, utf8StringLen, ds);
 }
 
+//
+// tohil_UndentPython - remove consistent indenting from a python code block
+//
+static int tohil_UndentPython(Tcl_Interp *interp, char *string) {
+    char *indent = malloc(strlen(string)+1);
+    char *code_ptr = string;
+    char *indent_ptr = indent;
+    int seen_code = 0;
+
+    // look for indent
+    while (*code_ptr) {
+        // skip blank lines
+        if (*code_ptr == '\n') {
+            ++code_ptr;
+            indent_ptr = indent;
+            continue;
+        }
+        // Even on Windows
+        if (code_ptr[0] == '\r' && code_ptr[1] == '\n') {
+            code_ptr += 2;
+            indent_ptr = indent;
+            continue;
+        }
+        // Got some code, we have an indent
+        if (!isspace(*code_ptr)) {
+            seen_code = 1;
+            *indent_ptr = '\0';
+            break;
+        }
+        // Copy blanks into indent;
+        *indent_ptr++ = *code_ptr++;
+    }
+
+    // Empty code block, so just pass it back unchanged.
+    if (!seen_code) {
+        free(indent);
+        return TCL_OK;
+    }
+
+    // walk string deleting indent on every line
+    //
+    // It has already been deleted and copied into the indent buffer
+    // for the first line, so we only start the check when we see a
+    // new newline.
+    //
+    char *working_ptr = string;
+    while (*code_ptr) {
+        char c = *working_ptr++ = *code_ptr++;
+        if (c == '\n') {
+            indent_ptr = indent;
+            while (*indent_ptr && *code_ptr) {
+                // Look for mismatches
+                if (*indent_ptr != *code_ptr) {
+                    // Fast forward to end of line
+                    while (*code_ptr != '\n' && isspace(*code_ptr))
+                        code_ptr++;
+                    // Mismatch on blank line, ignore
+                    if (!*code_ptr || *code_ptr == '\n')
+                        break;
+                    Tcl_SetResult(interp, "can't undent Python block (possibly mixed spaces and tabs)", TCL_STATIC);
+                    free(indent);
+                    return TCL_ERROR;
+                }
+                ++indent_ptr;
+                ++code_ptr;
+            }
+        }
+    }
+    *working_ptr = '\0';
+
+    // free indent
+    free(indent);
+    return TCL_OK;
+}
+
+
 //
 // turn a tcl list into a python list
 //
@@ -816,7 +892,7 @@ TohilCall_Cmd(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *cons
     PyObject *pMainModule = PyImport_AddModule("__main__");
     if (pMainModule == NULL) {
         Tcl_DStringFree(&objandfn_ds);
-        if(nonevalue) Tcl_DStringFree(&nonevalue_ds);
+        if (nonevalue) Tcl_DStringFree(&nonevalue_ds);
         return Tohil_ReturnExceptionToTcl(interp, "unable to add module __main__ to python interpreter");
     }
 
@@ -833,7 +909,7 @@ TohilCall_Cmd(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *cons
         if (pObjStr == NULL) {
             Py_DECREF(pObjParent);
             Tcl_DStringFree(&objandfn_ds);
-            if(nonevalue) Tcl_DStringFree(&nonevalue_ds);
+            if (nonevalue) Tcl_DStringFree(&nonevalue_ds);
             return Tohil_ReturnExceptionToTcl(interp, "failed unicode translation of call function in python interpreter");
         }
 
@@ -842,7 +918,7 @@ TohilCall_Cmd(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *cons
         Py_DECREF(pObjParent);
         if (pObj == NULL) {
             Tcl_DStringFree(&objandfn_ds);
-            if(nonevalue) Tcl_DStringFree(&nonevalue_ds);
+            if (nonevalue) Tcl_DStringFree(&nonevalue_ds);
             return Tohil_ReturnExceptionToTcl(interp, "failed to find dotted attribute in python interpreter");
         }
 
@@ -869,7 +945,7 @@ TohilCall_Cmd(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *cons
         char errorString[CALL_ERROR_STRING_SIZE];
         snprintf(errorString, CALL_ERROR_STRING_SIZE, "name '%.200s' is not defined.", objandfn);
         Tcl_DStringFree(&objandfn_ds);
-        if(nonevalue) Tcl_DStringFree(&nonevalue_ds);
+        if (nonevalue) Tcl_DStringFree(&nonevalue_ds);
         PyErr_SetString(PyExc_NameError, errorString);
         return Tohil_ReturnExceptionToTcl(interp, "failed to find object/function in python interpreter");
     }
@@ -904,7 +980,7 @@ TohilCall_Cmd(ClientData clientData, Tcl_Interp *interp, int objc, Tcl_Obj *cons
         /* Steals a reference */
         PyTuple_SET_ITEM(pArgs, i - objStart, curarg);
     }
-    if(nonevalue) Tcl_DStringFree(&nonevalue_ds);
+    if (nonevalue) Tcl_DStringFree(&nonevalue_ds);
 
     PyObject *pRet = PyObject_Call(pFn, pArgs, kwObj);
     Py_DECREF(pFn);
@@ -977,7 +1053,13 @@ TohilExecEvalPython(int startSymbol, Tcl_Interp *interp, int objc, Tcl_Obj *cons
         return tohil_tcl_return(interp, TCL_ERROR);
     }
     Tcl_DString ds;
-    const char *cmd = tohil_TclObjToUTF8DString(interp, objv[1], &ds);
+    char *cmd = tohil_TclObjToUTF8DString(interp, objv[1], &ds);
+    if (startSymbol == Py_file_input) {
+        if (tohil_UndentPython(interp, cmd) == TCL_ERROR) {
+            Tcl_DStringFree(&ds);
+            return TCL_ERROR;
+        }
+    }
 
     // evaluate the command according to the start symbol
     PyObject *main_module = PyImport_AddModule("__main__");


Test patch:
diff --git a/tests/tohil.test b/tests/tohil.test
index acd488d..3947b79 100755
--- a/tests/tohil.test
+++ b/tests/tohil.test
@@ -55,6 +55,65 @@ test tohil_exec-1.6 {exec invoking python invoking tcl} \
 	-body {catch {tohil::exec "tohil.eval('nonesuch')"} catchResult catchDict; return [lindex [dict get $catchDict -errorcode] 1]} \
 	-result TclError
 
+test tohil_exec-1.7 {exec with indented code} \
+	-body {
+		tohil::exec {
+			a = 5
+		}
+	}
+
+test tohil_exec-1.8 {exec with indented code and mismatched spaces} \
+	-body {
+		tohil::exec {
+			a = 5
+		       b = 5
+		}
+	} \
+	-returnCodes error \
+	-result {can't undent Python block (possibly mixed spaces and tabs)}
+
+# Note that the blank line in the next test contains several spaces to try and trip up the undenter
+# this is deliberate, don't edit them out.
+test tohil_exec-1.9 {exec with indented code and blank lines} \
+	-body {
+		tohil::exec {
+			a = 5
+                      
+			b = 5
+		}
+	}
+
+test tohil_exec-1.10 {exec with indent that is a mix of tabs and spaces} \
+	-body {
+		tohil::exec {
+		    a = 5
+		    b = 10
+		}
+	}
+
+test tohil_exec-1.11 {exec with an indented if statement} \
+	-body {
+		tohil::exec {
+			a = 5
+			if a > 10:
+				b = a
+			else:
+				b = 5
+		}
+	}
+
+# Note that the carriage returns at the end of lines in the following test are intentional
+# to test parsing of Windows format files.
+test tohil_exec-1.12 {exec with carriage returns} \
+	-body {
+		tohil::exec {
+			a = 5
+			
+			b = 5
+		}
+	}
+
+
 # =========
 # tohil::import
 # =========