Repository: cantino/mcfly (language: rust)

Issue:
zsh's special history encoding is not handled
zsh seems containing a special encoding in its history format: http://www.zsh.org/mla/users/2011/msg00154.html, and currently it seems making that mcfly cannot handle some characters correctly. For example:

``` shell
❯ echo 'emoji😄'
emoji😄
❯ mcfly search emoji
echo 'emoji�������'
```

And the problem seems solved via decoding the zsh history first:

```shell
echo "$(cat  << 'EOF'
#define Meta ((char) 0x83)

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>

/* from zsh utils.c */
char *unmetafy(char *s, int *len)
{
  char *p, *t;

  for (p = s; *p && *p != Meta; p++);
  for (t = p; (*t = *p++);)
    if (*t++ == Meta)
      t[-1] = *p++ ^ 32;
  if (len)
    *len = t - s;
  return s;
}

int main(int argc, char *argv[]) {
  char *line = NULL;
  size_t size;

  while (getline(&line, &size, stdin) != -1) {
    unmetafy(line, NULL);
    printf("%s", line);
  }

  if (line) free(line);
  return EXIT_SUCCESS;
}
EOF
)"| gcc -x c - -o ~/.mcfly/zsh_history_format

mcfly_prompt_command () {
    local exit_code=$?
    if [[ ! -f "${MCFLY_HISTORY}" ]]
    then
        export MCFLY_HISTORY=$(mktemp -t mcfly.XXXXXXXX)
        tail -n100 "${HISTFILE}" |~/.mcfly/zsh_history_format >| ${MCFLY_HISTORY}
    fi
    # https://stackoverflow.com/questions/7756609/pass-stdout-as-file-name-for-command-line-util
    [ -z "${MCFLY_HISTORY_BUF}" ] && export MCFLY_HISTORY_BUF=mcfly_history_buf && mkfifo $MCFLY_HISTORY_BUF
    # fc -W "${MCFLY_HISTORY}"
    fc -W $MCFLY_HISTORY_BUF
    ~/.mcfly/zsh_history_format <$MCFLY_HISTORY_BUF > $MCFLY_HISTORY
    [ -n "$MCFLY_DEBUG" ] && echo "mcfly.zsh: Run mcfly add --exit ${exit_code}"
    $MCFLY_PATH --history_format zsh add --exit ${exit_code}
    return ${exit_code}
}

```




Discussion / hints:
I believe I'm having a similar problem, except that I'm stumbing into unicode problems on "normal" characters, not emoji.
![Screenshot 2021-09-07 at 10 41 28](https://user-images.githubusercontent.com/7012064/132313855-e18394ad-0c79-494e-a4cc-40dab89943e5.png)

You're right, I reproduced this issue. It's definitely a bug.
I think you have the right idea @onriv, but a better solution is to translate `unmetafy` into Rust and handle it in the mcfly binary. I can see if I can figure it out, although my C pointer math is pretty stale.
Found someone translated it: https://github.com/adder46/hstr-rs/blob/master/src/hstr.rs#L55 . I have never been programming with rust before. May it be helpful
Thanks!

Fix patch:
diff --git a/src/shell_history.rs b/src/shell_history.rs
index 1d244e42..0df2ae60 100644
--- a/src/shell_history.rs
+++ b/src/shell_history.rs
@@ -21,6 +21,22 @@ fn read_ignoring_utf_errors(path: &Path) -> String {
     String::from_utf8_lossy(&buffer).to_string()
 }
 
+// Zsh uses a meta char (0x83) to signify that the previous character should be ^ 32.
+fn read_and_unmetafy(path: &Path) -> String {
+    let mut f =
+        File::open(path).unwrap_or_else(|_| panic!("McFly error: {:?} file not found", &path));
+    let mut buffer = Vec::new();
+    f.read_to_end(&mut buffer)
+        .unwrap_or_else(|_| panic!("McFly error: Unable to read from {:?}", &path));
+    for index in (0..buffer.len()).rev() {
+        if buffer[index] == 0x83 {
+            buffer.remove(index);
+            buffer[index] ^= 32;
+        }
+    }
+    String::from_utf8_lossy(&buffer).to_string()
+}
+
 #[allow(clippy::if_same_then_else)]
 fn has_leading_timestamp(line: &str) -> bool {
     let mut matched_chars = 0;
@@ -95,7 +111,7 @@ impl fmt::Display for HistoryCommand {
 
 pub fn full_history(path: &Path, history_format: HistoryFormat) -> Vec<HistoryCommand> {
     match history_format {
-        HistoryFormat::Bash | HistoryFormat::Zsh { .. } => {
+        HistoryFormat::Bash => {
             let history_contents = read_ignoring_utf_errors(path);
             let zsh_timestamp_and_duration_regex = Regex::new(r"^: \d+:\d+;").unwrap();
             let when = SystemTime::now()
@@ -109,6 +125,20 @@ pub fn full_history(path: &Path, history_format: HistoryFormat) -> Vec<HistoryCo
                 .map(|line| HistoryCommand::new(line, when, history_format))
                 .collect()
         }
+        HistoryFormat::Zsh { .. } => {
+            let history_contents = read_and_unmetafy(path);
+            let zsh_timestamp_and_duration_regex = Regex::new(r"^: \d+:\d+;").unwrap();
+            let when = SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap_or_else(|err| panic!("McFly error: Time went backwards ({})", err))
+                .as_secs() as i64;
+            history_contents
+                .split('\n')
+                .filter(|line| !has_leading_timestamp(line) && !line.is_empty())
+                .map(|line| zsh_timestamp_and_duration_regex.replace(line, ""))
+                .map(|line| HistoryCommand::new(line, when, history_format))
+                .collect()
+        }
         HistoryFormat::Fish => {
             // Fish history format is not technically YAML.  This is a naive parser of the format,
             // only caring about command strings (which are always on one line, with embedded


Test patch: