WeixiangYan/CodeTransOcean huggingface dataset:
Task: Tokenize a string with escaping

C: #include <stdlib.h>
#include <stdio.h>

#define STR_DEMO "one^|uno||three^^^^|four^^^|^cuatro|"
#define SEP '|'
#define ESC '^'

typedef char* Str; 


unsigned int ElQ( const char *s, char sep, char esc );
Str *Tokenize( char *s, char sep, char esc, unsigned int *q );



int main() {
    char s[] = STR_DEMO;
    unsigned int i, q;

    Str *list = Tokenize( s, SEP, ESC, &q );

    if( list != NULL ) {
        printf( "\n Original string: %s\n\n", STR_DEMO );
        printf( " %d tokens:\n\n", q );

        for( i=0; i<q; ++i )
            printf( " %4d. %s\n", i+1, list[i] );

        free( list );
    }

    return 0;
}



unsigned int ElQ( const char *s, char sep, char esc ) {
    unsigned int q, e;
    const char *p;

    for( e=0, q=1, p=s; *p; ++p ) {
        if( *p == esc )
            e = !e;
        else if( *p == sep )
            q += !e;
        else e = 0;
    }

    return q;
}



Str *Tokenize( char *s, char sep, char esc, unsigned int *q ) {
    Str *list = NULL;

    *q = ElQ( s, sep, esc );
    list = malloc( *q * sizeof(Str) );

    if( list != NULL ) {
        unsigned int e, i;
        char *p;

        i = 0;
        list[i++] = s;

        for( e=0, p=s; *p; ++p ) {
            if( *p == esc ) {
                e = !e;
            }
            else if( *p == sep && !e ) {
                list[i++] = p+1;
                *p = '\0';
            }
            else {
                e = 0;
            }
        }
    }

    return list;
}











Ada: with Ada.Text_Io;
with Ada.Containers.Indefinite_Vectors;
with Ada.Strings.Unbounded;

procedure Tokenize is

   package String_Vectors is
     new Ada.Containers.Indefinite_Vectors (Positive, String);
   use String_Vectors;

   function Split (Text      : String;
                   Separator : Character := '|';
                   Escape    : Character := '^') return Vector
   is
      use Ada.Strings.Unbounded;
      Result  : Vector;
      Escaped : Boolean := False;
      Accu    : Unbounded_String;
   begin

      for Char of Text loop

         case Escaped is

            when False =>
               if Char = Escape then
                  Escaped := True;
               elsif Char = Separator then
                  Append (Result, To_String (Accu));
                  Accu := Null_Unbounded_String;
               else
                  Append (Accu, Char);
               end if;

            when True =>
               Append (Accu, Char);
               Escaped := False;

         end case;

      end loop;
      Append (Result, To_String (Accu));

      return Result;
   end Split;

   procedure Put_Vector (List : Vector) is
      use Ada.Text_Io;
   begin
      for Element of List loop
         Put ("'"); Put (Element); Put ("'"); New_Line;
      end loop;
   end Put_Vector;

begin
   Put_Vector (Split ("one^|uno||three^^^^|four^^^|^cuatro|"));
end Tokenize;
Task: Tokenize a string with escaping


C++: #include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

using namespace std;

vector<string> tokenize(const string& input, char seperator, char escape) {
    vector<string> output;
    string token;

    bool inEsc = false;
    for (char ch : input) {
        if (inEsc) {
            inEsc = false;
        } else if (ch == escape) {
            inEsc = true;
            continue;
        } else if (ch == seperator) {
            output.push_back(token);
            token = "";
            continue;
        }
        token += ch;
    }
    if (inEsc)
        throw new invalid_argument("Invalid terminal escape");

    output.push_back(token);
    return output;
}

int main() {
    string sample = "one^|uno||three^^^^|four^^^|^cuatro|";

    cout << sample << endl;
    cout << '[';
    for (auto t : tokenize(sample, '|', '^')) {
        cout << '"' << t << "\", ";
    }
    cout << ']' << endl;

    return 0;
}










Ada: with Ada.Text_Io;
with Ada.Containers.Indefinite_Vectors;
with Ada.Strings.Unbounded;

procedure Tokenize is

   package String_Vectors is
     new Ada.Containers.Indefinite_Vectors (Positive, String);
   use String_Vectors;

   function Split (Text      : String;
                   Separator : Character := '|';
                   Escape    : Character := '^') return Vector
   is
      use Ada.Strings.Unbounded;
      Result  : Vector;
      Escaped : Boolean := False;
      Accu    : Unbounded_String;
   begin

      for Char of Text loop

         case Escaped is

            when False =>
               if Char = Escape then
                  Escaped := True;
               elsif Char = Separator then
                  Append (Result, To_String (Accu));
                  Accu := Null_Unbounded_String;
               else
                  Append (Accu, Char);
               end if;

            when True =>
               Append (Accu, Char);
               Escaped := False;

         end case;

      end loop;
      Append (Result, To_String (Accu));

      return Result;
   end Split;

   procedure Put_Vector (List : Vector) is
      use Ada.Text_Io;
   begin
      for Element of List loop
         Put ("'"); Put (Element); Put ("'"); New_Line;
      end loop;
   end Put_Vector;

begin
   Put_Vector (Split ("one^|uno||three^^^^|four^^^|^cuatro|"));
end Tokenize;
Task: Tokenize a string with escaping




Go: package main

import (
	"errors"
	"fmt"
)

func TokenizeString(s string, sep, escape rune) (tokens []string, err error) {
	var runes []rune
	inEscape := false
	for _, r := range s {
		switch {
		case inEscape:
			inEscape = false
			fallthrough
		default:
			runes = append(runes, r)
		case r == escape:
			inEscape = true
		case r == sep:
			tokens = append(tokens, string(runes))
			runes = runes[:0]
		}
	}
	tokens = append(tokens, string(runes))
	if inEscape {
		err = errors.New("invalid terminal escape")
	}
	return tokens, err
}

func main() {
	const sample = "one^|uno||three^^^^|four^^^|^cuatro|"
	const separator = '|'
	const escape = '^'

	fmt.Printf("Input:   %q\n", sample)
	tokens, err := TokenizeString(sample, separator, escape)
	if err != nil {
		fmt.Println("error:", err)
	} else {
		fmt.Printf("Tokens: %q\n", tokens)
	}
}








Ada: with Ada.Text_Io;
with Ada.Containers.Indefinite_Vectors;
with Ada.Strings.Unbounded;

procedure Tokenize is

   package String_Vectors is
     new Ada.Containers.Indefinite_Vectors (Positive, String);
   use String_Vectors;

   function Split (Text      : String;
                   Separator : Character := '|';
                   Escape    : Character := '^') return Vector
   is
      use Ada.Strings.Unbounded;
      Result  : Vector;
      Escaped : Boolean := False;
      Accu    : Unbounded_String;
   begin

      for Char of Text loop

         case Escaped is

            when False =>
               if Char = Escape then
                  Escaped := True;
               elsif Char = Separator then
                  Append (Result, To_String (Accu));
                  Accu := Null_Unbounded_String;
               else
                  Append (Accu, Char);
               end if;

            when True =>
               Append (Accu, Char);
               Escaped := False;

         end case;

      end loop;
      Append (Result, To_String (Accu));

      return Result;
   end Split;

   procedure Put_Vector (List : Vector) is
      use Ada.Text_Io;
   begin
      for Element of List loop
         Put ("'"); Put (Element); Put ("'"); New_Line;
      end loop;
   end Put_Vector;

begin
   Put_Vector (Split ("one^|uno||three^^^^|four^^^|^cuatro|"));
end Tokenize;
Task: Tokenize a string with escaping





Java: import java.util.*;

public class TokenizeStringWithEscaping {

    public static void main(String[] args) {
        String sample = "one^|uno||three^^^^|four^^^|^cuatro|";
        char separator = '|';
        char escape = '^';

        System.out.println(sample);
        try {
            System.out.println(tokenizeString(sample, separator, escape));
        } catch (Exception e) {
            System.out.println(e);
        }
    }

    public static List<String> tokenizeString(String s, char sep, char escape)
            throws Exception {
        List<String> tokens = new ArrayList<>();
        StringBuilder sb = new StringBuilder();

        boolean inEscape = false;
        for (char c : s.toCharArray()) {
            if (inEscape) {
                inEscape = false;
            } else if (c == escape) {
                inEscape = true;
                continue;
            } else if (c == sep) {
                tokens.add(sb.toString());
                sb.setLength(0);
                continue;
            }
            sb.append(c);
        }
        if (inEscape)
            throw new Exception("Invalid terminal escape");

        tokens.add(sb.toString());

        return tokens;
    }
}







Ada: with Ada.Text_Io;
with Ada.Containers.Indefinite_Vectors;
with Ada.Strings.Unbounded;

procedure Tokenize is

   package String_Vectors is
     new Ada.Containers.Indefinite_Vectors (Positive, String);
   use String_Vectors;

   function Split (Text      : String;
                   Separator : Character := '|';
                   Escape    : Character := '^') return Vector
   is
      use Ada.Strings.Unbounded;
      Result  : Vector;
      Escaped : Boolean := False;
      Accu    : Unbounded_String;
   begin

      for Char of Text loop

         case Escaped is

            when False =>
               if Char = Escape then
                  Escaped := True;
               elsif Char = Separator then
                  Append (Result, To_String (Accu));
                  Accu := Null_Unbounded_String;
               else
                  Append (Accu, Char);
               end if;

            when True =>
               Append (Accu, Char);
               Escaped := False;

         end case;

      end loop;
      Append (Result, To_String (Accu));

      return Result;
   end Split;

   procedure Put_Vector (List : Vector) is
      use Ada.Text_Io;
   begin
      for Element of List loop
         Put ("'"); Put (Element); Put ("'"); New_Line;
      end loop;
   end Put_Vector;

begin
   Put_Vector (Split ("one^|uno||three^^^^|four^^^|^cuatro|"));
end Tokenize;
Task: Tokenize a string with escaping







Python: def token_with_escape(a, escape = '^', separator = '|'):
    
    result = []
    token = ''
    state = 0
    for c in a:
        if state == 0:
            if c == escape:
                state = 1
            elif c == separator:
                result.append(token)
                token = ''
            else:
                token += c
        elif state == 1:
            token += c
            state = 0
    result.append(token)
    return result





Ada: with Ada.Text_Io;
with Ada.Containers.Indefinite_Vectors;
with Ada.Strings.Unbounded;

procedure Tokenize is

   package String_Vectors is
     new Ada.Containers.Indefinite_Vectors (Positive, String);
   use String_Vectors;

   function Split (Text      : String;
                   Separator : Character := '|';
                   Escape    : Character := '^') return Vector
   is
      use Ada.Strings.Unbounded;
      Result  : Vector;
      Escaped : Boolean := False;
      Accu    : Unbounded_String;
   begin

      for Char of Text loop

         case Escaped is

            when False =>
               if Char = Escape then
                  Escaped := True;
               elsif Char = Separator then
                  Append (Result, To_String (Accu));
                  Accu := Null_Unbounded_String;
               else
                  Append (Accu, Char);
               end if;

            when True =>
               Append (Accu, Char);
               Escaped := False;

         end case;

      end loop;
      Append (Result, To_String (Accu));

      return Result;
   end Split;

   procedure Put_Vector (List : Vector) is
      use Ada.Text_Io;
   begin
      for Element of List loop
         Put ("'"); Put (Element); Put ("'"); New_Line;
      end loop;
   end Put_Vector;

begin
   Put_Vector (Split ("one^|uno||three^^^^|four^^^|^cuatro|"));
end Tokenize;
Task: Tokenize a string with escaping








VB: Private Function tokenize(s As String, sep As String, esc As String) As Collection
    Dim ret As New Collection
    Dim this As String
    Dim skip As Boolean
     
    If Len(s) <> 0 Then
        For i = 1 To Len(s)
            si = Mid(s, i, 1)
            If skip Then
                this = this & si
                skip = False
            Else
                If si = esc Then
                    skip = True
                Else
                    If si = sep Then
                        ret.Add this
                        this = ""
                    Else
                        this = this & si
                    End If
                End If
            End If
        Next i
        ret.Add this
    End If
    Set tokenize = ret
End Function

Public Sub main()
    Dim out As Collection
    Set out = tokenize("one^|uno||three^^^^|four^^^|^cuatro|", "|", "^")
    Dim outstring() As String
    ReDim outstring(out.Count - 1)
    For i = 0 To out.Count - 1
        outstring(i) = out(i + 1)
    Next i
    Debug.Print Join(outstring, ", ")
End Sub




Ada: with Ada.Text_Io;
with Ada.Containers.Indefinite_Vectors;
with Ada.Strings.Unbounded;

procedure Tokenize is

   package String_Vectors is
     new Ada.Containers.Indefinite_Vectors (Positive, String);
   use String_Vectors;

   function Split (Text      : String;
                   Separator : Character := '|';
                   Escape    : Character := '^') return Vector
   is
      use Ada.Strings.Unbounded;
      Result  : Vector;
      Escaped : Boolean := False;
      Accu    : Unbounded_String;
   begin

      for Char of Text loop

         case Escaped is

            when False =>
               if Char = Escape then
                  Escaped := True;
               elsif Char = Separator then
                  Append (Result, To_String (Accu));
                  Accu := Null_Unbounded_String;
               else
                  Append (Accu, Char);
               end if;

            when True =>
               Append (Accu, Char);
               Escaped := False;

         end case;

      end loop;
      Append (Result, To_String (Accu));

      return Result;
   end Split;

   procedure Put_Vector (List : Vector) is
      use Ada.Text_Io;
   begin
      for Element of List loop
         Put ("'"); Put (Element); Put ("'"); New_Line;
      end loop;
   end Put_Vector;

begin
   Put_Vector (Split ("one^|uno||three^^^^|four^^^|^cuatro|"));
end Tokenize;
Task: Tokenize a string with escaping

C: #include <stdlib.h>
#include <stdio.h>

#define STR_DEMO "one^|uno||three^^^^|four^^^|^cuatro|"
#define SEP '|'
#define ESC '^'

typedef char* Str; 


unsigned int ElQ( const char *s, char sep, char esc );
Str *Tokenize( char *s, char sep, char esc, unsigned int *q );



int main() {
    char s[] = STR_DEMO;
    unsigned int i, q;

    Str *list = Tokenize( s, SEP, ESC, &q );

    if( list != NULL ) {
        printf( "\n Original string: %s\n\n", STR_DEMO );
        printf( " %d tokens:\n\n", q );

        for( i=0; i<q; ++i )
            printf( " %4d. %s\n", i+1, list[i] );

        free( list );
    }

    return 0;
}



unsigned int ElQ( const char *s, char sep, char esc ) {
    unsigned int q, e;
    const char *p;

    for( e=0, q=1, p=s; *p; ++p ) {
        if( *p == esc )
            e = !e;
        else if( *p == sep )
            q += !e;
        else e = 0;
    }

    return q;
}



Str *Tokenize( char *s, char sep, char esc, unsigned int *q ) {
    Str *list = NULL;

    *q = ElQ( s, sep, esc );
    list = malloc( *q * sizeof(Str) );

    if( list != NULL ) {
        unsigned int e, i;
        char *p;

        i = 0;
        list[i++] = s;

        for( e=0, p=s; *p; ++p ) {
            if( *p == esc ) {
                e = !e;
            }
            else if( *p == sep && !e ) {
                list[i++] = p+1;
                *p = '\0';
            }
            else {
                e = 0;
            }
        }
    }

    return list;
}
Task: Tokenize a string with escaping



C#: using System;
using System.Text;
using System.Collections.Generic;

public class TokenizeAStringWithEscaping
{
    public static void Main() {
        string testcase = "one^|uno||three^^^^|four^^^|^cuatro|";
        foreach (var token in testcase.Tokenize(separator: '|', escape: '^')) {
            Console.WriteLine(": " + token); 
        }
    }
}

public static class Extensions
{
    public static IEnumerable<string> Tokenize(this string input, char separator, char escape) {
        if (input == null) yield break;
        var buffer = new StringBuilder();
        bool escaping = false;
        foreach (char c in input) {
            if (escaping) {
                buffer.Append(c);
                escaping = false;
            } else if (c == escape) {
                escaping = true;
            } else if (c == separator) {
                yield return buffer.Flush();
            } else {
                buffer.Append(c);
            }
        }
        if (buffer.Length > 0 || input[input.Length-1] == separator) yield return buffer.Flush();
    }
    
    public static string Flush(this StringBuilder stringBuilder) {
        string result = stringBuilder.ToString();
        stringBuilder.Clear();
        return result;
    }
}
Task: Tokenize a string with escaping


C++: #include <iostream>
#include <stdexcept>
#include <string>
#include <vector>

using namespace std;

vector<string> tokenize(const string& input, char seperator, char escape) {
    vector<string> output;
    string token;

    bool inEsc = false;
    for (char ch : input) {
        if (inEsc) {
            inEsc = false;
        } else if (ch == escape) {
            inEsc = true;
            continue;
        } else if (ch == seperator) {
            output.push_back(token);
            token = "";
            continue;
        }
        token += ch;
    }
    if (inEsc)
        throw new invalid_argument("Invalid terminal escape");

    output.push_back(token);
    return output;
}

int main() {
    string sample = "one^|uno||three^^^^|four^^^|^cuatro|";

    cout << sample << endl;
    cout << '[';
    for (auto t : tokenize(sample, '|', '^')) {
        cout << '"' << t << "\", ";
    }
    cout << ']' << endl;

    return 0;
}
Task: Tokenize a string with escaping





Java: import java.util.*;

public class TokenizeStringWithEscaping {

    public static void main(String[] args) {
        String sample = "one^|uno||three^^^^|four^^^|^cuatro|";
        char separator = '|';
        char escape = '^';

        System.out.println(sample);
        try {
            System.out.println(tokenizeString(sample, separator, escape));
        } catch (Exception e) {
            System.out.println(e);
        }
    }

    public static List<String> tokenizeString(String s, char sep, char escape)
            throws Exception {
        List<String> tokens = new ArrayList<>();
        StringBuilder sb = new StringBuilder();

        boolean inEscape = false;
        for (char c : s.toCharArray()) {
            if (inEscape) {
                inEscape = false;
            } else if (c == escape) {
                inEscape = true;
                continue;
            } else if (c == sep) {
                tokens.add(sb.toString());
                sb.setLength(0);
                continue;
            }
            sb.append(c);
        }
        if (inEscape)
            throw new Exception("Invalid terminal escape");

        tokens.add(sb.toString());

        return tokens;
    }
}
Task: Tokenize a string with escaping







Python: def token_with_escape(a, escape = '^', separator = '|'):
    
    result = []
    token = ''
    state = 0
    for c in a:
        if state == 0:
            if c == escape:
                state = 1
            elif c == separator:
                result.append(token)
                token = ''
            else:
                token += c
        elif state == 1:
            token += c
            state = 0
    result.append(token)
    return result
Task: Tokenize a string with escaping








VB: Private Function tokenize(s As String, sep As String, esc As String) As Collection
    Dim ret As New Collection
    Dim this As String
    Dim skip As Boolean
     
    If Len(s) <> 0 Then
        For i = 1 To Len(s)
            si = Mid(s, i, 1)
            If skip Then
                this = this & si
                skip = False
            Else
                If si = esc Then
                    skip = True
                Else
                    If si = sep Then
                        ret.Add this
                        this = ""
                    Else
                        this = this & si
                    End If
                End If
            End If
        Next i
        ret.Add this
    End If
    Set tokenize = ret
End Function

Public Sub main()
    Dim out As Collection
    Set out = tokenize("one^|uno||three^^^^|four^^^|^cuatro|", "|", "^")
    Dim outstring() As String
    ReDim outstring(out.Count - 1)
    For i = 0 To out.Count - 1
        outstring(i) = out(i + 1)
    Next i
    Debug.Print Join(outstring, ", ")
End Sub
Task: Tokenize a string with escaping




Go: package main

import (
	"errors"
	"fmt"
)

func TokenizeString(s string, sep, escape rune) (tokens []string, err error) {
	var runes []rune
	inEscape := false
	for _, r := range s {
		switch {
		case inEscape:
			inEscape = false
			fallthrough
		default:
			runes = append(runes, r)
		case r == escape:
			inEscape = true
		case r == sep:
			tokens = append(tokens, string(runes))
			runes = runes[:0]
		}
	}
	tokens = append(tokens, string(runes))
	if inEscape {
		err = errors.New("invalid terminal escape")
	}
	return tokens, err
}

func main() {
	const sample = "one^|uno||three^^^^|four^^^|^cuatro|"
	const separator = '|'
	const escape = '^'

	fmt.Printf("Input:   %q\n", sample)
	tokens, err := TokenizeString(sample, separator, escape)
	if err != nil {
		fmt.Println("error:", err)
	} else {
		fmt.Printf("Tokens: %q\n", tokens)
	}
}