Task: UTF-8 encode and decode




Go: package main

import (
    "bytes"
    "encoding/hex"
    "fmt"
    "log"
    "strings"
)

var testCases = []struct {
    rune
    string
}{
    {'A', "41"},
    {'ö', "C3 B6"},
    {'Ж', "D0 96"},
    {'€', "E2 82 AC"},
    {'𝄞', "F0 9D 84 9E"},
}

func main() {
    for _, tc := range testCases {
        
        u := fmt.Sprintf("U+%04X", tc.rune)
        b, err := hex.DecodeString(strings.Replace(tc.string, " ", "", -1))
        if err != nil {
            log.Fatal("bad test data")
        }
        
        e := encodeUTF8(tc.rune)
        d := decodeUTF8(b)
        
        fmt.Printf("%c  %-7s  %X\n", d, u, e)
        
        if !bytes.Equal(e, b) {
            log.Fatal("encodeUTF8 wrong")
        }
        if d != tc.rune {
            log.Fatal("decodeUTF8 wrong")
        }
    }
}

const (
    
    b2Lead = 0xC0 
    b2Mask = 0x1F 

    
    b3Lead = 0xE0 
    b3Mask = 0x0F 

    
    b4Lead = 0xF0 
    b4Mask = 0x07 

    
    mbLead = 0x80 
    mbMask = 0x3F 
)

func encodeUTF8(r rune) []byte {
    switch i := uint32(r); {
    case i <= 1<<7-1: 
        return []byte{byte(r)}
    case i <= 1<<11-1: 
        return []byte{
            b2Lead | byte(r>>6),
            mbLead | byte(r)&mbMask}
    case i <= 1<<16-1: 
        return []byte{
            b3Lead | byte(r>>12),
            mbLead | byte(r>>6)&mbMask,
            mbLead | byte(r)&mbMask}
    default:
        return []byte{
            b4Lead | byte(r>>18),
            mbLead | byte(r>>12)&mbMask,
            mbLead | byte(r>>6)&mbMask,
            mbLead | byte(r)&mbMask}
    }
}

func decodeUTF8(b []byte) rune {
    switch b0 := b[0]; {
    case b0 < 0x80:
        return rune(b0)
    case b0 < 0xE0:
        return rune(b0&b2Mask)<<6 |
            rune(b[1]&mbMask)
    case b0 < 0xF0:
        return rune(b0&b3Mask)<<12 |
            rune(b[1]&mbMask)<<6 |
            rune(b[2]&mbMask)
    default:
        return rune(b0&b4Mask)<<18 |
            rune(b[1]&mbMask)<<12 |
            rune(b[2]&mbMask)<<6 |
            rune(b[3]&mbMask)
    }
}








Ada: with Ada.Strings.Fixed; use Ada.Strings.Fixed;
with Ada.Strings.UTF_Encoding.Wide_Wide_Strings;
with Ada.Integer_Text_IO;
with Ada.Text_IO;
with Ada.Wide_Wide_Text_IO;

procedure UTF8_Encode_And_Decode
is
   package TIO renames Ada.Text_IO;
   package WWTIO renames Ada.Wide_Wide_Text_IO;
   package WWS renames Ada.Strings.UTF_Encoding.Wide_Wide_Strings;

   function To_Hex
     (i : in Integer;
      width : in Natural := 0;
      fill : in Character := '0') return String
   is
      holder : String(1 .. 20);
   begin
      Ada.Integer_Text_IO.Put(holder, i, 16);
      declare
         hex : constant String := holder(Index(holder, "#")+1 .. holder'Last-1);
         filled : String := Natural'Max(width, hex'Length) * fill;
      begin
         filled(filled'Last - hex'Length + 1 .. filled'Last) := hex;
         return filled;
      end;
   end To_Hex;

   input : constant Wide_Wide_String := "AöЖ€𝄞";
begin
   TIO.Put_Line("Character   Unicode    UTF-8 encoding (hex)");
   TIO.Put_Line(43 * '-');
   for WWC of input loop
      WWTIO.Put(WWC & "           ");
      declare
         filled : String := 11 * ' ';
         unicode : constant String := "U+" & To_Hex(Wide_Wide_Character'Pos(WWC), width => 4);
         utf8_string : constant String := WWS.Encode((1 => WWC));
      begin
         filled(filled'First .. filled'First + unicode'Length - 1) := unicode;
         TIO.Put(filled);
         for C of utf8_string loop
            TIO.Put(To_Hex(Character'Pos(C)) & " ");
         end loop;
         TIO.New_Line;
      end;
   end loop;
end UTF8_Encode_And_Decode;