Task: UTF-8 encode and decode




Go: package main

import (
    "bytes"
    "encoding/hex"
    "fmt"
    "log"
    "strings"
)

var testCases = []struct {
    rune
    string
}{
    {'A', "41"},
    {'ö', "C3 B6"},
    {'Ж', "D0 96"},
    {'€', "E2 82 AC"},
    {'𝄞', "F0 9D 84 9E"},
}

func main() {
    for _, tc := range testCases {
        
        u := fmt.Sprintf("U+%04X", tc.rune)
        b, err := hex.DecodeString(strings.Replace(tc.string, " ", "", -1))
        if err != nil {
            log.Fatal("bad test data")
        }
        
        e := encodeUTF8(tc.rune)
        d := decodeUTF8(b)
        
        fmt.Printf("%c  %-7s  %X\n", d, u, e)
        
        if !bytes.Equal(e, b) {
            log.Fatal("encodeUTF8 wrong")
        }
        if d != tc.rune {
            log.Fatal("decodeUTF8 wrong")
        }
    }
}

const (
    
    b2Lead = 0xC0 
    b2Mask = 0x1F 

    
    b3Lead = 0xE0 
    b3Mask = 0x0F 

    
    b4Lead = 0xF0 
    b4Mask = 0x07 

    
    mbLead = 0x80 
    mbMask = 0x3F 
)

func encodeUTF8(r rune) []byte {
    switch i := uint32(r); {
    case i <= 1<<7-1: 
        return []byte{byte(r)}
    case i <= 1<<11-1: 
        return []byte{
            b2Lead | byte(r>>6),
            mbLead | byte(r)&mbMask}
    case i <= 1<<16-1: 
        return []byte{
            b3Lead | byte(r>>12),
            mbLead | byte(r>>6)&mbMask,
            mbLead | byte(r)&mbMask}
    default:
        return []byte{
            b4Lead | byte(r>>18),
            mbLead | byte(r>>12)&mbMask,
            mbLead | byte(r>>6)&mbMask,
            mbLead | byte(r)&mbMask}
    }
}

func decodeUTF8(b []byte) rune {
    switch b0 := b[0]; {
    case b0 < 0x80:
        return rune(b0)
    case b0 < 0xE0:
        return rune(b0&b2Mask)<<6 |
            rune(b[1]&mbMask)
    case b0 < 0xF0:
        return rune(b0&b3Mask)<<12 |
            rune(b[1]&mbMask)<<6 |
            rune(b[2]&mbMask)
    default:
        return rune(b0&b4Mask)<<18 |
            rune(b[1]&mbMask)<<12 |
            rune(b[2]&mbMask)<<6 |
            rune(b[3]&mbMask)
    }
}


Java: import java.nio.charset.StandardCharsets;
import java.util.Formatter;

public class UTF8EncodeDecode {

    public static byte[] utf8encode(int codepoint) {
        return new String(new int[]{codepoint}, 0, 1).getBytes(StandardCharsets.UTF_8);
    }

    public static int utf8decode(byte[] bytes) {
        return new String(bytes, StandardCharsets.UTF_8).codePointAt(0);
    }

    public static void main(String[] args) {
        System.out.printf("%-7s %-43s %7s\t%s\t%7s%n",
                "Char", "Name", "Unicode", "UTF-8 encoded", "Decoded");

        for (int codepoint : new int[]{0x0041, 0x00F6, 0x0416, 0x20AC, 0x1D11E}) {
            byte[] encoded = utf8encode(codepoint);
            Formatter formatter = new Formatter();
            for (byte b : encoded) {
                formatter.format("%02X ", b);
            }
            String encodedHex = formatter.toString();
            int decoded = utf8decode(encoded);
            System.out.printf("%-7c %-43s U+%04X\t%-12s\tU+%04X%n",
                    codepoint, Character.getName(codepoint), codepoint, encodedHex, decoded);
        }
    }
}