did you tried to read it as int and parse bytes using bitwise operation?
int idx = *(__global const int *)(srcptr +..
dst[0] = lut_l[idx&0xff];
dst[1] = lut_l[(idx>>8)&0xff];
dst[2] = lut_l[(idx>>16)&0xff];
dst[3] = lut_l[(idx>>24)&0xff];