Function at 0x180056460
Capabilities:
Encrypt data using rc4 ksa

Disassembly:
mov qword [rsp + 0x10], rbx
mov dword [rsp + 0x20], r9d
push rbp
push rsi
push rdi
push r12
push r13
push r14
push r15
sub rsp, 0x60
mov r12d, dword [rsp + 0xc0]
xor r11d, r11d
mov ebp, dword [rsp + 0xd0]
mov rdi, rcx
mov eax, r9d
cmp ebp, 0x10
mov r9d, dword [rsp + 0xc8]
mov r10d, r11d
sete r10b
mov esi, r8d
inc r10d
mov r13, rdx
mov ecx, r10d
imul ecx, eax
mov rax, qword [rdi]
mov r15d, dword [rax + 8]
mov r14d, ecx
imul r14d, r12d
mov eax, r15d
imul eax, r10d
mov dword [rsp + 0x34], r15d
mov dword [rsp + 0x38], r14d
mov dword [rsp + 0x2c], eax
test r12d, r12d
js 0x180057017
test r9d, r9d
js 0x180057017
mov ebx, 0x7fffffff
je 0x1800564fe
xor edx, edx
mov eax, ebx
div r9d
cmp r12d, eax
jg 0x180057017
mov r8d, r12d
imul r8d, r9d
test r8d, r8d
js 0x180057017
test ecx, ecx
js 0x180057017
je 0x180056526
mov eax, ebx
cdq
idiv ecx
cmp r8d, eax
jg 0x180057017
mov eax, r9d
imul eax, r14d
movsxd rcx, eax
call qword [sym.imp.api_ms_win_crt_heap_l1_1_0.dll_malloc]
mov qword [rdi + 0x18], rax
test rax, rax
je 0x18005701b
test r15d, r15d
js 0x180056ffb
test r12d, r12d
je 0x180056561
xor edx, edx
mov eax, ebx
div r12d
cmp r15d, eax
jg 0x180056ffb
mov r14d, r15d
imul r14d, r12d
test r14d, r14d
js 0x180056ffb
test ebp, ebp
js 0x180056ffb
je 0x18005658a
xor edx, edx
mov eax, ebx
div ebp
cmp r14d, eax
jg 0x180056ffb
mov eax, r14d
imul eax, ebp
cmp eax, 0x7ffffff8
jg 0x180056ffb
mov ecx, dword [rsp + 0xc8]
imul ebp, r14d
add ebp, 7
shr ebp, 3
mov dword [rsp + 0xa0], ebp
test ecx, ecx
je 0x1800565c5
xor edx, edx
mov eax, ebx
div ecx
cmp ebp, eax
jg 0x180056ffb
mov eax, ebp
sub ebx, ebp
imul eax, ecx
cmp eax, ebx
jg 0x180056ffb
lea eax, [rbp + 1]
imul eax, ecx
cmp esi, eax
jae 0x1800565fd
mov edx, dword [0x1800e47e4]
mov rax, qword gs:[0x58]
mov rax, qword [rax + rdx*8]
lea rdx, str.not_enough_pixels
jmp 0x180057035
lea ecx, [rbp*2]
call qword [sym.imp.api_ms_win_crt_heap_l1_1_0.dll_malloc]
mov qword [rsp + 0x20], rax
mov r10, rax
test rax, rax
je 0x18005701b
movsxd r11, dword [rsp + 0xd0]
mov ebx, 1
cmp r11d, 8
jge 0x180056638
mov ecx, ebx
mov dword [rsp + 0x2c], ebx
mov eax, ebp
jmp 0x18005663f
mov ecx, dword [rsp + 0x2c]
mov eax, r12d
xor r8d, r8d
mov r12d, r8d
mov dword [rsp + 0x30], r8d
cmp dword [rsp + 0xc8], r8d
jbe 0x180056fe4
mov rdx, qword [rdi + 0x18]
mov qword [rsp + 0x48], rdx
lea rdx, [0x180084b10]
add rdx, r11
movsxd rbp, ecx
imul eax, ecx
mov qword [rsp + 0x50], rdx
mov qword [rsp + 0x40], rbp
movsxd rdx, eax
mov qword [rsp + 0x58], rdx
mov rsi, rdx
mov dword [rsp + 0x28], eax
nop word [rax + rax]
mov ecx, dword [rsp + 0xa0]
mov r9d, r12d
movzx eax, byte [r13]
not r9d
and r9d, 1
mov ebx, r12d
and ebx, 1
imul r9d, ecx
mov edi, r12d
imul ebx, ecx
imul edi, dword [rsp + 0x38]
inc r13
add r9, r10
add rbx, r10
add rdi, qword [rsp + 0x48]
cmp eax, 4
ja 0x180056fb6
lea r10, [0x180000000]
test r12d, r12d
jne 0x1800566ea
movzx eax, byte [rax + r10 + 0x847c8]
mov rsi, rdx
cmp eax, 5
ja case.default.0x180056700
cdqe
mov ecx, dword [r10 + rax*4 + 0x57058]
add rcx, r10
jmp rcx
mov r8, rsi
mov rdx, r13
mov rcx, rbx
call sub.VCRUNTIME140.dll_memcpy
jmp 0x180056bc8
mov r8, rbp
mov rdx, r13
mov rcx, rbx
call sub.VCRUNTIME140.dll_memcpy
cmp rbp, rsi
jge 0x180056bc8
mov rdx, rbx
mov rcx, r13
sub rdx, rbp
mov r8, rbx
sub rdx, r13
add rcx, rbp
sub r8, r13
sub rsi, rbp
nop dword [rax]
nop dword [rax + rax]
movzx eax, byte [rdx + rcx]
add al, byte [rcx]
mov byte [r8 + rcx], al
lea rcx, [rcx + 1]
sub rsi, 1
jne 0x180056750
jmp 0x180056bc8
mov r10d, dword [rsp + 0x28]
mov rdx, r8
test r10d, r10d
jle 0x1800568b1
cmp r10d, 8
jb 0x1800568b1
lea eax, [r10 - 1]
movsxd rcx, eax
lea rax, [rcx + r13]
lea r8, [rcx + rbx]
cmp rbx, rax
ja 0x1800567a1
cmp r8, r13
jae 0x1800568b1
lea rax, [rcx + r9]
cmp rbx, rax
ja 0x1800567b3
cmp r8, r9
jae 0x1800568b1
cmp r10d, 0x40
jb 0x180056862
mov ecx, r10d
and ecx, 0x8000003f
jge 0x1800567cf
dec ecx
or ecx, 0xffffffc0
inc ecx
mov eax, r10d
sub eax, ecx
mov rcx, r9
movsxd r10, eax
sub rcx, r13
lea rax, [r13 + 0x10]
mov r8, rbx
sub r8, r13
nop word [rax + rax]
movdqu xmm0, xmmword [rax - 0x10]
add rdx, 0x40
movdqu xmm1, xmmword [rcx + rax - 0x10]
movdqu xmm2, xmmword [rax + rcx]
lea rax, [rax + 0x40]
paddb xmm1, xmm0
movdqu xmm0, xmmword [rax - 0x40]
movdqu xmmword [r8 + rax - 0x50], xmm1
paddb xmm2, xmm0
movdqu xmm0, xmmword [rax - 0x30]
movdqu xmm1, xmmword [rcx + rax - 0x30]
movdqu xmmword [rax + r8 - 0x40], xmm2
paddb xmm1, xmm0
movdqu xmm0, xmmword [rax - 0x20]
movdqu xmm2, xmmword [rcx + rax - 0x20]
movdqu xmmword [r8 + rax - 0x30], xmm1
paddb xmm2, xmm0
movdqu xmmword [r8 + rax - 0x20], xmm2
cmp rdx, r10
jl 0x1800567f0
mov r10d, dword [rsp + 0x28]
mov eax, r10d
and al, 0x3f
cmp al, 8
jb 0x1800568b1
mov ecx, r10d
and ecx, 0x80000007
jge 0x180056874
dec ecx
or ecx, 0xfffffff8
inc ecx
mov eax, r10d
sub eax, ecx
mov rcx, r9
movsxd r10, eax
sub rcx, r13
lea rax, [rdx + r13]
mov r8, rbx
sub r8, r13
nop dword [rax]
movq xmm1, qword [rax + rcx]
add rdx, 8
movq xmm0, qword [rax]
lea rax, [rax + 8]
paddb xmm1, xmm0
movq qword [rax + r8 - 8], xmm1
cmp rdx, r10
jl 0x180056890
cmp rdx, rsi
jge 0x180056bd0
mov r8, rbx
lea rcx, [rdx + r13]
sub r8, r13
sub r9, r13
sub rsi, rdx
nop word [rax + rax]
movzx eax, byte [rcx + r9]
add al, byte [rcx]
mov byte [rcx + r8], al
lea rcx, [rcx + 1]
sub rsi, 1
jne 0x1800568d0
jmp 0x180056bd0
test rbp, rbp
jle 0x18005691a
mov r8, r9
mov r10, rbx
sub r8, r13
mov rcx, r13
sub r10, r13
mov rdx, rbp
movzx eax, byte [rcx + r8]
lea rcx, [rcx + 1]
shr al, 1
add al, byte [rcx - 1]
mov byte [rcx + r10 - 1], al
sub rdx, 1
jne 0x180056901
cmp rbp, rsi
jge 0x180056bd0
mov r8, rbx
lea rdx, [r9 + rbp]
sub r8, rbp
mov r10, r13
mov r11, rbx
sub r8, r9
sub r10, r9
sub r11, r9
sub rsi, rbp
nop
movzx ecx, byte [rdx + r8]
movzx eax, byte [rdx]
lea rdx, [rdx + 1]
add ecx, eax
shr ecx, 1
add cl, byte [rdx + r10 - 1]
mov byte [rdx + r11 - 1], cl
sub rsi, 1
jne 0x180056940
jmp 0x180056bc8
mov r10d, dword [rsp + 0x2c]
mov rdx, r8
test r10d, r10d
jle 0x180056ab1
cmp r10d, 8
jb 0x180056ab1
lea eax, [r10 - 1]
movsxd rcx, eax
lea rax, [rcx + r13]
lea r8, [rcx + rbx]
cmp rbx, rax
ja 0x18005699d
cmp r8, r13
jae 0x180056ab1
lea rax, [rcx + r9]
cmp rbx, rax
ja 0x1800569af
cmp r8, r9
jae 0x180056ab1
cmp r10d, 0x40
jb 0x180056a62
mov ecx, r10d
and ecx, 0x8000003f
jge 0x1800569cb
dec ecx
or ecx, 0xffffffc0
inc ecx
mov eax, r10d
sub eax, ecx
mov rcx, r9
movsxd r10, eax
sub rcx, r13
lea rax, [r13 + 0x10]
mov r8, rbx
sub r8, r13
nop dword [rax]
nop word [rax + rax]
movdqu xmm0, xmmword [rax - 0x10]
add rdx, 0x40
movdqu xmm1, xmmword [rcx + rax - 0x10]
movdqu xmm2, xmmword [rax + rcx]
lea rax, [rax + 0x40]
paddb xmm1, xmm0
movdqu xmm0, xmmword [rax - 0x40]
movdqu xmmword [r8 + rax - 0x50], xmm1
paddb xmm2, xmm0
movdqu xmm0, xmmword [rax - 0x30]
movdqu xmm1, xmmword [rcx + rax - 0x30]
movdqu xmmword [rax + r8 - 0x40], xmm2
paddb xmm1, xmm0
movdqu xmm0, xmmword [rax - 0x20]
movdqu xmm2, xmmword [rcx + rax - 0x20]
movdqu xmmword [r8 + rax - 0x30], xmm1
paddb xmm2, xmm0
movdqu xmmword [r8 + rax - 0x20], xmm2
cmp rdx, r10
jl 0x1800569f0
mov r10d, dword [rsp + 0x2c]
mov eax, r10d
and al, 0x3f
cmp al, 8
jb 0x180056ab1
mov ecx, r10d
and ecx, 0x80000007
jge 0x180056a74
dec ecx
or ecx, 0xfffffff8
inc ecx
mov eax, r10d
sub eax, ecx
mov rcx, r9
movsxd r10, eax
sub rcx, r13
lea rax, [rdx + r13]
mov r8, rbx
sub r8, r13
nop dword [rax]
movq xmm1, qword [rcx + rax]
add rdx, 8
movq xmm0, qword [rax]
lea rax, [rax + 8]
paddb xmm1, xmm0
movq qword [r8 + rax - 8], xmm1
cmp rdx, r10
jl 0x180056a90
cmp rdx, rbp
jge 0x180056aed
mov r10, r9
lea rcx, [rdx + r13]
mov r11, rbx
sub r10, r13
sub r11, r13
mov r8, rbp
sub r8, rdx
nop dword [rax]
movzx eax, byte [r10 + rcx]
add al, byte [rcx]
mov byte [r11 + rcx], al
lea rcx, [rcx + 1]
sub r8, 1
jne 0x180056ad0
mov r11d, dword [rsp + 0xd0]
cmp rbp, rsi
jge 0x180056bd0
mov r12, qword [rsp + 0x40]
mov rax, rbp
sub rax, rbx
mov r11, r9
sub r11, rbx
mov r10, rbx
sub rsi, r12
lea rbp, [rax + r9]
lea r15, [rax + r13]
nop word [rax + rax]
movzx eax, byte [r10 + rbp]
movzx r9d, byte [r10]
mov edx, eax
movzx ecx, byte [r11 + r10]
lea r10, [r10 + 1]
lea r8d, [rcx + rcx*2]
sub r8d, r9d
sub r8d, eax
cmp r9d, eax
cmovb edx, r9d
cmovb r9d, eax
cmp r9d, r8d
movzx eax, dl
cmovle ecx, eax
cmp r8d, edx
movzx ecx, cl
movzx eax, r9b
cmovle ecx, eax
add cl, byte [r15 + r10 - 1]
mov byte [r10 + r12 - 1], cl
sub rsi, 1
jne 0x180056b20
mov r12d, dword [rsp + 0x30]
mov r15d, dword [rsp + 0x34]
jmp 0x180056bc8
mov r8, rbp
mov rdx, r13
mov rcx, rbx
call sub.VCRUNTIME140.dll_memcpy
cmp rbp, rsi
jge 0x180056bc8
mov rdx, rbx
mov rcx, r13
sub rdx, rbp
mov r8, rbx
sub rdx, r13
add rcx, rbp
sub r8, r13
sub rsi, rbp
nop word [rax + rax]
movzx eax, byte [rcx + rdx]
lea rcx, [rcx + 1]
shr al, 1
add al, byte [rcx - 1]
mov byte [rcx + r8 - 1], al
sub rsi, 1
jne 0x180056bb0
mov r11d, dword [rsp + 0xd0]
mov ecx, dword [rsp + 0xa0]
movsxd rsi, dword [rsp + 0x28]
add r13, rsi
cmp r11d, 8
jge 0x180056dde
cmp dword [rsp + 0xd8], 0
jne 0x180056bfe
mov rax, qword [rsp + 0x50]
movzx r15d, byte [rax]
jmp 0x180056c01
mov r15b, 1
xor r8b, r8b
mov r9, rdi
cmp r11d, 4
jne 0x180056c63
xor r11d, r11d
test r14d, r14d
je 0x180056cfd
nop dword [rax]
mov r10d, r11d
and r10d, 1
jne 0x180056c2d
movzx r8d, byte [rbx]
movzx eax, r8b
movzx ecx, r15b
shr al, 4
inc r11d
movzx edx, al
lea rax, [rbx + 1]
imul edx, ecx
shl r8b, 4
mov byte [r9], dl
inc r9
test r10d, r10d
cmovne rax, rbx
mov rbx, rax
cmp r11d, r14d
jb 0x180056c20
jmp 0x180056cfd
cmp r11d, 2
jne 0x180056cb5
xor r11d, r11d
test r14d, r14d
je 0x180056cfd
mov r10d, r11d
and r10d, 3
jne 0x180056c82
movzx r8d, byte [rbx]
movzx eax, r8b
movzx ecx, r15b
shr al, 6
inc r11d
movzx edx, al
lea rax, [rbx + 1]
imul edx, ecx
shl r8b, 2
mov byte [r9], dl
inc r9
test r10d, r10d
cmovne rax, rbx
mov rbx, rax
cmp r11d, r14d
jb 0x180056c75
jmp 0x180056cfd
xor r11d, r11d
test r14d, r14d
je 0x180056cfd
nop dword [rax]
mov r10d, r11d
and r10d, 7
jne 0x180056ccd
movzx r8d, byte [rbx]
movzx eax, r8b
movzx ecx, r15b
shr al, 7
add r8b, r8b
movzx edx, al
inc r11d
imul edx, ecx
lea rax, [rbx + 1]
mov byte [r9], dl
inc r9
test r10d, r10d
cmovne rax, rbx
mov rbx, rax
cmp r11d, r14d
jb 0x180056cc0
mov r15d, dword [rsp + 0x34]
cmp r15d, dword [rsp + 0xb8]
je 0x180056d42
movsxd rcx, dword [rsp + 0xc0]
dec rcx
cmp r15d, 1
jne 0x180056d88
test rcx, rcx
js 0x180056d42
nop dword [rax]
nop word [rax + rax]
mov byte [rdi + rcx*2 + 1], 0xff
movzx eax, byte [rcx + rdi]
mov byte [rdi + rcx*2], al
sub rcx, 1
jns 0x180056d30
mov r11d, dword [rsp + 0xd0]
mov r10, qword [rsp + 0x20]
mov rbp, qword [rsp + 0x40]
inc r12d
mov r8d, 0
mov rdx, qword [rsp + 0x58]
mov dword [rsp + 0x30], r12d
cmp r12d, dword [rsp + 0xc8]
jb 0x180056690
mov rcx, r10
call qword [sym.imp.api_ms_win_crt_heap_l1_1_0.dll_free]
mov eax, 1
jmp 0x180057040
test rcx, rcx
js 0x180056d42
lea r8, [rcx*2 + 1]
add r8, rdi
lea rdx, [rcx*4 + 2]
add r8, rcx
add rdx, rdi
nop word [rax + rax]
sub rcx, 1
mov byte [rdx + 1], 0xff
movzx eax, byte [r8 + 1]
lea r8, [r8 - 3]
mov byte [rdx], al
lea rdx, [rdx - 4]
movzx eax, byte [r8 + 3]
mov byte [rdx + 3], al
movzx eax, byte [r8 + 2]
mov byte [rdx + 2], al
jns 0x180056db0
jmp 0x180056d42
jne 0x180056e8e
cmp r15d, dword [rsp + 0xb8]
jne 0x180056e01
mov r8d, r14d
mov rdx, rbx
mov rcx, rdi
call sub.VCRUNTIME140.dll_memcpy
jmp 0x180056d42
movsxd rcx, dword [rsp + 0xc0]
dec rcx
cmp r15d, 1
jne 0x180056e37
test rcx, rcx
js 0x180056d4a
nop dword [rax + rax]
mov byte [rdi + rcx*2 + 1], 0xff
movzx eax, byte [rcx + rbx]
mov byte [rdi + rcx*2], al
sub rcx, 1
jns 0x180056e20
jmp 0x180056d4a
test rcx, rcx
js 0x180056d4a
lea r8, [rcx*2 + 1]
add r8, rbx
lea rdx, [rcx*4 + 2]
add r8, rcx
add rdx, rdi
nop dword [rax]
sub rcx, 1
mov byte [rdx + 1], 0xff
movzx eax, byte [r8 + 1]
lea r8, [r8 - 3]
mov byte [rdx], al
lea rdx, [rdx - 4]
movzx eax, byte [r8 + 3]
mov byte [rdx + 3], al
movzx eax, byte [r8 + 2]
mov byte [rdx + 2], al
jns 0x180056e60
jmp 0x180056d4a
cmp r11d, 0x10
jne 0x180056d4a
cmp r15d, dword [rsp + 0xb8]
jne 0x180056f08
mov r10, qword [rsp + 0x20]
test r14d, r14d
je 0x180056d4f
mov edx, r12d
mov r8, rdi
and edx, 1
imul ecx, edx
imul edx, dword [rsp + 0xa0]
sub r8, rcx
sub r8, r10
mov r9d, edx
add r8, rbx
mov edx, r14d
sub r9, rdi
add r9, r10
nop dword [rax]
movzx eax, byte [rbx]
lea rbx, [rbx + 2]
movzx ecx, byte [r9 + r8 + 1]
lea r8, [r8 + 2]
shl ax, 8
or cx, ax
mov word [r8 - 2], cx
sub rdx, 1
jne 0x180056ee0
jmp 0x180056d4f
mov eax, dword [rsp + 0xc0]
cmp r15d, 1
jne 0x180056f4b
test eax, eax
je 0x180056d4a
mov edx, eax
nop
movzx ecx, byte [rbx]
lea rdi, [rdi + 4]
movzx eax, byte [rbx + 1]
lea rbx, [rbx + 2]
shl cx, 8
or cx, ax
mov word [rdi - 2], 0xffff
mov word [rdi - 4], cx
sub rdx, 1
jne 0x180056f20
jmp 0x180056d42
test eax, eax
je 0x180056d4a
add rdi, 4
mov rdx, rax
nop word [rax + rax]
movzx ecx, byte [rbx]
lea rdi, [rdi + 8]
movzx eax, byte [rbx + 1]
lea rbx, [rbx + 6]
shl cx, 8
or cx, ax
mov word [rdi - 0xc], cx
movzx ecx, byte [rbx - 4]
movzx eax, byte [rbx - 3]
shl cx, 8
or cx, ax
mov word [rdi - 0xa], cx
movzx ecx, byte [rbx - 2]
movzx eax, byte [rbx - 1]
shl cx, 8
or cx, ax
mov word [rdi - 6], 0xffff
mov word [rdi - 8], cx
sub rdx, 1
jne 0x180056f60
mov r12d, dword [rsp + 0x30]
jmp 0x180056d4a
mov edx, dword [0x1800e47e4]
mov rax, qword gs:[0x58]
mov ecx, 0x28
mov rax, qword [rax + rdx*8]
lea rdx, str.invalid_filter
mov qword [rcx + rax], rdx
mov rcx, r10
call qword [sym.imp.api_ms_win_crt_heap_l1_1_0.dll_free]
jmp 0x18005703e
mov rcx, r10
call qword [sym.imp.api_ms_win_crt_heap_l1_1_0.dll_free]
mov r11, r12
test ebx, ebx
setne r11b
mov eax, r11d
jmp 0x180057040
mov edx, dword [0x1800e47e4]
mov rax, qword gs:[0x58]
mov rax, qword [rax + rdx*8]
lea rdx, str.too_large
jmp 0x180057035
mov qword [rdi + 0x18], r11
mov edx, dword [0x1800e47e4]
mov rax, qword gs:[0x58]
mov rax, qword [rax + rdx*8]
lea rdx, str.outofmem
mov ecx, 0x28
mov qword [rcx + rax], rdx
xor eax, eax
mov rbx, qword [rsp + 0xa8]
add rsp, 0x60
pop r15
pop r14
pop r13
pop r12
pop rdi
pop rsi
pop rbp
ret