// File: /workspace/github_cpp_makefile_star2_O2/mhe[P]raytracerframework_cpp/lodepng.cpp
// Function: filter(unsigned char*, unsigned char const*, unsigned int, unsigned int, LodePNG_InfoColor const*)
static unsigned filter(unsigned char* out, const unsigned char* in, unsigned w, unsigned h, const LodePNG_InfoColor* info)
{
  /*
  For PNG filter method 0
  out must be a buffer with as size: h + (w * h * bpp + 7) / 8, because there are the scanlines with 1 extra byte per scanline
  
  There is a nice heuristic described here: http://www.cs.toronto.edu/~cosmin/pngtech/optipng.html. It says:
   *  If the image type is Palette, or the bit depth is smaller than 8, then do not filter the image (i.e. use fixed filtering, with the filter None).
   * (The other case) If the image type is Grayscale or RGB (with or without Alpha), and the bit depth is not smaller than 8, then use adaptive filtering heuristic as follows: independently for each row, apply all five filters and select the filter that produces the smallest sum of absolute values per row.
  
  Here the above method is used mostly. Note though that it appears to be better to use the adaptive filtering on the plasma 8-bit palette example, but that image isn't the best reference for palette images in general.
  */
  
  unsigned bpp = LodePNG_InfoColor_getBpp(info);
  size_t linebytes = (w * bpp + 7) / 8; /*the width of a scanline in bytes, not including the filter type*/
  size_t bytewidth = (bpp + 7) / 8; /*bytewidth is used for filtering, is 1 when bpp < 8, number of bytes per pixel otherwise*/
  const unsigned char* prevline = 0;
  unsigned x, y;
  unsigned heuristic;
  unsigned error = 0;
  
  if(bpp == 0) return 31; /*invalid color type*/
  
  /*choose heuristic as described above*/
  if(info->colorType == 3 || info->bitDepth < 8) heuristic = 0;
  else heuristic = 1;
  
  if(heuristic == 0) /*None filtertype for everything*/
  {
    for(y = 0; y < h; y++)
    {
      size_t outindex = (1 + linebytes) * y; /*the extra filterbyte added to each row*/
      size_t inindex = linebytes * y;
      const unsigned TYPE = 0;
      out[outindex] = TYPE; /*filter type byte*/
      filterScanline(&out[outindex + 1], &in[inindex], prevline, linebytes, bytewidth, TYPE);
      prevline = &in[inindex];
    }
  }
  else if(heuristic == 1) /*adaptive filtering*/
  {
    size_t sum[5];
    ucvector attempt[5]; /*five filtering attempts, one for each filter type*/
    size_t smallest = 0;
    unsigned type, bestType = 0;
    
    for(type = 0; type < 5; type++) ucvector_init(&attempt[type]);
    for(type = 0; type < 5; type++)
    {
      if(!ucvector_resize(&attempt[type], linebytes)) { error = 9949; break; }
    }
    
    if(!error)
    {
      for(y = 0; y < h; y++)
      {
        /*try the 5 filter types*/
        for(type = 0; type < 5; type++)
        {
          filterScanline(attempt[type].data, &in[y * linebytes], prevline, linebytes, bytewidth, type);
          
          /*calculate the sum of the result*/
          sum[type] = 0;
          for(x = 0; x < attempt[type].size; x+=3) sum[type] += attempt[type].data[x]; /*note that not all pixels are checked to speed this up while still having probably the best choice*/
        
          /*check if this is smallest sum (or if type == 0 it's the first case so always store the values)*/
          if(type == 0 || sum[type] < smallest)
          {
            bestType = type;
            smallest = sum[type];
          }
        }
        
        prevline = &in[y * linebytes];
    
        /*now fill the out values*/
        out[y * (linebytes + 1)] = bestType; /*the first byte of a scanline will be the filter type*/
        for(x = 0; x < linebytes; x++) out[y * (linebytes + 1) + 1 + x] = attempt[bestType].data[x];
      }
    }
    
    for(type = 0; type < 5; type++) ucvector_cleanup(&attempt[type]);
  }
  #if 0 /*deflate the scanline with a fixed tree after every filter attempt to see which one deflates best. This is slow, and _does not work as expected_: the heuristic gives smaller result!*/
  else if(heuristic == 2) /*adaptive filtering by using deflate*/
  {
    size_t size[5];
    ucvector attempt[5]; /*five filtering attempts, one for each filter type*/
    size_t smallest;
    unsigned type = 0, bestType = 0;
    unsigned char* dummy;
    LodeZlib_DeflateSettings deflatesettings = LodeZlib_defaultDeflateSettings;
    deflatesettings.btype = 1; /*use fixed tree on the attempts so that the tree is not adapted to the filtertype on purpose, to simulate the true case where the tree is the same for the whole image*/
    for(type = 0; type < 5; type++) { ucvector_init(&attempt[type]); ucvector_resize(&attempt[type], linebytes); }
    for(y = 0; y < h; y++) /*try the 5 filter types*/
    {
      for(type = 0; type < 5; type++)
      {
        filterScanline(attempt[type].data, &in[y * linebytes], prevline, linebytes, bytewidth, type);
        size[type] = 0; dummy = 0;
        LodePNG_compress(&dummy, &size[type], attempt[type].data, attempt[type].size, &deflatesettings);
        free(dummy);
        /*check if this is smallest size (or if type == 0 it's the first case so always store the values)*/
        if(type == 0 || size[type] < smallest) { bestType = type; smallest = size[type]; }
      }
      prevline = &in[y * linebytes];
      out[y * (linebytes + 1)] = bestType; /*the first byte of a scanline will be the filter type*/
      for(x = 0; x < linebytes; x++) out[y * (linebytes + 1) + 1 + x] = attempt[bestType].data[x];
    }
    for(type = 0; type < 5; type++) ucvector_cleanup(&attempt[type]);
  }
  #endif
  
  return error;
}

// Assembly:
pushq %rbp
movq %rsp, %rbp
pushq %r15
pushq %r14
pushq %r13
pushq %r12
pushq %rbx
subq $0xc8, %rsp
movq %r8, %r14
movl %ecx, %r12d
movl %edx, %ebx
movq %rsi, %r15
movq %rdi, %r13
movq %r8, %rdi
callq 0x11cfc
testl %eax, %eax
je 0x17a81
movl %eax, %r8d
imull %eax, %ebx
addl $0x7, %ebx
shrl $0x3, %ebx
addl $0x7, %r8d
shrl $0x3, %r8d
cmpl $0x3, (%r14)
je 0x17a3a
cmpl $0x8, 0x4(%r14)
jae 0x17a9a
leaq 0x1(%rbx), %rax
movq %rax, -0x30(%rbp)
movl %r12d, %r12d
incq %r13
xorl %edx, %edx
subq $0x1, %r12
jb 0x17a7c
movq %r15, %r14
movb $0x0, -0x1(%r13)
movq %r13, %rdi
movq %r15, %rsi
movq %rbx, %rcx
movq %r8, %r15
xorl %r9d, %r9d
callq 0x17c34
movq %r15, %r8
addq -0x30(%rbp), %r13
leaq (%r14,%rbx), %r15
movq %r14, %rdx
jmp 0x17a4a
xorl %r14d, %r14d
jmp 0x17a85
pushq $0x1f
popq %r14
movl %r14d, %eax
addq $0xc8, %rsp
popq %rbx
popq %r12
popq %r13
popq %r14
popq %r15
popq %rbp
retq
xorl %eax, %eax
xorps %xmm0, %xmm0
cmpq $0x78, %rax
je 0x17abc
movups %xmm0, -0xf0(%rbp,%rax)
andq $0x0, -0xe0(%rbp,%rax)
addq $0x18, %rax
jmp 0x17a9f
movq %r8, -0x78(%rbp)
xorl %ecx, %ecx
leaq 0x18(%rcx), %r14
cmpq $0x90, %r14
je 0x17af4
leaq (%rcx,%rbp), %rdi
addq $-0xf0, %rdi
movq %rbx, %rsi
callq 0x10bdb
movq %r14, %rcx
testl %eax, %eax
jne 0x17ac2
movl $0x26dd, %r14d          # imm = 0x26DD
jmp 0x17c12
movq %r15, -0x60(%rbp)
leaq 0x1(%rbx), %rax
movq %rax, -0x38(%rbp)
movl %r12d, %eax
movq %rax, -0x70(%rbp)
leaq 0x1(%r13), %rax
movq %rax, -0x30(%rbp)
xorl %eax, %eax
movq $0x0, -0x48(%rbp)
xorl %r15d, %r15d
xorl %r12d, %r12d
movq %rbx, -0x50(%rbp)
movq %r13, -0x68(%rbp)
cmpq -0x70(%rbp), %rax
je 0x17c0f
movq %rax, -0x40(%rbp)
imulq %rbx, %rax
addq -0x60(%rbp), %rax
movq %rax, -0x58(%rbp)
xorl %r13d, %r13d
cmpq $0x5, %r13
je 0x17bad
imulq $0x18, %r13, %rbx
movq -0xf0(%rbp,%rbx), %r14
movzbl %r13b, %r9d
movq %r14, %rdi
movq -0x58(%rbp), %rsi
movq -0x48(%rbp), %rdx
movq -0x50(%rbp), %rcx
movq -0x78(%rbp), %r8
callq 0x17c34
movq -0xe8(%rbp,%rbx), %rcx
xorl %eax, %eax
xorl %edx, %edx
movl %edx, %edx
cmpq %rdx, %rcx
jbe 0x17b92
movzbl (%r14,%rdx), %esi
addq %rsi, %rax
addl $0x3, %edx
jmp 0x17b7e
testq %r13, %r13
sete %cl
cmpq %r15, %rax
setb %dl
orb %cl, %dl
cmovneq %rax, %r15
cmovnel %r13d, %r12d
incq %r13
jmp 0x17b44
movq -0x38(%rbp), %rax
imulq -0x40(%rbp), %rax
movq -0x68(%rbp), %r13
movb %r12b, (%r13,%rax)
movl %r12d, %eax
imulq $0x18, %rax, %rax
addq %rbp, %rax
addq $-0xf0, %rax
xorl %ecx, %ecx
movq -0x50(%rbp), %rbx
cmpq %rcx, %rbx
je 0x17bec
movq (%rax), %rdx
movb (%rdx,%rcx), %dl
movq -0x30(%rbp), %rsi
movb %dl, (%rsi,%rcx)
incq %rcx
jmp 0x17bd5
movq -0x40(%rbp), %rcx
incq %rcx
movq -0x30(%rbp), %rax
addq -0x38(%rbp), %rax
movq %rax, -0x30(%rbp)
movq %rcx, %rax
movq -0x58(%rbp), %rcx
movq %rcx, -0x48(%rbp)
jmp 0x17b27
xorl %r14d, %r14d
xorl %ebx, %ebx
cmpq $0x78, %rbx
je 0x17a85
leaq (%rbx,%rbp), %rdi
addq $-0xf0, %rdi
callq 0x11873
addq $0x18, %rbx
jmp 0x17c14