text: /*
   * AltiVec optimizations for libjpeg-turbo
   *
   * Copyright (C) 2015, D. R. Commander.  All Rights Reserved.
   *
   * This software is provided 'as-is', without any express or implied
   * warranty.  In no event will the authors be held liable for any damages
   * arising from the use of this software.
   *
   * Permission is granted to anyone to use this software for any purpose,
   * including commercial applications, and to alter it and redistribute it
   * freely, subject to the following restrictions:
   *
   * 1. The origin of this software must not be misrepresented; you must not
   *    claim that you wrote the original software. If you use this software
   *    in a product, an acknowledgment in the product documentation would be
   *    appreciated but is not required.
   * 2. Altered source versions must be plainly marked as such, and must not be
   *    misrepresented as being the original software.
   * 3. This notice may not be removed or altered from any source distribution.
   */
  
  /* CHROMA UPSAMPLING */
  
  #include "jsimd_altivec.h"
  
  
  void
  jsimd_h2v1_fancy_upsample_altivec (int max_v_samp_factor,
                                     JDIMENSION downsampled_width,
                                     JSAMPARRAY input_data,
                                     JSAMPARRAY *output_data_ptr)
  {
    JSAMPARRAY output_data = *output_data_ptr;
    JSAMPROW inptr, outptr;
    int inrow, incol;
  
    __vector unsigned char this0, last0, p_last0, next0 = {0}, p_next0,
      out;
    __vector short this0e, this0o, this0l, this0h, last0l, last0h,
      next0l, next0h, outle, outhe, outlo, outho;
  
    /* Constants */
    __vector unsigned char pb_zero = { __16X(0) }, pb_three = { __16X(3) },
      last_index_col0 = {0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14},
      last_index = {15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30},
      next_index = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16},
      next_index_lastcol = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,15},
  #if __BIG_ENDIAN__
      merge_pack_index = {1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31};
  #else
      merge_pack_index = {0,16,2,18,4,20,6,22,8,24,10,26,12,28,14,30};
  #endif
    __vector short pw_one = { __8X(1) }, pw_two = { __8X(2) };
  
    for (inrow = 0; inrow < max_v_samp_factor; inrow++) {
      inptr = input_data[inrow];
      outptr = output_data[inrow];
  
      if (downsampled_width & 15)
        inptr[downsampled_width] = inptr[downsampled_width - 1];
  
      this0 = vec_ld(0, inptr);
      p_last0 = vec_perm(this0, this0, last_index_col0);
      last0 = this0;
  
      for (incol = downsampled_width; incol > 0;
           incol -= 16, inptr += 16, outptr += 32) {
  
        if (downsampled_width - incol > 0) {
          p_last0 = vec_perm(last0, this0, last_index);
          last0 = this0;
        }
  
        if (incol <= 16)
          p_next0 = vec_perm(this0, this0, next_index_lastcol);
        else {
          next0 = vec_ld(16, inptr);
          p_next0 = vec_perm(this0, next0, next_index);
        }
  
        this0e = (__vector short)vec_mule(this0, pb_three);
        this0o = (__vector short)vec_mulo(this0, pb_three);
        this0l = vec_mergeh(this0e, this0o);
        this0h = vec_mergel(this0e, this0o);
  
        last0l = (__vector short)VEC_UNPACKHU(p_last0);
        last0h = (__vector short)VEC_UNPACKLU(p_last0);
        last0l = vec_add(last0l, pw_one);
  
        next0l = (__vector short)VEC_UNPACKHU(p_next0);
        next0h = (__vector short)VEC_UNPACKLU(p_next0);
        next0l = vec_add(next0l, pw_two);
  
        outle = vec_add(this0l, last0l);
        outlo = vec_add(this0l, next0l);
        outle = vec_sr(outle, (__vector unsigned short)pw_two);
        outlo = vec_sr(outlo, (__vector unsigned short)pw_two);
  
        out = vec_perm((__vector unsigned char)outle,
                       (__vector unsigned char)outlo, merge_pack_index);
        vec_st(out, 0, outptr);
  
        if (incol > 8) {
          last0h = vec_add(last0h, pw_one);
          next0h = vec_add(next0h, pw_two);
  
          outhe = vec_add(this0h, last0h);
          outho = vec_add(this0h, next0h);
          outhe = vec_sr(outhe, (__vector unsigned short)pw_two);
          outho = vec_sr(outho, (__vector unsigned short)pw_two);
  
          out = vec_perm((__vector unsigned char)outhe,
                         (__vector unsigned char)outho, merge_pack_index);
          vec_st(out, 16, outptr);
        }
  
        this0 = next0;
      }
    }
  }
  
  
  void
  jsimd_h2v2_fancy_upsample_altivec (int max_v_samp_factor,
                                     JDIMENSION downsampled_width,
                                     JSAMPARRAY input_data,
                                     JSAMPARRAY *output_data_ptr)
  {
    JSAMPARRAY output_data = *output_data_ptr;
    JSAMPROW inptr_1, inptr0, inptr1, outptr0, outptr1;
    int inrow, outrow, incol;
  
    __vector unsigned char this_1, this0, this1, out;
    __vector short this_1l, this_1h, this0l, this0h, this1l, this1h,
      lastcolsum_1h, lastcolsum1h,
      p_lastcolsum_1l, p_lastcolsum_1h, p_lastcolsum1l, p_lastcolsum1h,
      thiscolsum_1l, thiscolsum_1h, thiscolsum1l, thiscolsum1h,
      nextcolsum_1l = {0}, nextcolsum_1h = {0},
      nextcolsum1l = {0}, nextcolsum1h = {0},
      p_nextcolsum_1l, p_nextcolsum_1h, p_nextcolsum1l, p_nextcolsum1h,
      tmpl, tmph, outle, outhe, outlo, outho;
  
    /* Constants */
    __vector unsigned char pb_zero = { __16X(0) },
      last_index_col0 = {0,1,0,1,2,3,4,5,6,7,8,9,10,11,12,13},
      last_index={14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29},
      next_index = {2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17},
      next_index_lastcol = {2,3,4,5,6,7,8,9,10,11,12,13,14,15,14,15},
  #if __BIG_ENDIAN__
      merge_pack_index = {1,17,3,19,5,21,7,23,9,25,11,27,13,29,15,31};
  #else
      merge_pack_index = {0,16,2,18,4,20,6,22,8,24,10,26,12,28,14,30};
  #endif
    __vector short pw_zero = { __8X(0) }, pw_three = { __8X(3) },
      pw_seven = { __8X(7) }, pw_eight = { __8X(8) };
    __vector unsigned short pw_four = { __8X(4) };
  
    for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) {
  
      inptr_1 = input_data[inrow - 1];
      inptr0 = input_data[inrow];
      inptr1 = input_data[inrow + 1];
      outptr0 = output_data[outrow++];
      outptr1 = output_data[outrow++];
  
      if (downsampled_width & 15) {
        inptr_1[downsampled_width] = inptr_1[downsampled_width - 1];
        inptr0[downsampled_width] = inptr0[downsampled_width - 1];
        inptr1[downsampled_width] = inptr1[downsampled_width - 1];
      }
  
      this0 = vec_ld(0, inptr0);
      this0l = (__vector short)VEC_UNPACKHU(this0);
      this0h = (__vector short)VEC_UNPACKLU(this0);
      this0l = vec_mladd(this0l, pw_three, pw_zero);
      this0h = vec_mladd(this0h, pw_three, pw_zero);
  
      this_1 = vec_ld(0, inptr_1);
      this_1l = (__vector short)VEC_UNPACKHU(this_1);
      this_1h = (__vector short)VEC_UNPACKLU(this_1);
      thiscolsum_1l = vec_add(this0l, this_1l);
      thiscolsum_1h = vec_add(this0h, this_1h);
      lastcolsum_1h = thiscolsum_1h;
      p_lastcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1l, last_index_col0);
      p_lastcolsum_1h = vec_perm(thiscolsum_1l, thiscolsum_1h, last_index);
  
      this1 = vec_ld(0, inptr1);
      this1l = (__vector short)VEC_UNPACKHU(this1);
      this1h = (__vector short)VEC_UNPACKLU(this1);
      thiscolsum1l = vec_add(this0l, this1l);
      thiscolsum1h = vec_add(this0h, this1h);
      lastcolsum1h = thiscolsum1h;
      p_lastcolsum1l = vec_perm(thiscolsum1l, thiscolsum1l, last_index_col0);
      p_lastcolsum1h = vec_perm(thiscolsum1l, thiscolsum1h, last_index);
  
      for (incol = downsampled_width; incol > 0;
           incol -= 16, inptr_1 += 16, inptr0 += 16, inptr1 += 16,
           outptr0 += 32, outptr1 += 32) {
  
        if (downsampled_width - incol > 0) {
          p_lastcolsum_1l = vec_perm(lastcolsum_1h, thiscolsum_1l, last_index);
          p_lastcolsum_1h = vec_perm(thiscolsum_1l, thiscolsum_1h, last_index);
          p_lastcolsum1l = vec_perm(lastcolsum1h, thiscolsum1l, last_index);
          p_lastcolsum1h = vec_perm(thiscolsum1l, thiscolsum1h, last_index);
          lastcolsum_1h = thiscolsum_1h;  lastcolsum1h = thiscolsum1h;
        }
  
        if (incol <= 16) {
          p_nextcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1h, next_index);
          p_nextcolsum_1h = vec_perm(thiscolsum_1h, thiscolsum_1h,
                                     next_index_lastcol);
          p_nextcolsum1l = vec_perm(thiscolsum1l, thiscolsum1h, next_index);
          p_nextcolsum1h = vec_perm(thiscolsum1h, thiscolsum1h,
                                    next_index_lastcol);
        } else {
          this0 = vec_ld(16, inptr0);
          this0l = (__vector short)VEC_UNPACKHU(this0);
          this0h = (__vector short)VEC_UNPACKLU(this0);
          this0l = vec_mladd(this0l, pw_three, pw_zero);
          this0h = vec_mladd(this0h, pw_three, pw_zero);
  
          this_1 = vec_ld(16, inptr_1);
          this_1l = (__vector short)VEC_UNPACKHU(this_1);
          this_1h = (__vector short)VEC_UNPACKLU(this_1);
          nextcolsum_1l = vec_add(this0l, this_1l);
          nextcolsum_1h = vec_add(this0h, this_1h);
          p_nextcolsum_1l = vec_perm(thiscolsum_1l, thiscolsum_1h, next_index);
          p_nextcolsum_1h = vec_perm(thiscolsum_1h, nextcolsum_1l, next_index);
  
          this1 = vec_ld(16, inptr1);
          this1l = (__vector short)VEC_UNPACKHU(this1);
          this1h = (__vector short)VEC_UNPACKLU(this1);
          nextcolsum1l = vec_add(this0l, this1l);
          nextcolsum1h = vec_add(this0h, this1h);
          p_nextcolsum1l = vec_perm(thiscolsum1l, thiscolsum1h, next_index);
          p_nextcolsum1h = vec_perm(thiscolsum1h, nextcolsum1l, next_index);
        }
  
        /* Process the upper row */
  
        tmpl = vec_mladd(thiscolsum_1l, pw_three, pw_zero);
        outle = vec_add(tmpl, p_lastcolsum_1l);
        outle = vec_add(outle, pw_eight);
        outle = vec_sr(outle, pw_four);
  
        outlo = vec_add(tmpl, p_nextcolsum_1l);
        outlo = vec_add(outlo, pw_seven);
        outlo = vec_sr(outlo, pw_four);
  
        out = vec_perm((__vector unsigned char)outle,
                       (__vector unsigned char)outlo, merge_pack_index);
        vec_st(out, 0, outptr0);
  
        if (incol > 8) {
          tmph = vec_mladd(thiscolsum_1h, pw_three, pw_zero);
          outhe = vec_add(tmph, p_lastcolsum_1h);
          outhe = vec_add(outhe, pw_eight);
          outhe = vec_sr(outhe, pw_four);
  
          outho = vec_add(tmph, p_nextcolsum_1h);
          outho = vec_add(outho, pw_seven);
          outho = vec_sr(outho, pw_four);
  
          out = vec_perm((__vector unsigned char)outhe,
                         (__vector unsigned char)outho, merge_pack_index);
          vec_st(out, 16, outptr0);
        }
  
        /* Process the lower row */
  
        tmpl = vec_mladd(thiscolsum1l, pw_three, pw_zero);
        outle = vec_add(tmpl, p_lastcolsum1l);
        outle = vec_add(outle, pw_eight);
        outle = vec_sr(outle, pw_four);
  
        outlo = vec_add(tmpl, p_nextcolsum1l);
        outlo = vec_add(outlo, pw_seven);
        outlo = vec_sr(outlo, pw_four);
  
        out = vec_perm((__vector unsigned char)outle,
                       (__vector unsigned char)outlo, merge_pack_index);
        vec_st(out, 0, outptr1);
  
        if (incol > 8) {
          tmph = vec_mladd(thiscolsum1h, pw_three, pw_zero);
          outhe = vec_add(tmph, p_lastcolsum1h);
          outhe = vec_add(outhe, pw_eight);
          outhe = vec_sr(outhe, pw_four);
  
          outho = vec_add(tmph, p_nextcolsum1h);
          outho = vec_add(outho, pw_seven);
          outho = vec_sr(outho, pw_four);
  
          out = vec_perm((__vector unsigned char)outhe,
                         (__vector unsigned char)outho, merge_pack_index);
          vec_st(out, 16, outptr1);
        }
  
        thiscolsum_1l = nextcolsum_1l;  thiscolsum_1h = nextcolsum_1h;
        thiscolsum1l = nextcolsum1l;  thiscolsum1h = nextcolsum1h;
      }
    }
  }
  
  
  /* These are rarely used (mainly just for decompressing YCCK images) */
  
  void
  jsimd_h2v1_upsample_altivec (int max_v_samp_factor,
                               JDIMENSION output_width,
                               JSAMPARRAY input_data,
                               JSAMPARRAY *output_data_ptr)
  {
    JSAMPARRAY output_data = *output_data_ptr;
    JSAMPROW inptr, outptr;
    int inrow, incol;
  
    __vector unsigned char in, inl, inh;
  
    for (inrow = 0; inrow < max_v_samp_factor; inrow++) {
      inptr = input_data[inrow];
      outptr = output_data[inrow];
  
      for (incol = (output_width + 31) & (~31); incol > 0;
           incol -= 64, inptr += 32, outptr += 64) {
  
        in = vec_ld(0, inptr);
        inl = vec_mergeh(in, in);
        inh = vec_mergel(in, in);
  
        vec_st(inl, 0, outptr);
        vec_st(inh, 16, outptr);
  
        if (incol > 32) {
          in = vec_ld(16, inptr);
          inl = vec_mergeh(in, in);
          inh = vec_mergel(in, in);
  
          vec_st(inl, 32, outptr);
          vec_st(inh, 48, outptr);
        }
      }
    }
  }
  
  
  void
  jsimd_h2v2_upsample_altivec (int max_v_samp_factor,
                               JDIMENSION output_width,
                               JSAMPARRAY input_data,
                               JSAMPARRAY *output_data_ptr)
  {
    JSAMPARRAY output_data = *output_data_ptr;
    JSAMPROW inptr, outptr0, outptr1;
    int inrow, outrow, incol;
  
    __vector unsigned char in, inl, inh;
  
    for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) {
  
      inptr = input_data[inrow];
      outptr0 = output_data[outrow++];
      outptr1 = output_data[outrow++];
  
      for (incol = (output_width + 31) & (~31); incol > 0;
           incol -= 64, inptr += 32, outptr0 += 64, outptr1 += 64) {
  
        in = vec_ld(0, inptr);
        inl = vec_mergeh(in, in);
        inh = vec_mergel(in, in);
  
        vec_st(inl, 0, outptr0);
        vec_st(inl, 0, outptr1);
  
        vec_st(inh, 16, outptr0);
        vec_st(inh, 16, outptr1);
  
        if (incol > 32) {
          in = vec_ld(16, inptr);
          inl = vec_mergeh(in, in);
          inh = vec_mergel(in, in);
  
          vec_st(inl, 32, outptr0);
          vec_st(inl, 32, outptr1);
  
          vec_st(inh, 48, outptr0);
          vec_st(inh, 48, outptr1);
        }
      }
    }
  }
  